From 97deca0f57f64c6957c2a0aa88d4a0f9074f39c5 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 26 May 2026 14:15:26 +0800 Subject: [PATCH] Update LLM configuration to support FastGPT integration. Modify requirements to include fastgpt-python-sdk, enhance greeting messages, and adjust LLM service creation to handle app_id. Implement welcome text fetching for FastGPT and improve context handling in the pipeline based on LLM provider. Update related configurations and properties for better integration. --- config/fastgpt.example.json | 44 +++++++++++------- engine/config.py | 37 +++++++++++++++ engine/fastgpt_llm.py | 90 ++++++++++++++++++++++++++++++++++++- engine/main.py | 3 ++ engine/pipeline.py | 19 +++++--- engine/services.py | 9 +++- requirements.txt | 2 +- 7 files changed, 180 insertions(+), 24 deletions(-) diff --git a/config/fastgpt.example.json b/config/fastgpt.example.json index 78a8ce8..fce3aa7 100644 --- a/config/fastgpt.example.json +++ b/config/fastgpt.example.json @@ -14,36 +14,50 @@ }, "agent": { "system_prompt": "FastGPT app owns the system prompt when send_system_prompt is false.", - "greeting": "你好", - "greeting_mode": "generated" + "greeting": "您好，这里是无锡交警，我将为您远程处理交通事故。请将人员撤离至路侧安全区域，开启危险报警双闪灯、放置三角警告牌、做好安全防护，谨防二次事故伤害。若您已经准备好了，请点击继续办理，如需人工服务，请说转人工。", + "greeting_mode": "fixed" }, "services": { "stt": { - "provider": "openai", - "api_key": "YOUR_STT_KEY", - "base_url": "https://api.openai.com/v1", - "model": "gpt-4o-mini-transcribe", - "language": "zh" + "provider": "xfyun", + "app_id": "416ce125", + "api_key": "c65342fe603126c3610031d8429bb36d", + "api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4", + "base_url": "wss://iat-api.xfyun.cn/v2/iat", + "language": "zh_cn", + "domain": "iat", + "accent": "mandarin", + "encoding": "raw", + "frame_size": 1280, + "timeout_sec": 10.0 }, "llm": { "provider": "fastgpt", - "api_key": "fastgpt-xxxxx", - "base_url": "http://localhost:3000", + "api_key": "fastgpt-gPywgEQUC0BPEZxmKowfQOOsLNHcn9hthEVUH21ZsNld1v4IzvRakT6r", + "base_url": "http://localhost:3030", "model": "my-voice-app", + "app_id": "6a139bdd53e3f8d9f274230b", "chat_id": null, "variables": { - "user_name": "访客" + "user_name": "Alex" }, "detail": false, "timeout_sec": 60.0, "send_system_prompt": false }, "tts": { - "provider": "openai", - "api_key": "YOUR_TTS_KEY", - "base_url": "https://api.openai.com/v1", - "model": "gpt-4o-mini-tts", - "voice": "alloy" + "provider": "xfyun", + "app_id": "416ce125", + "api_key": "c65342fe603126c3610031d8429bb36d", + "api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4", + "base_url": "wss://tts-api.xfyun.cn/v2/tts", + "voice": "x4_xiaoyan", + "aue": "raw", + "tte": "UTF8", + "speed": 50, + "volume": 50, + "pitch": 50, + "source_sample_rate_hz": 16000 } } } diff --git a/engine/config.py b/engine/config.py index 4971323..acd3875 100644 --- a/engine/config.py +++ b/engine/config.py @@ -4,6 +4,9 @@ import json from dataclasses import dataclass, field from pathlib import Path +SUPPORTED_LLM_PROVIDERS = frozenset({"openai", "fastgpt"}) +_LLM_PROVIDER_ALIASES = {"llm": "openai", "openai": "openai", "fastgpt": "fastgpt"} + @dataclass(frozen=True) class ServerConfig: @@ -99,10 +102,17 @@ class AgentConfig: @dataclass(frozen=True) class LLMConfig: + """LLM backend selection via ``provider``. + + Set ``provider`` to ``"openai"`` (alias ``"llm"``) for OpenAI-compatible chat + completions, or ``"fastgpt"`` for FastGPT server-side memory via ``chat_id``. + """ + provider: str = "openai" api_key: str = "" base_url: str | None = None model: str = "gpt-4o-mini" + app_id: str | None = None temperature: float | None = 0.7 chat_id: str | None = None variables: dict[str, str] = field(default_factory=dict) @@ -110,6 +120,19 @@ class LLMConfig: timeout_sec: float = 60.0 send_system_prompt: bool = False + @property + def is_fastgpt(self) -> bool: + return self.provider == "fastgpt" + + @property + def is_openai(self) -> bool: + return self.provider == "openai" + + @property + def uses_local_context_history(self) -> bool: + """Whether the pipeline should seed and maintain local LLM context history.""" + return not self.is_fastgpt or self.send_system_prompt + @dataclass(frozen=True) class STTConfig: @@ -186,8 +209,11 @@ def config_from_dict(data: dict) -> EngineConfig: stt["language"] = None llm = _dict(services.get("llm")) + llm["provider"] = _normalize_llm_provider(llm.get("provider", LLMConfig().provider)) if llm.get("chat_id") == "": llm["chat_id"] = None + if llm.get("app_id") == "": + llm["app_id"] = None if not isinstance(llm.get("variables"), dict): llm["variables"] = {} @@ -227,3 +253,14 @@ def config_from_dict(data: dict) -> EngineConfig: def _dict(value: object) -> dict: return dict(value) if isinstance(value, dict) else {} + + +def _normalize_llm_provider(value: object) -> str: + provider = str(value or LLMConfig().provider).strip().lower() + normalized = _LLM_PROVIDER_ALIASES.get(provider) + if normalized is None: + supported = ", ".join(sorted(SUPPORTED_LLM_PROVIDERS | {"llm"})) + raise ValueError( + f"services.llm.provider must be one of: {supported}; got {value!r}" + ) + return normalized diff --git a/engine/fastgpt_llm.py b/engine/fastgpt_llm.py index 862091e..b05a0f2 100644 --- a/engine/fastgpt_llm.py +++ b/engine/fastgpt_llm.py @@ -13,6 +13,7 @@ from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, + InterruptionFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -134,6 +135,24 @@ class FastGPTLLMSettings(LLMSettings): detail: bool = False +def _default_fastgpt_settings(*, model: str = "fastgpt") -> FastGPTLLMSettings: + return FastGPTLLMSettings( + model=model, + system_instruction=None, + temperature=None, + max_tokens=None, + top_p=None, + top_k=None, + frequency_penalty=None, + presence_penalty=None, + seed=None, + filter_incomplete_user_turns=False, + user_turn_completion_config=None, + variables={}, + detail=False, + ) + + class FastGPTLLMService(LLMService): """FastGPT LLM service using chatId server-side memory and workflow variables.""" @@ -145,18 +164,20 @@ class FastGPTLLMService(LLMService): api_key: str, base_url: str, chat_id: str | None = None, + app_id: str | None = None, send_system_prompt: bool = False, greeting_prompt: str | None = None, timeout: float = 60.0, settings: FastGPTLLMSettings | None = None, **kwargs, ) -> None: - default_settings = self.Settings(model="fastgpt") + default_settings = _default_fastgpt_settings() if settings is not None: default_settings.apply_update(settings) super().__init__(settings=default_settings, **kwargs) self._chat_id = chat_id or f"voice_{uuid.uuid4().hex[:16]}" + self._app_id = (app_id or "").strip() self._send_system_prompt = send_system_prompt self._greeting_prompt = (greeting_prompt or "你好").strip() or "你好" self._client = AsyncChatClient( @@ -166,6 +187,10 @@ class FastGPTLLMService(LLMService): ) self._active_response = None + @property + def app_id(self) -> str: + return self._app_id + @property def chat_id(self) -> str: return self._chat_id @@ -184,6 +209,63 @@ class FastGPTLLMService(LLMService): await self._close_active_response() await super().cancel(frame) + async def _handle_interruptions(self, _: InterruptionFrame) -> None: + await self._close_active_response() + await super()._handle_interruptions(_) + + @staticmethod + def _welcome_text_from_init_payload(payload: Any) -> str: + if not isinstance(payload, dict): + return "" + + for container in (payload.get("app"), payload.get("data"), payload): + if not isinstance(container, dict): + continue + nested_app = container.get("app") + if isinstance(nested_app, dict): + text = FastGPTLLMService._welcome_text_from_app(nested_app) + if text: + return text + text = FastGPTLLMService._welcome_text_from_app(container) + if text: + return text + return "" + + @staticmethod + def _welcome_text_from_app(app_payload: dict[str, Any]) -> str: + chat_config = ( + app_payload.get("chatConfig") + if isinstance(app_payload.get("chatConfig"), dict) + else {} + ) + return _first_nonempty_text( + chat_config.get("welcomeText"), + app_payload.get("welcomeText"), + ) + + async def fetch_welcome_text(self) -> str | None: + """Return FastGPT app welcome text from chat init when ``app_id`` is configured.""" + if not self._app_id: + return None + + try: + response = await self._client.get_chat_init( + appId=self._app_id, + chatId=self._chat_id, + ) + response.raise_for_status() + text = self._welcome_text_from_init_payload(response.json()) + if text: + logger.info(f"FastGPT welcomeText loaded for appId={self._app_id}") + return text or None + except FastGPTError as exc: + logger.warning(f"FastGPT chat init failed: {exc}") + except httpx.HTTPError as exc: + logger.warning(f"FastGPT chat init HTTP error: {exc}") + except Exception as exc: + logger.warning(f"FastGPT chat init error: {exc}") + return None + async def _close_active_response(self) -> None: response = self._active_response self._active_response = None @@ -217,6 +299,12 @@ class FastGPTLLMService(LLMService): messages = self._build_fastgpt_messages(context) variables = self._settings.variables or None + logger.info( + "FastGPT chat completion " + f"chatId={self._chat_id} appId={self._app_id or '-'} " + f"variables={sorted((variables or {}).keys())} messages={messages!r}" + ) + await self.start_ttfb_metrics() try: diff --git a/engine/main.py b/engine/main.py index 2e8dd95..d7e49a7 100644 --- a/engine/main.py +++ b/engine/main.py @@ -59,6 +59,9 @@ def create_app(config_path: str = "config.json") -> FastAPI: "product_image_input": True, }, "demo": webpage_mount, + "llm_backend": ( + "fastgpt" if config.services.llm.is_fastgpt else "openai" + ), "llm_provider": config.services.llm.provider, "stt_provider": config.services.stt.provider, "tts_provider": config.services.tts.provider, diff --git a/engine/pipeline.py b/engine/pipeline.py index 562fd73..ccf13e4 100644 --- a/engine/pipeline.py +++ b/engine/pipeline.py @@ -34,6 +34,7 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies from .config import EngineConfig from .context_sync import AssistantContextSyncProcessor +from .fastgpt_llm import FastGPTLLMService from .product_protocol import ProductWebsocketSerializer from .services import create_llm_service, create_stt_service, create_tts_service from .text_input import ProductTextInputProcessor @@ -94,14 +95,15 @@ async def run_pipeline_with_serializer( session_variables={"session_id": chat_id, "channel": "voice"}, greeting_prompt=config.agent.greeting, ) - if llm_config.provider == "fastgpt": - logger.info(f"FastGPT chatId={chat_id}") + if llm_config.is_fastgpt: + logger.info(f"LLM backend=fastgpt chatId={chat_id} appId={llm_config.app_id or '-'}") + else: + logger.info(f"LLM backend=openai model={llm_config.model}") tts = create_tts_service(config.services.tts, config.audio) - use_fastgpt = llm_config.provider == "fastgpt" and not llm_config.send_system_prompt messages: list[dict[str, str]] = [] - if not use_fastgpt: + if llm_config.uses_local_context_history: messages = [{"role": "system", "content": config.agent.system_prompt}] if config.agent.greeting and config.agent.greeting_mode == "generated": messages.append({"role": "system", "content": config.agent.greeting}) @@ -183,7 +185,14 @@ async def run_pipeline_with_serializer( if config.agent.greeting_mode == "fixed" and config.agent.greeting: await task.queue_frames([TTSSpeakFrame(config.agent.greeting)]) elif config.agent.greeting_mode == "generated": - await task.queue_frames([LLMRunFrame()]) + if isinstance(llm, FastGPTLLMService): + welcome = await llm.fetch_welcome_text() + if welcome: + await task.queue_frames([TTSSpeakFrame(welcome)]) + else: + await task.queue_frames([LLMRunFrame()]) + else: + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(_transport, _client): diff --git a/engine/services.py b/engine/services.py index 4272003..d25cf41 100644 --- a/engine/services.py +++ b/engine/services.py @@ -54,12 +54,13 @@ def create_llm_service( session_variables: dict | None = None, greeting_prompt: str | None = None, ): - if config.provider == "fastgpt": + if config.is_fastgpt: variables = {**config.variables, **(session_variables or {})} return FastGPTLLMService( api_key=config.api_key, base_url=config.base_url or "http://localhost:3000", chat_id=chat_id or config.chat_id, + app_id=config.app_id, send_system_prompt=config.send_system_prompt, greeting_prompt=greeting_prompt, timeout=config.timeout_sec, @@ -70,7 +71,11 @@ def create_llm_service( ), ) - _require_provider(config.provider, "openai", "llm") + if not config.is_openai: + supported = ", ".join(sorted(("openai", "fastgpt", "llm"))) + raise ValueError( + f"Unsupported llm provider {config.provider!r}; expected one of: {supported}" + ) return OpenAILLMService( api_key=config.api_key or None, base_url=config.base_url, diff --git a/requirements.txt b/requirements.txt index ab1b369..9cd62cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ fastapi>=0.115.6,<1 uvicorn[standard]>=0.32.0,<1 -e ../pipecat[websocket,openai,silero] - +-e ../fastgpt-python-sdk