Update LLM configuration to support FastGPT integration. Modify requirements to include fastgpt-python-sdk, enhance greeting messages, and adjust LLM service creation to handle app_id. Implement welcome text fetching for FastGPT and improve context handling in the pipeline based on LLM provider. Update related configurations and properties for better integration.

config/fastgpt.example.json

@@ -14,36 +14,50 @@
   },
   "agent": {
     "system_prompt": "FastGPT app owns the system prompt when send_system_prompt is false.",
-    "greeting": "你好",
+    "greeting": "您好，这里是无锡交警，我将为您远程处理交通事故。请将人员撤离至路侧安全区域，开启危险报警双闪灯、放置三角警告牌、做好安全防护，谨防二次事故伤害。若您已经准备好了，请点击继续办理，如需人工服务，请说转人工。",
-    "greeting_mode": "generated"
+    "greeting_mode": "fixed"
   },
   "services": {
     "stt": {
-      "provider": "openai",
+      "provider": "xfyun",
-      "api_key": "YOUR_STT_KEY",
+      "app_id": "416ce125",
-      "base_url": "https://api.openai.com/v1",
+      "api_key": "c65342fe603126c3610031d8429bb36d",
-      "model": "gpt-4o-mini-transcribe",
+      "api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
-      "language": "zh"
+      "base_url": "wss://iat-api.xfyun.cn/v2/iat",
+      "language": "zh_cn",
+      "domain": "iat",
+      "accent": "mandarin",
+      "encoding": "raw",
+      "frame_size": 1280,
+      "timeout_sec": 10.0
     },
     "llm": {
       "provider": "fastgpt",
-      "api_key": "fastgpt-xxxxx",
+      "api_key": "fastgpt-gPywgEQUC0BPEZxmKowfQOOsLNHcn9hthEVUH21ZsNld1v4IzvRakT6r",
-      "base_url": "http://localhost:3000",
+      "base_url": "http://localhost:3030",
       "model": "my-voice-app",
+      "app_id": "6a139bdd53e3f8d9f274230b",
       "chat_id": null,
       "variables": {
-        "user_name": "访客"
+        "user_name": "Alex"
       },
       "detail": false,
       "timeout_sec": 60.0,
       "send_system_prompt": false
     },
     "tts": {
-      "provider": "openai",
+      "provider": "xfyun",
-      "api_key": "YOUR_TTS_KEY",
+      "app_id": "416ce125",
-      "base_url": "https://api.openai.com/v1",
+      "api_key": "c65342fe603126c3610031d8429bb36d",
-      "model": "gpt-4o-mini-tts",
+      "api_secret": "MzkyYmI5OWEyODQzN2FiN2VhN2UzYzU4",
-      "voice": "alloy"
+      "base_url": "wss://tts-api.xfyun.cn/v2/tts",
+      "voice": "x4_xiaoyan",
+      "aue": "raw",
+      "tte": "UTF8",
+      "speed": 50,
+      "volume": 50,
+      "pitch": 50,
+      "source_sample_rate_hz": 16000
     }
   }
 }

engine/config.py

@@ -4,6 +4,9 @@ import json
 from dataclasses import dataclass, field
 from pathlib import Path
 
+SUPPORTED_LLM_PROVIDERS = frozenset({"openai", "fastgpt"})
+_LLM_PROVIDER_ALIASES = {"llm": "openai", "openai": "openai", "fastgpt": "fastgpt"}
+
 
 @dataclass(frozen=True)
 class ServerConfig:
@@ -99,10 +102,17 @@ class AgentConfig:
 
 @dataclass(frozen=True)
 class LLMConfig:
+    """LLM backend selection via ``provider``.
+
+    Set ``provider`` to ``"openai"`` (alias ``"llm"``) for OpenAI-compatible chat
+    completions, or ``"fastgpt"`` for FastGPT server-side memory via ``chat_id``.
+    """
+
     provider: str = "openai"
     api_key: str = ""
     base_url: str | None = None
     model: str = "gpt-4o-mini"
+    app_id: str | None = None
     temperature: float | None = 0.7
     chat_id: str | None = None
     variables: dict[str, str] = field(default_factory=dict)
@@ -110,6 +120,19 @@ class LLMConfig:
     timeout_sec: float = 60.0
     send_system_prompt: bool = False
 
+    @property
+    def is_fastgpt(self) -> bool:
+        return self.provider == "fastgpt"
+
+    @property
+    def is_openai(self) -> bool:
+        return self.provider == "openai"
+
+    @property
+    def uses_local_context_history(self) -> bool:
+        """Whether the pipeline should seed and maintain local LLM context history."""
+        return not self.is_fastgpt or self.send_system_prompt
+
 
 @dataclass(frozen=True)
 class STTConfig:
@@ -186,8 +209,11 @@ def config_from_dict(data: dict) -> EngineConfig:
         stt["language"] = None
 
     llm = _dict(services.get("llm"))
+    llm["provider"] = _normalize_llm_provider(llm.get("provider", LLMConfig().provider))
     if llm.get("chat_id") == "":
         llm["chat_id"] = None
+    if llm.get("app_id") == "":
+        llm["app_id"] = None
     if not isinstance(llm.get("variables"), dict):
         llm["variables"] = {}
 
@@ -227,3 +253,14 @@ def config_from_dict(data: dict) -> EngineConfig:
 
 def _dict(value: object) -> dict:
     return dict(value) if isinstance(value, dict) else {}
+
+
+def _normalize_llm_provider(value: object) -> str:
+    provider = str(value or LLMConfig().provider).strip().lower()
+    normalized = _LLM_PROVIDER_ALIASES.get(provider)
+    if normalized is None:
+        supported = ", ".join(sorted(SUPPORTED_LLM_PROVIDERS | {"llm"}))
+        raise ValueError(
+            f"services.llm.provider must be one of: {supported}; got {value!r}"
+        )
+    return normalized

engine/fastgpt_llm.py

@@ -13,6 +13,7 @@ from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
     Frame,
+    InterruptionFrame,
     LLMContextFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
@@ -134,6 +135,24 @@ class FastGPTLLMSettings(LLMSettings):
     detail: bool = False
 
 
+def _default_fastgpt_settings(*, model: str = "fastgpt") -> FastGPTLLMSettings:
+    return FastGPTLLMSettings(
+        model=model,
+        system_instruction=None,
+        temperature=None,
+        max_tokens=None,
+        top_p=None,
+        top_k=None,
+        frequency_penalty=None,
+        presence_penalty=None,
+        seed=None,
+        filter_incomplete_user_turns=False,
+        user_turn_completion_config=None,
+        variables={},
+        detail=False,
+    )
+
+
 class FastGPTLLMService(LLMService):
     """FastGPT LLM service using chatId server-side memory and workflow variables."""
 
@@ -145,18 +164,20 @@ class FastGPTLLMService(LLMService):
         api_key: str,
         base_url: str,
         chat_id: str | None = None,
+        app_id: str | None = None,
         send_system_prompt: bool = False,
         greeting_prompt: str | None = None,
         timeout: float = 60.0,
         settings: FastGPTLLMSettings | None = None,
         **kwargs,
     ) -> None:
-        default_settings = self.Settings(model="fastgpt")
+        default_settings = _default_fastgpt_settings()
         if settings is not None:
             default_settings.apply_update(settings)
         super().__init__(settings=default_settings, **kwargs)
 
         self._chat_id = chat_id or f"voice_{uuid.uuid4().hex[:16]}"
+        self._app_id = (app_id or "").strip()
         self._send_system_prompt = send_system_prompt
         self._greeting_prompt = (greeting_prompt or "你好").strip() or "你好"
         self._client = AsyncChatClient(
@@ -166,6 +187,10 @@ class FastGPTLLMService(LLMService):
         )
         self._active_response = None
 
+    @property
+    def app_id(self) -> str:
+        return self._app_id
+
     @property
     def chat_id(self) -> str:
         return self._chat_id
@@ -184,6 +209,63 @@ class FastGPTLLMService(LLMService):
         await self._close_active_response()
         await super().cancel(frame)
 
+    async def _handle_interruptions(self, _: InterruptionFrame) -> None:
+        await self._close_active_response()
+        await super()._handle_interruptions(_)
+
+    @staticmethod
+    def _welcome_text_from_init_payload(payload: Any) -> str:
+        if not isinstance(payload, dict):
+            return ""
+
+        for container in (payload.get("app"), payload.get("data"), payload):
+            if not isinstance(container, dict):
+                continue
+            nested_app = container.get("app")
+            if isinstance(nested_app, dict):
+                text = FastGPTLLMService._welcome_text_from_app(nested_app)
+                if text:
+                    return text
+            text = FastGPTLLMService._welcome_text_from_app(container)
+            if text:
+                return text
+        return ""
+
+    @staticmethod
+    def _welcome_text_from_app(app_payload: dict[str, Any]) -> str:
+        chat_config = (
+            app_payload.get("chatConfig")
+            if isinstance(app_payload.get("chatConfig"), dict)
+            else {}
+        )
+        return _first_nonempty_text(
+            chat_config.get("welcomeText"),
+            app_payload.get("welcomeText"),
+        )
+
+    async def fetch_welcome_text(self) -> str | None:
+        """Return FastGPT app welcome text from chat init when ``app_id`` is configured."""
+        if not self._app_id:
+            return None
+
+        try:
+            response = await self._client.get_chat_init(
+                appId=self._app_id,
+                chatId=self._chat_id,
+            )
+            response.raise_for_status()
+            text = self._welcome_text_from_init_payload(response.json())
+            if text:
+                logger.info(f"FastGPT welcomeText loaded for appId={self._app_id}")
+            return text or None
+        except FastGPTError as exc:
+            logger.warning(f"FastGPT chat init failed: {exc}")
+        except httpx.HTTPError as exc:
+            logger.warning(f"FastGPT chat init HTTP error: {exc}")
+        except Exception as exc:
+            logger.warning(f"FastGPT chat init error: {exc}")
+        return None
+
     async def _close_active_response(self) -> None:
         response = self._active_response
         self._active_response = None
@@ -217,6 +299,12 @@ class FastGPTLLMService(LLMService):
         messages = self._build_fastgpt_messages(context)
         variables = self._settings.variables or None
 
+        logger.info(
+            "FastGPT chat completion "
+            f"chatId={self._chat_id} appId={self._app_id or '-'} "
+            f"variables={sorted((variables or {}).keys())} messages={messages!r}"
+        )
+
         await self.start_ttfb_metrics()
 
         try:

engine/main.py

@@ -59,6 +59,9 @@ def create_app(config_path: str = "config.json") -> FastAPI:
                 "product_image_input": True,
             },
             "demo": webpage_mount,
+            "llm_backend": (
+                "fastgpt" if config.services.llm.is_fastgpt else "openai"
+            ),
             "llm_provider": config.services.llm.provider,
             "stt_provider": config.services.stt.provider,
             "tts_provider": config.services.tts.provider,

engine/pipeline.py

@@ -34,6 +34,7 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies
 
 from .config import EngineConfig
 from .context_sync import AssistantContextSyncProcessor
+from .fastgpt_llm import FastGPTLLMService
 from .product_protocol import ProductWebsocketSerializer
 from .services import create_llm_service, create_stt_service, create_tts_service
 from .text_input import ProductTextInputProcessor
@@ -94,14 +95,15 @@ async def run_pipeline_with_serializer(
         session_variables={"session_id": chat_id, "channel": "voice"},
         greeting_prompt=config.agent.greeting,
     )
-    if llm_config.provider == "fastgpt":
+    if llm_config.is_fastgpt:
-        logger.info(f"FastGPT chatId={chat_id}")
+        logger.info(f"LLM backend=fastgpt chatId={chat_id} appId={llm_config.app_id or '-'}")
+    else:
+        logger.info(f"LLM backend=openai model={llm_config.model}")
 
     tts = create_tts_service(config.services.tts, config.audio)
 
-    use_fastgpt = llm_config.provider == "fastgpt" and not llm_config.send_system_prompt
     messages: list[dict[str, str]] = []
-    if not use_fastgpt:
+    if llm_config.uses_local_context_history:
         messages = [{"role": "system", "content": config.agent.system_prompt}]
         if config.agent.greeting and config.agent.greeting_mode == "generated":
             messages.append({"role": "system", "content": config.agent.greeting})
@@ -183,7 +185,14 @@ async def run_pipeline_with_serializer(
         if config.agent.greeting_mode == "fixed" and config.agent.greeting:
             await task.queue_frames([TTSSpeakFrame(config.agent.greeting)])
         elif config.agent.greeting_mode == "generated":
-            await task.queue_frames([LLMRunFrame()])
+            if isinstance(llm, FastGPTLLMService):
+                welcome = await llm.fetch_welcome_text()
+                if welcome:
+                    await task.queue_frames([TTSSpeakFrame(welcome)])
+                else:
+                    await task.queue_frames([LLMRunFrame()])
+            else:
+                await task.queue_frames([LLMRunFrame()])
 
     @transport.event_handler("on_client_disconnected")
     async def on_client_disconnected(_transport, _client):

engine/services.py

@@ -54,12 +54,13 @@ def create_llm_service(
     session_variables: dict | None = None,
     greeting_prompt: str | None = None,
 ):
-    if config.provider == "fastgpt":
+    if config.is_fastgpt:
         variables = {**config.variables, **(session_variables or {})}
         return FastGPTLLMService(
             api_key=config.api_key,
             base_url=config.base_url or "http://localhost:3000",
             chat_id=chat_id or config.chat_id,
+            app_id=config.app_id,
             send_system_prompt=config.send_system_prompt,
             greeting_prompt=greeting_prompt,
             timeout=config.timeout_sec,
@@ -70,7 +71,11 @@ def create_llm_service(
             ),
         )
 
-    _require_provider(config.provider, "openai", "llm")
+    if not config.is_openai:
+        supported = ", ".join(sorted(("openai", "fastgpt", "llm")))
+        raise ValueError(
+            f"Unsupported llm provider {config.provider!r}; expected one of: {supported}"
+        )
     return OpenAILLMService(
         api_key=config.api_key or None,
         base_url=config.base_url,

requirements.txt

@@ -1,4 +1,4 @@
 fastapi>=0.115.6,<1
 uvicorn[standard]>=0.32.0,<1
 -e ../pipecat[websocket,openai,silero]
-
+-e ../fastgpt-python-sdk