Implement StepFun Realtime service and enhance AssistantConfig

- Add new fields to AssistantConfig for realtime interface configuration, including types, values, and secrets.
- Introduce StepFunRealtimeService to handle speech-to-speech processing via WebSocket, integrating STT, LLM, and TTS functionalities.
- Refactor pipeline execution to support a new realtime mode, allowing direct text input processing and immediate responses.
- Update model resource testing to include validation for StepFun Realtime connections.
- Enhance service factory to create realtime services based on configuration settings.
- Modify README documentation to reflect new realtime capabilities and usage instructions.

backend/services/pipecat/pipeline.py

@@ -10,7 +10,7 @@ from uuid import uuid4
 
 from loguru import logger
 from models import AssistantConfig
-from services.pipecat.service_factory import create_services
+from services.pipecat.service_factory import create_realtime_service, create_services
 
 from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.frames.frames import (
@@ -106,6 +106,33 @@ class TextInputProcessor(FrameProcessor):
             await self._call_event_handler("on_text_append", text)
 
 
+class RealtimeTextInputProcessor(FrameProcessor):
+    """Route text input directly to a realtime service without cascade semantics."""
+
+    def __init__(self):
+        super().__init__()
+        self._register_event_handler("on_text_input")
+        self._register_event_handler("on_text_append")
+
+    async def process_frame(self, frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if not isinstance(frame, InputTransportMessageFrame):
+            await self.push_frame(frame, direction)
+            return
+
+        parsed = _text_input(frame.message)
+        if not parsed:
+            await self.push_frame(frame, direction)
+            return
+
+        text, run_immediately = parsed
+        await self._call_event_handler(
+            "on_text_input" if run_immediately else "on_text_append",
+            text,
+        )
+
+
 class PassthroughLLMAssistantAggregator(LLMAssistantAggregator):
     """聚合 LLM 回复进上下文,同时继续把回复帧交给下游 TTS。"""
 
@@ -176,6 +203,10 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
     """
     logger.info(f"启动管线: assistant={cfg.name} mode={cfg.runtimeMode}")
 
+    if cfg.runtimeMode == "realtime":
+        await run_realtime_pipeline(transport, cfg)
+        return
+
     stt, llm, tts = create_services(cfg)
 
     context = LLMContext(messages=[{"role": "system", "content": cfg.prompt}])
@@ -327,3 +358,70 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None:
     await runner.add_workers(worker)
     await runner.run()
     logger.info("管线已结束")
+
+
+async def run_realtime_pipeline(transport, cfg: AssistantConfig) -> None:
+    """Run a speech-to-speech model that owns ASR, reasoning, and synthesis."""
+    realtime = create_realtime_service(cfg)
+    text_input = RealtimeTextInputProcessor()
+
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            text_input,
+            realtime,
+            transport.output(),
+        ]
+    )
+    worker = PipelineWorker(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=False,
+            audio_in_sample_rate=int(
+                cfg.realtime_values.get("inputSampleRate") or 24000
+            ),
+            audio_out_sample_rate=int(
+                cfg.realtime_values.get("outputSampleRate") or 24000
+            ),
+        ),
+        enable_rtvi=False,
+    )
+
+    async def queue_transcript(role: str, content: str) -> None:
+        if content:
+            await worker.queue_frame(
+                OutputTransportMessageUrgentFrame(
+                    message={
+                        "type": "transcript",
+                        "role": role,
+                        "content": content,
+                        "timestamp": time_now_iso8601(),
+                    },
+                )
+            )
+
+    @text_input.event_handler("on_text_input")
+    async def on_text_input(_processor, text):
+        await queue_transcript("user", text)
+        await realtime.interrupt()
+        await realtime.send_text(text, run_immediately=True)
+
+    @text_input.event_handler("on_text_append")
+    async def on_text_append(_processor, text):
+        await queue_transcript("user", text)
+        await realtime.send_text(text, run_immediately=False)
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(_transport, _client):
+        if cfg.greeting:
+            await realtime.speak(cfg.greeting)
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(_transport, _client):
+        logger.info("Realtime 对端断开,结束管线")
+        await worker.queue_frame(EndFrame())
+
+    runner = WorkerRunner(handle_sigint=False)
+    await runner.add_workers(worker)
+    await runner.run()
+    logger.info("Realtime 管线已结束")