Introduce new helper type for acting as a liason between rtvi messages and the llm

This somewhat parallels the helper system the RTVI clients have.
This commit is contained in:
mattie ruth backman
2025-04-14 17:08:57 -04:00
parent 005c0b55b6
commit 424c2a341b
4 changed files with 226 additions and 1 deletions

View File

@@ -16,7 +16,7 @@
* - Browser with WebRTC support
*/
import { RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
import { LLMHelper, RTVIClient, RTVIEvent } from '@pipecat-ai/client-js';
import { DailyTransport } from '@pipecat-ai/daily-transport';
/**
@@ -27,6 +27,7 @@ class ChatbotClient {
constructor() {
// Initialize client state
this.rtviClient = null;
this.llmHelper = null;
this.setupDOMElements();
this.setupEventListeners();
this.initializeClientAndTransport();
@@ -127,6 +128,13 @@ class ChatbotClient {
},
});
this.llmHelper = new LLMHelper({});
rtviClient.registerHelper('llm', this.llmHelper);
// Uncomment for debugging
// window.rtviClient = this.rtviClient;
// window.llmHelper = this.llmHelper;
// Set up listeners for media track events
this.setupTrackListeners();
}

View File

@@ -41,6 +41,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
from pipecat.processors.frameworks.rtvi_helpers.llm import RTVILLMHelper
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
from pipecat.services.openai.llm import OpenAILLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport
@@ -182,6 +184,11 @@ async def main():
# RTVI events for Pipecat client UI
#
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
llmHelper = RTVILLMHelper(
service="llm",
user_aggregator=context_aggregator.user(),
)
llmHelper.register_actions(rtvi)
pipeline = Pipeline(
[