diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index a520d17be..9b3ff1d7a 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -9,8 +9,9 @@ import os from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( CancelFrame, @@ -19,6 +20,7 @@ from pipecat.frames.frames import ( FunctionCallInProgressFrame, FunctionCallResultFrame, InterruptionFrame, + LLMContextFrame, LLMRunFrame, StartFrame, SystemFrame, @@ -32,10 +34,8 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor @@ -66,13 +66,13 @@ class StatementJudgeContextFilter(FrameProcessor): await self.push_frame(frame, direction) return - # We only want to handle OpenAILLMContextFrames, and only want to push through a simplified + # We only want to handle LLMContextFrames, and only want to push through a simplified # context frame that contains a system prompt and the most recent user messages, # concatenated. - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, LLMContextFrame): logger.debug(f"Context Frame: {frame}") # Take text content from the most recent user messages. - messages = frame.context.messages + messages = frame.context.get_messages() user_text_messages = [] last_assistant_message = None for message in reversed(messages): @@ -100,7 +100,7 @@ class StatementJudgeContextFilter(FrameProcessor): if last_assistant_message: messages.append(last_assistant_message) messages.append({"role": "user", "content": user_message}) - await self.push_frame(OpenAILLMContextFrame(OpenAILLMContext(messages))) + await self.push_frame(LLMContextFrame(LLMContext(messages))) class CompletenessCheck(FrameProcessor): @@ -231,7 +231,7 @@ class TurnDetectionLLM(Pipeline): async def pass_only_llm_trigger_frames(frame): return ( - isinstance(frame, OpenAILLMContextFrame) + isinstance(frame, LLMContextFrame) or isinstance(frame, InterruptionFrame) or isinstance(frame, FunctionCallInProgressFrame) or isinstance(frame, FunctionCallResultFrame) @@ -244,7 +244,7 @@ class TurnDetectionLLM(Pipeline): [ ParallelPipeline( [ - # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed + # Ignore everything except an LLMContextFrame. Pass a specially constructed # simplified context frame to the statement classifier LLM. The only frame this # sub-pipeline will output is a UserStoppedSpeakingFrame. statement_judge_context_filter, @@ -306,30 +306,23 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_function_calls_started(service, function_calls): await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { @@ -338,8 +331,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm_main.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) # LLM + turn detection (with an extra LLM as a judge) llm = TurnDetectionLLM(llm_main) diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 80811c2e7..461fab08d 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -9,8 +9,9 @@ import os from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( CancelFrame, @@ -19,6 +20,7 @@ from pipecat.frames.frames import ( FunctionCallInProgressFrame, FunctionCallResultFrame, InterruptionFrame, + LLMContextFrame, LLMRunFrame, StartFrame, SystemFrame, @@ -32,10 +34,8 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor @@ -272,11 +272,11 @@ class StatementJudgeContextFilter(FrameProcessor): await self.push_frame(frame, direction) return - # We only want to handle OpenAILLMContextFrames, and only want to push through a simplified + # We only want to handle LLMContextFrames, and only want to push through a simplified # context frame that contains a system prompt and the most recent user messages, - if isinstance(frame, OpenAILLMContextFrame): + if isinstance(frame, LLMContextFrame): # Take text content from the most recent user messages. - messages = frame.context.messages + messages = frame.context.get_messages() user_text_messages = [] last_assistant_message = None for message in reversed(messages): @@ -303,7 +303,7 @@ class StatementJudgeContextFilter(FrameProcessor): if last_assistant_message: messages.append(last_assistant_message) messages.append({"role": "user", "content": user_message}) - await self.push_frame(OpenAILLMContextFrame(OpenAILLMContext(messages))) + await self.push_frame(LLMContextFrame(LLMContext(messages))) class CompletenessCheck(FrameProcessor): @@ -425,7 +425,7 @@ class TurnDetectionLLM(Pipeline): async def pass_only_llm_trigger_frames(frame): return ( - isinstance(frame, OpenAILLMContextFrame) + isinstance(frame, LLMContextFrame) or isinstance(frame, InterruptionFrame) or isinstance(frame, FunctionCallInProgressFrame) or isinstance(frame, FunctionCallResultFrame) @@ -443,7 +443,7 @@ class TurnDetectionLLM(Pipeline): FunctionFilter(filter=block_user_stopped_speaking), ], [ - # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed + # Ignore everything except an LLMContextFrame. Pass a specially constructed # simplified context frame to the statement classifier LLM. The only frame this # sub-pipeline will output is a UserStoppedSpeakingFrame. statement_judge_context_filter, @@ -509,30 +509,23 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): async def on_function_calls_started(service, function_calls): await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { @@ -541,8 +534,8 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): }, ] - context = OpenAILLMContext(messages, tools) - context_aggregator = llm_main.create_context_aggregator(context) + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) # LLM + turn detection (with an extra LLM as a judge) llm = TurnDetectionLLM(llm_main)