diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 723997e7e..66c3f4219 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -13,10 +13,6 @@ from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator, -) from pipecat.processors.logger import FrameLogger from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMService @@ -36,12 +32,12 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(llm, function_name): +async def start_fetch_weather(llm, context, function_name): await llm.push_frame(TextFrame("Let me check on that.")) -async def fetch_weather_from_api(llm, function_name, args): - return {"conditions": "nice", "temperature": "75"} +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) async def main(): @@ -72,7 +68,6 @@ async def main(): # Register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. llm.register_function( - #"get_current_weather", None, fetch_weather_from_api, start_callback=start_fetch_weather) @@ -114,17 +109,17 @@ async def main(): ] context = OpenAILLMContext(messages, tools) - tma_in = LLMUserContextAggregator(context) - tma_out = LLMAssistantContextAggregator(context) + context_aggregator = llm.create_context_aggregator(context) + pipeline = Pipeline([ fl_in, transport.input(), - tma_in, + context_aggregator.user(), llm, fl_out, tts, transport.output(), - tma_out + context_aggregator.assistant(), ]) task = PipelineTask(pipeline) diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 2b762513e..a55dedc83 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -14,10 +14,6 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator -) from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.services.cartesia import CartesiaTTSService @@ -40,10 +36,10 @@ logger.add(sys.stderr, level="DEBUG") current_voice = "News Lady" -async def switch_voice(llm, args): +async def switch_voice(function_name, tool_call_id, args, llm, context, result_callback): global current_voice current_voice = args["voice"] - return {"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."} + await result_callback({"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."}) async def news_lady_filter(frame) -> bool: @@ -119,12 +115,11 @@ async def main(): ] context = OpenAILLMContext(messages, tools) - tma_in = LLMUserContextAggregator(context) - tma_out = LLMAssistantContextAggregator(context) + context_aggregator = llm.create_context_aggregator(context) pipeline = Pipeline([ transport.input(), # Transport user input - tma_in, # User responses + context_aggregator.user(), # User responses llm, # LLM ParallelPipeline( # TTS (one of the following vocies) [FunctionFilter(news_lady_filter), news_lady], # News Lady voice @@ -132,7 +127,7 @@ async def main(): [FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice ), transport.output(), # Transport bot output - tma_out # Assistant spoken responses + context_aggregator.assistant(), # Assistant spoken responses ]) task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index f2501613f..811cbbaeb 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -14,10 +14,6 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator -) from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.filters.function_filter import FunctionFilter from pipecat.services.elevenlabs import ElevenLabsTTSService @@ -41,10 +37,10 @@ logger.add(sys.stderr, level="DEBUG") current_language = "English" -async def switch_language(llm, args): +async def switch_language(function_name, tool_call_id, args, llm, context, result_callback): global current_language current_language = args["language"] - return {"voice": f"Your answers from now on should be in {current_language}."} + await result_callback({"voice": f"Your answers from now on should be in {current_language}."}) async def english_filter(frame) -> bool: @@ -117,20 +113,19 @@ async def main(): ] context = OpenAILLMContext(messages, tools) - tma_in = LLMUserContextAggregator(context) - tma_out = LLMAssistantContextAggregator(context) + context_aggregator = llm.create_context_aggregator(context) pipeline = Pipeline([ transport.input(), # Transport user input stt, # STT - tma_in, # User responses + context_aggregator.user(), # User responses llm, # LLM ParallelPipeline( # TTS (bot will speak the chosen language) [FunctionFilter(english_filter), english_tts], # English [FunctionFilter(spanish_filter), spanish_tts], # Spanish ), transport.output(), # Transport bot output - tma_out # Assistant spoken responses + context_aggregator.assistant() # Assistant spoken responses ]) task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) diff --git a/examples/foundational/19a-tools-anthropic.py b/examples/foundational/19a-tools-anthropic.py index e238de63c..728265cbb 100644 --- a/examples/foundational/19a-tools-anthropic.py +++ b/examples/foundational/19a-tools-anthropic.py @@ -9,18 +9,16 @@ import aiohttp import os import sys -from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.anthropic import AnthropicLLMService, AnthropicUserContextAggregator, AnthropicAssistantContextAggregator +from pipecat.services.anthropic import AnthropicLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from runner import configure @@ -34,7 +32,7 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def get_weather(function_name, tool_call_id, arguments, context, result_callback): +async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") @@ -98,7 +96,7 @@ async def main(): pipeline = Pipeline([ transport.input(), # Transport user input - context_aggregator.user(), # User speech to text + context_aggregator.user(), # User spoken responses llm, # LLM tts, # TTS transport.output(), # Transport bot output diff --git a/examples/foundational/19b-tools-video-anthropic.py b/examples/foundational/19b-tools-video-anthropic.py index 26d466e9e..9721ad2ae 100644 --- a/examples/foundational/19b-tools-video-anthropic.py +++ b/examples/foundational/19b-tools-video-anthropic.py @@ -32,21 +32,16 @@ load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") -# logger.add(sys.stderr, level="TRACE") video_participant_id = None -# globally declare llm so that we can access it in the get_image function -llm = None - -async def get_weather(function_name, tool_call_id, arguments, context, result_callback): +async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") -async def get_image(function_name, tool_call_id, arguments, context, result_callback): - global llm +async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] await llm.request_image_frame(user_id=video_participant_id, text_content=question) diff --git a/examples/foundational/19c-tools-togetherai.py b/examples/foundational/19c-tools-togetherai.py index c1ef328b9..c9a633b47 100644 --- a/examples/foundational/19c-tools-togetherai.py +++ b/examples/foundational/19c-tools-togetherai.py @@ -35,7 +35,13 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def get_current_weather(function_name, tool_call_id, arguments, context, result_callback): +async def get_current_weather( + function_name, + tool_call_id, + arguments, + llm, + context, + result_callback): logger.debug("IN get_current_weather") location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 0826921db..3cf65e95c 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -10,24 +10,14 @@ import os import sys import wave -from typing import List - -from openai._types import NotGiven, NOT_GIVEN - -from openai.types.chat import ( - ChatCompletionToolParam, -) - from pipecat.frames.frames import AudioRawFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator from pipecat.processors.logger import FrameLogger from pipecat.processors.frame_processor import FrameDirection from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService -from pipecat.services.ai_services import AIService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -64,20 +54,11 @@ for file in sound_files: class IntakeProcessor: - def __init__( - self, - context: OpenAILLMContext, - llm: AIService, - tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self._context: OpenAILLMContext = context - self._llm = llm + + def __init__(self, context: OpenAILLMContext): print(f"Initializing context from IntakeProcessor") - self._context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."}) - self._context.set_tools([ + context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."}) + context.set_tools([ { "type": "function", "function": { @@ -93,18 +74,17 @@ class IntakeProcessor: }, }, }]) - # Create an allowlist of functions that the LLM can call - self._functions = [ - "verify_birthday", - "list_prescriptions", - "list_allergies", - "list_conditions", - "list_visit_reasons", - ] - async def verify_birthday(self, llm, args): + async def verify_birthday( + self, + function_name, + tool_call_id, + args, + llm, + context, + result_callback): if args["birthday"] == "1983-01-01": - self._context.set_tools( + context.set_tools( [ { "type": "function", @@ -134,18 +114,18 @@ class IntakeProcessor: }, }]) # It's a bit weird to push this to the LLM, but it gets it into the pipeline - await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM) + # await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM) # We don't need the function call in the context, so just return a new # system message and let the framework re-prompt - return [{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}] + await result_callback([{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}]) else: # The user provided an incorrect birthday; ask them to try again - return [{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}] + await result_callback([{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}]) - async def start_prescriptions(self, llm): + async def start_prescriptions(self, llm, context, function_name): print(f"!!! doing start prescriptions") # Move on to allergies - self._context.set_tools( + context.set_tools( [ { "type": "function", @@ -169,18 +149,18 @@ class IntakeProcessor: }, }, }]) - self._context.add_message( + context.add_message( { "role": "system", "content": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function."}) print(f"!!! about to await llm process frame in start prescrpitions") - await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM) + await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) print(f"!!! past await process frame in start prescriptions") - async def start_allergies(self, llm): + async def start_allergies(self, llm, context, function_name): print("!!! doing start allergies") # Move on to conditions - self._context.set_tools( + context.set_tools( [ { "type": "function", @@ -205,16 +185,16 @@ class IntakeProcessor: }, }, ]) - self._context.add_message( + context.add_message( { "role": "system", "content": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function."}) - await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM) + await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) - async def start_conditions(self, llm): + async def start_conditions(self, llm, context, function_name): print("!!! doing start conditions") # Move on to visit reasons - self._context.set_tools( + context.set_tools( [ { "type": "function", @@ -238,23 +218,25 @@ class IntakeProcessor: }, }, }]) - self._context.add_message( - {"role": "system", "content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."}) - await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM) + context.add_message( + { + "role": "system", + "content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."}) + await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) - async def start_visit_reasons(self, llm): + async def start_visit_reasons(self, llm, context, function_name): print("!!! doing start visit reasons") # move to finish call - self._context.set_tools([]) - self._context.add_message({"role": "system", - "content": "Now, thank the user and end the conversation."}) - await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM) + context.set_tools([]) + context.add_message({"role": "system", + "content": "Now, thank the user and end the conversation."}) + await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) - async def save_data(self, llm, args): + async def save_data(self, function_name, tool_call_id, args, llm, context, result_callback): logger.info(f"!!! Saving data: {args}") # Since this is supposed to be "async", returning None from the callback # will prevent adding anything to context or re-prompting - return None + await result_callback(None) async def main(): @@ -302,10 +284,9 @@ async def main(): messages = [] context = OpenAILLMContext(messages=messages) - user_context = LLMUserContextAggregator(context) - assistant_context = LLMAssistantContextAggregator(context) + context_aggregator = llm.create_context_aggregator(context) - intake = IntakeProcessor(context, llm) + intake = IntakeProcessor(context) llm.register_function("verify_birthday", intake.verify_birthday) llm.register_function( "list_prescriptions", @@ -328,12 +309,12 @@ async def main(): pipeline = Pipeline([ transport.input(), # Transport input - user_context, # User responses + context_aggregator.user(), # User responses llm, # LLM fl, # Frame logger tts, # TTS transport.output(), # Transport output - assistant_context, # Assistant responses + context_aggregator.assistant(), # Assistant responses ]) task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False)) diff --git a/pyproject.toml b/pyproject.toml index 994a4b5e9..ea6e4eb9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "loguru~=0.7.2", "Pillow~=10.4.0", "protobuf~=4.25.4", + "pydantic~=2.8.2", "pyloudnorm~=0.1.1", ] @@ -42,6 +43,7 @@ examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ] fal = [ "fal-client~=0.4.1" ] gladia = [ "websockets~=12.0" ] google = [ "google-generativeai~=0.7.2" ] +gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.37.2" ] langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] local = [ "pyaudio~=0.2.14" ] diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 9cd4288fe..0e50d9412 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import Any, List, Mapping, Tuple, Optional +from typing import Any, List, Mapping, Optional, Tuple from dataclasses import dataclass, field @@ -419,7 +419,7 @@ class TTSStoppedFrame(ControlFrame): class UserImageRequestFrame(ControlFrame): """A frame user to request an image from the given user.""" user_id: str - context: Optional[any] + context: Optional[Any] = None def __str__(self): return f"{self.name}, user: {self.user_id}" diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 009040996..0d8b19a36 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -4,25 +4,33 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from dataclasses import dataclass import io import json -from typing import List +from dataclasses import dataclass + +from typing import Any, Awaitable, Callable, List from PIL import Image from pipecat.frames.frames import Frame, VisionImageRawFrame, FunctionCallInProgressFrame, FunctionCallResultFrame from pipecat.processors.frame_processor import FrameProcessor +from loguru import logger -from openai._types import NOT_GIVEN, NotGiven +try: + from openai._types import NOT_GIVEN, NotGiven -from openai.types.chat import ( - ChatCompletionToolParam, - ChatCompletionToolChoiceOptionParam, - ChatCompletionMessageParam -) + from openai.types.chat import ( + ChatCompletionToolParam, + ChatCompletionToolChoiceOptionParam, + ChatCompletionMessageParam + ) +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.") + raise Exception(f"Missing module: {e}") # JSON custom encoder to handle bytes arrays so that we can log contexts # with images to the console. @@ -121,14 +129,20 @@ class OpenAILLMContext: tools = NOT_GIVEN self._tools = tools - async def call_function( - self, - f: callable, - *, - function_name: str, - tool_call_id: str, - arguments: str, - llm: FrameProcessor) -> None: + async def call_function(self, + f: Callable[[str, + str, + Any, + FrameProcessor, + 'OpenAILLMContext', + Callable[[Any], + Awaitable[None]]], + Awaitable[None]], + *, + function_name: str, + tool_call_id: str, + arguments: str, + llm: FrameProcessor) -> None: # Push a SystemFrame downstream. This frame will let our assistant context aggregator # know that we are in the middle of a function call. Some contexts/aggregators may @@ -146,8 +160,7 @@ class OpenAILLMContext: tool_call_id=tool_call_id, arguments=arguments, result=result)) - await f(function_name=function_name, tool_call_id=tool_call_id, arguments=arguments, - context=self, result_callback=function_call_result_callback) + await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback) @dataclass diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 797ef288f..01bc6947a 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -25,6 +25,7 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, FunctionCallResultFrame, UserStoppedSpeakingFrame) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.base_transport import BaseTransport @@ -310,7 +311,8 @@ class RTVIProcessor(FrameProcessor): function_name: str, tool_call_id: str, arguments: dict, - context, + llm: FrameProcessor, + context: OpenAILLMContext, result_callback): fn = RTVILLMFunctionCallMessageData( function_name=function_name, @@ -319,7 +321,11 @@ class RTVIProcessor(FrameProcessor): message = RTVILLMFunctionCallMessage(data=fn) await self._push_transport_message(message, exclude_none=False) - async def handle_function_call_start(self, function_name: str): + async def handle_function_call_start( + self, + llm: FrameProcessor, + context: OpenAILLMContext, + function_name: str): fn = RTVILLMFunctionCallStartMessageData(function_name=function_name) message = RTVILLMFunctionCallStartMessage(data=fn) await self._push_transport_message(message, exclude_none=False) diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 89f7cd4d6..50e0baabf 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -27,7 +27,8 @@ try: from gi.repository import Gst, GstApp except ModuleNotFoundError as e: logger.error(f"Exception: {e}") - logger.error("In order to use GStreamer processors, you need to install GStreamer in your system`.") + logger.error( + "In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system.") raise Exception(f"Missing module: {e}") diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 33abf4e15..817af145f 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -137,11 +137,11 @@ class LLMService(AIService): llm=self) # QUESTION FOR CB: maybe this isn't needed anymore? - async def call_start_function(self, function_name: str): + async def call_start_function(self, context: OpenAILLMContext, function_name: str): if function_name in self._start_callbacks.keys(): - await self._start_callbacks[function_name](self) + await self._start_callbacks[function_name](self, context, function_name) elif None in self._start_callbacks.keys(): - return await self._start_callbacks[None](function_name) + return await self._start_callbacks[None](self, context, function_name) class TTSService(AIService): diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index e12f51ed7..76cdaef2c 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -481,9 +481,6 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): elif isinstance(frame, AnthropicImageMessageFrame): self._pending_image_frame_message = frame - def add_message(self, message): - self._user_context_aggregator.add_message(message) - async def _push_aggregation(self): if not self._aggregation: return diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 52da196d2..0cad2b8bb 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -167,7 +167,7 @@ class BaseOpenAILLMService(LLMService): if tool_call.function and tool_call.function.name: function_name += tool_call.function.name tool_call_id = tool_call.id - await self.call_start_function(function_name) + await self.call_start_function(context, function_name) if tool_call.function and tool_call.function.arguments: # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments @@ -387,9 +387,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): self._function_call_in_progress = None self._function_call_result = None - def add_message(self, message): - self._user_context_aggregator.add_message(message) - async def _push_aggregation(self): if not (self._aggregation or self._function_call_result): return diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index 0e2a404df..588e7d2ae 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -24,7 +24,7 @@ try: except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( - "In order to use local audio, you need to `pip install pipecat-ai[audio]`. On MacOS, you also need to `brew install portaudio`.") + "In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.") raise Exception(f"Missing module: {e}") try: