From 012dbffd9422f5537244c1472fda6882d6e886bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 May 2024 10:54:02 -0700 Subject: [PATCH 1/3] update CHANGELOG.md for function calling --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e11cd7b4b..0686d95b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added function calling (LLMService.register_function()). This will allow the + LLM to call functions you have registered when needed. For example, if you + register a function to get the weather in Los Angeles and ask the LLM about + the weather in Los Angeles, the LLM will call your function. + See https://platform.openai.com/docs/guides/function-calling + - Added Cartesia TTS support (https://cartesia.ai/) ### Fixed From 3655c4a0fc6a922c8072e55a9305ec36e5a445bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 May 2024 10:54:21 -0700 Subject: [PATCH 2/3] services: move function calling registration to LLMService --- src/pipecat/services/ai_services.py | 25 ++++++++++++++++++++++ src/pipecat/services/openai.py | 32 ++++++----------------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 02bb5ecef..a7f74ccc3 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -43,6 +43,31 @@ class LLMService(AIService): def __init__(self): super().__init__() + self._callbacks = {} + self._start_callbacks = {} + + # TODO-CB: callback function type + def register_function(self, function_name: str, callback, start_callback=None): + self._callbacks[function_name] = callback + if start_callback: + self._start_callbacks[function_name] = start_callback + + def unregister_function(self, function_name: str): + del self._callbacks[function_name] + if self._start_callbacks[function_name]: + del self._start_callbacks[function_name] + + def has_function(self, function_name: str): + return function_name in self._callbacks.keys() + + async def call_function(self, function_name: str, args): + if function_name in self._callbacks.keys(): + return await self._callbacks[function_name](self, args) + return None + + async def call_start_function(self, function_name: str): + if function_name in self._start_callbacks.keys(): + await self._start_callbacks[function_name](self) class TTSService(AIService): diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 5c9298d69..96c855fa4 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -29,12 +29,7 @@ from pipecat.frames.frames import ( from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext, OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import LLMService, ImageGenService -from openai.types.chat import ( - ChatCompletionSystemMessageParam, - ChatCompletionFunctionMessageParam, - ChatCompletionToolParam, - ChatCompletionUserMessageParam, -) + from loguru import logger try: @@ -43,7 +38,9 @@ try: from openai.types.chat import ( ChatCompletion, ChatCompletionChunk, + ChatCompletionFunctionMessageParam, ChatCompletionMessageParam, + ChatCompletionToolParam ) except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -70,23 +67,10 @@ class BaseOpenAILLMService(LLMService): super().__init__() self._model: str = model self._client = self.create_client(api_key=api_key, base_url=base_url) - self._callbacks = {} - self._start_callbacks = {} def create_client(self, api_key=None, base_url=None): return AsyncOpenAI(api_key=api_key, base_url=base_url) - # TODO-CB: callback function type - def register_function(self, function_name, callback, start_callback=None): - self._callbacks[function_name] = callback - if start_callback: - self._start_callbacks[function_name] = start_callback - - def unregister_function(self, function_name): - del self._callbacks[function_name] - if self._start_callbacks[function_name]: - del self._start_callbacks[function_name] - async def _stream_chat_completions( self, context: OpenAILLMContext ) -> AsyncStream[ChatCompletionChunk]: @@ -159,10 +143,7 @@ class BaseOpenAILLMService(LLMService): if tool_call.function and tool_call.function.name: function_name += tool_call.function.name tool_call_id = tool_call.id - # only send a function start frame if we're not handling the function call - if function_name in self._callbacks.keys(): - if function_name in self._start_callbacks.keys(): - await self._start_callbacks[function_name](self) + await self.call_start_function(function_name) if tool_call.function and tool_call.function.arguments: # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments @@ -176,9 +157,8 @@ class BaseOpenAILLMService(LLMService): # the context, and re-prompt to get a chat answer. If we don't have a registered # handler, raise an exception. if function_name and arguments: - if function_name in self._callbacks.keys(): + if self.has_function(function_name): await self._handle_function_call(context, tool_call_id, function_name, arguments) - else: raise OpenAIUnhandledFunctionException( f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function.") @@ -191,7 +171,7 @@ class BaseOpenAILLMService(LLMService): arguments ): arguments = json.loads(arguments) - result = await self._callbacks[function_name](self, arguments) + result = await self.call_function(function_name, arguments) arguments = json.dumps(arguments) if isinstance(result, (str, dict)): # Handle it in "full magic mode" From 42f772beedd0ab4827fcdd4082d2eac55c924b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 May 2024 11:05:08 -0700 Subject: [PATCH 3/3] examples: some function calling examples cleanup --- examples/foundational/14-function-calling.py | 13 ++--- examples/patient-intake/bot.py | 50 +++++++++----------- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index aab799ae9..14f834fe1 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -7,9 +7,9 @@ import asyncio import aiohttp import os -import json import sys +from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask @@ -17,18 +17,13 @@ from pipecat.processors.aggregators.llm_response import ( LLMAssistantContextAggregator, LLMUserContextAggregator, ) -from pipecat.services.openai import OpenAILLMContext from pipecat.processors.logger import FrameLogger from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.openai import OpenAILLMService +from pipecat.services.openai import OpenAILLMContext, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from openai.types.chat import ( - ChatCompletionToolParam, -) -from pipecat.frames.frames import ( - TextFrame -) + +from openai.types.chat import ChatCompletionToolParam from runner import configure diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index a24161e1a..170b08a07 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -1,11 +1,15 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import asyncio import aiohttp -import copy -import json import os -import re import sys import wave + from typing import List from openai._types import NotGiven, NOT_GIVEN @@ -14,23 +18,18 @@ from openai.types.chat import ( ChatCompletionToolParam, ) +from pipecat.frames.frames import AudioRawFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask +from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator from pipecat.processors.logger import FrameLogger -from pipecat.frames.frames import ( - Frame, - LLMMessagesFrame, - AudioRawFrame, -) -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frame_processor import FrameDirection from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.openai import OpenAILLMService +from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService from pipecat.services.ai_services import AIService -from pipecat.transports.services.daily import DailyParams, DailyTranscriptionSettings, DailyTransport +from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame from runner import configure @@ -242,7 +241,6 @@ class IntakeProcessor: self._context.add_message( {"role": "system", "content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."}) await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM) - pass async def start_visit_reasons(self, llm): print("!!! doing start visit reasons") @@ -251,7 +249,6 @@ class IntakeProcessor: self._context.add_message({"role": "system", "content": "Now, thank the user and end the conversation."}) await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM) - pass async def save_data(self, llm, args): logger.info(f"!!! Saving data: {args}") @@ -305,12 +302,10 @@ async def main(room_url: str, token): model="gpt-4o") messages = [] - context = OpenAILLMContext( - messages=messages, - ) + context = OpenAILLMContext(messages=messages) user_context = LLMUserContextAggregator(context) assistant_context = LLMAssistantContextAggregator(context) - # checklist = ChecklistProcessor(context, llm) + intake = IntakeProcessor(context, llm) llm.register_function("verify_birthday", intake.verify_birthday) llm.register_function( @@ -329,19 +324,20 @@ async def main(room_url: str, token): "list_visit_reasons", intake.save_data, start_callback=intake.start_visit_reasons) + fl = FrameLogger("LLM Output") pipeline = Pipeline([ - transport.input(), - user_context, - llm, - fl, - tts, - transport.output(), - assistant_context, + transport.input(), # Transport input + user_context, # User responses + llm, # LLM + fl, # Frame logger + tts, # TTS + transport.output(), # Transport output + assistant_context, # Assistant responses ]) - task = PipelineTask(pipeline, allow_interruptions=False) + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant):