fix function calling examples

This commit is contained in:
Aleix Conchillo Flaqué
2024-08-17 23:05:11 -07:00
parent ebc4e0924b
commit 6520f20ffe
16 changed files with 121 additions and 140 deletions

View File

@@ -13,10 +13,6 @@ from pipecat.frames.frames import TextFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.llm_response import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from pipecat.processors.logger import FrameLogger
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.openai import OpenAILLMContext, OpenAILLMService
@@ -36,12 +32,12 @@ logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def start_fetch_weather(llm, function_name):
async def start_fetch_weather(llm, context, function_name):
await llm.push_frame(TextFrame("Let me check on that."))
async def fetch_weather_from_api(llm, function_name, args):
return {"conditions": "nice", "temperature": "75"}
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
await result_callback({"conditions": "nice", "temperature": "75"})
async def main():
@@ -72,7 +68,6 @@ async def main():
# Register a function_name of None to get all functions
# sent to the same callback with an additional function_name parameter.
llm.register_function(
#"get_current_weather",
None,
fetch_weather_from_api,
start_callback=start_fetch_weather)
@@ -114,17 +109,17 @@ async def main():
]
context = OpenAILLMContext(messages, tools)
tma_in = LLMUserContextAggregator(context)
tma_out = LLMAssistantContextAggregator(context)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline([
fl_in,
transport.input(),
tma_in,
context_aggregator.user(),
llm,
fl_out,
tts,
transport.output(),
tma_out
context_aggregator.assistant(),
])
task = PipelineTask(pipeline)

View File

@@ -14,10 +14,6 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.llm_response import (
LLMAssistantContextAggregator,
LLMUserContextAggregator
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.services.cartesia import CartesiaTTSService
@@ -40,10 +36,10 @@ logger.add(sys.stderr, level="DEBUG")
current_voice = "News Lady"
async def switch_voice(llm, args):
async def switch_voice(function_name, tool_call_id, args, llm, context, result_callback):
global current_voice
current_voice = args["voice"]
return {"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."}
await result_callback({"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."})
async def news_lady_filter(frame) -> bool:
@@ -119,12 +115,11 @@ async def main():
]
context = OpenAILLMContext(messages, tools)
tma_in = LLMUserContextAggregator(context)
tma_out = LLMAssistantContextAggregator(context)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline([
transport.input(), # Transport user input
tma_in, # User responses
context_aggregator.user(), # User responses
llm, # LLM
ParallelPipeline( # TTS (one of the following vocies)
[FunctionFilter(news_lady_filter), news_lady], # News Lady voice
@@ -132,7 +127,7 @@ async def main():
[FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice
),
transport.output(), # Transport bot output
tma_out # Assistant spoken responses
context_aggregator.assistant(), # Assistant spoken responses
])
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))

View File

@@ -14,10 +14,6 @@ from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.llm_response import (
LLMAssistantContextAggregator,
LLMUserContextAggregator
)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.filters.function_filter import FunctionFilter
from pipecat.services.elevenlabs import ElevenLabsTTSService
@@ -41,10 +37,10 @@ logger.add(sys.stderr, level="DEBUG")
current_language = "English"
async def switch_language(llm, args):
async def switch_language(function_name, tool_call_id, args, llm, context, result_callback):
global current_language
current_language = args["language"]
return {"voice": f"Your answers from now on should be in {current_language}."}
await result_callback({"voice": f"Your answers from now on should be in {current_language}."})
async def english_filter(frame) -> bool:
@@ -117,20 +113,19 @@ async def main():
]
context = OpenAILLMContext(messages, tools)
tma_in = LLMUserContextAggregator(context)
tma_out = LLMAssistantContextAggregator(context)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline([
transport.input(), # Transport user input
stt, # STT
tma_in, # User responses
context_aggregator.user(), # User responses
llm, # LLM
ParallelPipeline( # TTS (bot will speak the chosen language)
[FunctionFilter(english_filter), english_tts], # English
[FunctionFilter(spanish_filter), spanish_tts], # Spanish
),
transport.output(), # Transport bot output
tma_out # Assistant spoken responses
context_aggregator.assistant() # Assistant spoken responses
])
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))

View File

@@ -9,18 +9,16 @@ import aiohttp
import os
import sys
from pipecat.frames.frames import LLMMessagesFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.services.cartesia import CartesiaTTSService
from pipecat.services.anthropic import AnthropicLLMService, AnthropicUserContextAggregator, AnthropicAssistantContextAggregator
from pipecat.services.anthropic import AnthropicLLMService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVADAnalyzer
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from runner import configure
@@ -34,7 +32,7 @@ logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def get_weather(function_name, tool_call_id, arguments, context, result_callback):
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
location = arguments["location"]
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
@@ -98,7 +96,7 @@ async def main():
pipeline = Pipeline([
transport.input(), # Transport user input
context_aggregator.user(), # User speech to text
context_aggregator.user(), # User spoken responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output

View File

@@ -32,21 +32,16 @@ load_dotenv(override=True)
logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
# logger.add(sys.stderr, level="TRACE")
video_participant_id = None
# globally declare llm so that we can access it in the get_image function
llm = None
async def get_weather(function_name, tool_call_id, arguments, context, result_callback):
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
location = arguments["location"]
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
async def get_image(function_name, tool_call_id, arguments, context, result_callback):
global llm
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
question = arguments["question"]
await llm.request_image_frame(user_id=video_participant_id, text_content=question)

View File

@@ -35,7 +35,13 @@ logger.remove(0)
logger.add(sys.stderr, level="DEBUG")
async def get_current_weather(function_name, tool_call_id, arguments, context, result_callback):
async def get_current_weather(
function_name,
tool_call_id,
arguments,
llm,
context,
result_callback):
logger.debug("IN get_current_weather")
location = arguments["location"]
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")

View File

@@ -10,24 +10,14 @@ import os
import sys
import wave
from typing import List
from openai._types import NotGiven, NOT_GIVEN
from openai.types.chat import (
ChatCompletionToolParam,
)
from pipecat.frames.frames import AudioRawFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
from pipecat.processors.logger import FrameLogger
from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService
from pipecat.services.ai_services import AIService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVADAnalyzer
@@ -64,20 +54,11 @@ for file in sound_files:
class IntakeProcessor:
def __init__(
self,
context: OpenAILLMContext,
llm: AIService,
tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self._context: OpenAILLMContext = context
self._llm = llm
def __init__(self, context: OpenAILLMContext):
print(f"Initializing context from IntakeProcessor")
self._context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."})
self._context.set_tools([
context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."})
context.set_tools([
{
"type": "function",
"function": {
@@ -93,18 +74,17 @@ class IntakeProcessor:
},
},
}])
# Create an allowlist of functions that the LLM can call
self._functions = [
"verify_birthday",
"list_prescriptions",
"list_allergies",
"list_conditions",
"list_visit_reasons",
]
async def verify_birthday(self, llm, args):
async def verify_birthday(
self,
function_name,
tool_call_id,
args,
llm,
context,
result_callback):
if args["birthday"] == "1983-01-01":
self._context.set_tools(
context.set_tools(
[
{
"type": "function",
@@ -134,18 +114,18 @@ class IntakeProcessor:
},
}])
# It's a bit weird to push this to the LLM, but it gets it into the pipeline
await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM)
# await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM)
# We don't need the function call in the context, so just return a new
# system message and let the framework re-prompt
return [{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}]
await result_callback([{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}])
else:
# The user provided an incorrect birthday; ask them to try again
return [{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}]
await result_callback([{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}])
async def start_prescriptions(self, llm):
async def start_prescriptions(self, llm, context, function_name):
print(f"!!! doing start prescriptions")
# Move on to allergies
self._context.set_tools(
context.set_tools(
[
{
"type": "function",
@@ -169,18 +149,18 @@ class IntakeProcessor:
},
},
}])
self._context.add_message(
context.add_message(
{
"role": "system",
"content": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function."})
print(f"!!! about to await llm process frame in start prescrpitions")
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
print(f"!!! past await process frame in start prescriptions")
async def start_allergies(self, llm):
async def start_allergies(self, llm, context, function_name):
print("!!! doing start allergies")
# Move on to conditions
self._context.set_tools(
context.set_tools(
[
{
"type": "function",
@@ -205,16 +185,16 @@ class IntakeProcessor:
},
},
])
self._context.add_message(
context.add_message(
{
"role": "system",
"content": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function."})
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
async def start_conditions(self, llm):
async def start_conditions(self, llm, context, function_name):
print("!!! doing start conditions")
# Move on to visit reasons
self._context.set_tools(
context.set_tools(
[
{
"type": "function",
@@ -238,23 +218,25 @@ class IntakeProcessor:
},
},
}])
self._context.add_message(
{"role": "system", "content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."})
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
context.add_message(
{
"role": "system",
"content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."})
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
async def start_visit_reasons(self, llm):
async def start_visit_reasons(self, llm, context, function_name):
print("!!! doing start visit reasons")
# move to finish call
self._context.set_tools([])
self._context.add_message({"role": "system",
"content": "Now, thank the user and end the conversation."})
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
context.set_tools([])
context.add_message({"role": "system",
"content": "Now, thank the user and end the conversation."})
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
async def save_data(self, llm, args):
async def save_data(self, function_name, tool_call_id, args, llm, context, result_callback):
logger.info(f"!!! Saving data: {args}")
# Since this is supposed to be "async", returning None from the callback
# will prevent adding anything to context or re-prompting
return None
await result_callback(None)
async def main():
@@ -302,10 +284,9 @@ async def main():
messages = []
context = OpenAILLMContext(messages=messages)
user_context = LLMUserContextAggregator(context)
assistant_context = LLMAssistantContextAggregator(context)
context_aggregator = llm.create_context_aggregator(context)
intake = IntakeProcessor(context, llm)
intake = IntakeProcessor(context)
llm.register_function("verify_birthday", intake.verify_birthday)
llm.register_function(
"list_prescriptions",
@@ -328,12 +309,12 @@ async def main():
pipeline = Pipeline([
transport.input(), # Transport input
user_context, # User responses
context_aggregator.user(), # User responses
llm, # LLM
fl, # Frame logger
tts, # TTS
transport.output(), # Transport output
assistant_context, # Assistant responses
context_aggregator.assistant(), # Assistant responses
])
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False))

View File

@@ -25,6 +25,7 @@ dependencies = [
"loguru~=0.7.2",
"Pillow~=10.4.0",
"protobuf~=4.25.4",
"pydantic~=2.8.2",
"pyloudnorm~=0.1.1",
]
@@ -42,6 +43,7 @@ examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ]
fal = [ "fal-client~=0.4.1" ]
gladia = [ "websockets~=12.0" ]
google = [ "google-generativeai~=0.7.2" ]
gstreamer = [ "pygobject~=3.48.2" ]
fireworks = [ "openai~=1.37.2" ]
langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ]
local = [ "pyaudio~=0.2.14" ]

View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from typing import Any, List, Mapping, Tuple, Optional
from typing import Any, List, Mapping, Optional, Tuple
from dataclasses import dataclass, field
@@ -419,7 +419,7 @@ class TTSStoppedFrame(ControlFrame):
class UserImageRequestFrame(ControlFrame):
"""A frame user to request an image from the given user."""
user_id: str
context: Optional[any]
context: Optional[Any] = None
def __str__(self):
return f"{self.name}, user: {self.user_id}"

View File

@@ -4,25 +4,33 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
from dataclasses import dataclass
import io
import json
from typing import List
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, List
from PIL import Image
from pipecat.frames.frames import Frame, VisionImageRawFrame, FunctionCallInProgressFrame, FunctionCallResultFrame
from pipecat.processors.frame_processor import FrameProcessor
from loguru import logger
from openai._types import NOT_GIVEN, NotGiven
try:
from openai._types import NOT_GIVEN, NotGiven
from openai.types.chat import (
ChatCompletionToolParam,
ChatCompletionToolChoiceOptionParam,
ChatCompletionMessageParam
)
from openai.types.chat import (
ChatCompletionToolParam,
ChatCompletionToolChoiceOptionParam,
ChatCompletionMessageParam
)
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
raise Exception(f"Missing module: {e}")
# JSON custom encoder to handle bytes arrays so that we can log contexts
# with images to the console.
@@ -121,14 +129,20 @@ class OpenAILLMContext:
tools = NOT_GIVEN
self._tools = tools
async def call_function(
self,
f: callable,
*,
function_name: str,
tool_call_id: str,
arguments: str,
llm: FrameProcessor) -> None:
async def call_function(self,
f: Callable[[str,
str,
Any,
FrameProcessor,
'OpenAILLMContext',
Callable[[Any],
Awaitable[None]]],
Awaitable[None]],
*,
function_name: str,
tool_call_id: str,
arguments: str,
llm: FrameProcessor) -> None:
# Push a SystemFrame downstream. This frame will let our assistant context aggregator
# know that we are in the middle of a function call. Some contexts/aggregators may
@@ -146,8 +160,7 @@ class OpenAILLMContext:
tool_call_id=tool_call_id,
arguments=arguments,
result=result))
await f(function_name=function_name, tool_call_id=tool_call_id, arguments=arguments,
context=self, result_callback=function_call_result_callback)
await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback)
@dataclass

View File

@@ -25,6 +25,7 @@ from pipecat.frames.frames import (
UserStartedSpeakingFrame,
FunctionCallResultFrame,
UserStoppedSpeakingFrame)
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import BaseTransport
@@ -310,7 +311,8 @@ class RTVIProcessor(FrameProcessor):
function_name: str,
tool_call_id: str,
arguments: dict,
context,
llm: FrameProcessor,
context: OpenAILLMContext,
result_callback):
fn = RTVILLMFunctionCallMessageData(
function_name=function_name,
@@ -319,7 +321,11 @@ class RTVIProcessor(FrameProcessor):
message = RTVILLMFunctionCallMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)
async def handle_function_call_start(self, function_name: str):
async def handle_function_call_start(
self,
llm: FrameProcessor,
context: OpenAILLMContext,
function_name: str):
fn = RTVILLMFunctionCallStartMessageData(function_name=function_name)
message = RTVILLMFunctionCallStartMessage(data=fn)
await self._push_transport_message(message, exclude_none=False)

View File

@@ -27,7 +27,8 @@ try:
from gi.repository import Gst, GstApp
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error("In order to use GStreamer processors, you need to install GStreamer in your system`.")
logger.error(
"In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system.")
raise Exception(f"Missing module: {e}")

View File

@@ -137,11 +137,11 @@ class LLMService(AIService):
llm=self)
# QUESTION FOR CB: maybe this isn't needed anymore?
async def call_start_function(self, function_name: str):
async def call_start_function(self, context: OpenAILLMContext, function_name: str):
if function_name in self._start_callbacks.keys():
await self._start_callbacks[function_name](self)
await self._start_callbacks[function_name](self, context, function_name)
elif None in self._start_callbacks.keys():
return await self._start_callbacks[None](function_name)
return await self._start_callbacks[None](self, context, function_name)
class TTSService(AIService):

View File

@@ -481,9 +481,6 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
elif isinstance(frame, AnthropicImageMessageFrame):
self._pending_image_frame_message = frame
def add_message(self, message):
self._user_context_aggregator.add_message(message)
async def _push_aggregation(self):
if not self._aggregation:
return

View File

@@ -167,7 +167,7 @@ class BaseOpenAILLMService(LLMService):
if tool_call.function and tool_call.function.name:
function_name += tool_call.function.name
tool_call_id = tool_call.id
await self.call_start_function(function_name)
await self.call_start_function(context, function_name)
if tool_call.function and tool_call.function.arguments:
# Keep iterating through the response to collect all the argument fragments
arguments += tool_call.function.arguments
@@ -387,9 +387,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
self._function_call_in_progress = None
self._function_call_result = None
def add_message(self, message):
self._user_context_aggregator.add_message(message)
async def _push_aggregation(self):
if not (self._aggregation or self._function_call_result):
return

View File

@@ -24,7 +24,7 @@ try:
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use local audio, you need to `pip install pipecat-ai[audio]`. On MacOS, you also need to `brew install portaudio`.")
"In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.")
raise Exception(f"Missing module: {e}")
try: