fix function calling examples
This commit is contained in:
@@ -13,10 +13,6 @@ from pipecat.frames.frames import TextFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineTask
|
||||
from pipecat.processors.aggregators.llm_response import (
|
||||
LLMAssistantContextAggregator,
|
||||
LLMUserContextAggregator,
|
||||
)
|
||||
from pipecat.processors.logger import FrameLogger
|
||||
from pipecat.services.elevenlabs import ElevenLabsTTSService
|
||||
from pipecat.services.openai import OpenAILLMContext, OpenAILLMService
|
||||
@@ -36,12 +32,12 @@ logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
|
||||
async def start_fetch_weather(llm, function_name):
|
||||
async def start_fetch_weather(llm, context, function_name):
|
||||
await llm.push_frame(TextFrame("Let me check on that."))
|
||||
|
||||
|
||||
async def fetch_weather_from_api(llm, function_name, args):
|
||||
return {"conditions": "nice", "temperature": "75"}
|
||||
async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback):
|
||||
await result_callback({"conditions": "nice", "temperature": "75"})
|
||||
|
||||
|
||||
async def main():
|
||||
@@ -72,7 +68,6 @@ async def main():
|
||||
# Register a function_name of None to get all functions
|
||||
# sent to the same callback with an additional function_name parameter.
|
||||
llm.register_function(
|
||||
#"get_current_weather",
|
||||
None,
|
||||
fetch_weather_from_api,
|
||||
start_callback=start_fetch_weather)
|
||||
@@ -114,17 +109,17 @@ async def main():
|
||||
]
|
||||
|
||||
context = OpenAILLMContext(messages, tools)
|
||||
tma_in = LLMUserContextAggregator(context)
|
||||
tma_out = LLMAssistantContextAggregator(context)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
pipeline = Pipeline([
|
||||
fl_in,
|
||||
transport.input(),
|
||||
tma_in,
|
||||
context_aggregator.user(),
|
||||
llm,
|
||||
fl_out,
|
||||
tts,
|
||||
transport.output(),
|
||||
tma_out
|
||||
context_aggregator.assistant(),
|
||||
])
|
||||
|
||||
task = PipelineTask(pipeline)
|
||||
|
||||
@@ -14,10 +14,6 @@ from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_response import (
|
||||
LLMAssistantContextAggregator,
|
||||
LLMUserContextAggregator
|
||||
)
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.filters.function_filter import FunctionFilter
|
||||
from pipecat.services.cartesia import CartesiaTTSService
|
||||
@@ -40,10 +36,10 @@ logger.add(sys.stderr, level="DEBUG")
|
||||
current_voice = "News Lady"
|
||||
|
||||
|
||||
async def switch_voice(llm, args):
|
||||
async def switch_voice(function_name, tool_call_id, args, llm, context, result_callback):
|
||||
global current_voice
|
||||
current_voice = args["voice"]
|
||||
return {"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."}
|
||||
await result_callback({"voice": f"You are now using your {current_voice} voice. Your responses should now be as if you were a {current_voice}."})
|
||||
|
||||
|
||||
async def news_lady_filter(frame) -> bool:
|
||||
@@ -119,12 +115,11 @@ async def main():
|
||||
]
|
||||
|
||||
context = OpenAILLMContext(messages, tools)
|
||||
tma_in = LLMUserContextAggregator(context)
|
||||
tma_out = LLMAssistantContextAggregator(context)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
pipeline = Pipeline([
|
||||
transport.input(), # Transport user input
|
||||
tma_in, # User responses
|
||||
context_aggregator.user(), # User responses
|
||||
llm, # LLM
|
||||
ParallelPipeline( # TTS (one of the following vocies)
|
||||
[FunctionFilter(news_lady_filter), news_lady], # News Lady voice
|
||||
@@ -132,7 +127,7 @@ async def main():
|
||||
[FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice
|
||||
),
|
||||
transport.output(), # Transport bot output
|
||||
tma_out # Assistant spoken responses
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
])
|
||||
|
||||
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
|
||||
|
||||
@@ -14,10 +14,6 @@ from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.parallel_pipeline import ParallelPipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_response import (
|
||||
LLMAssistantContextAggregator,
|
||||
LLMUserContextAggregator
|
||||
)
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.filters.function_filter import FunctionFilter
|
||||
from pipecat.services.elevenlabs import ElevenLabsTTSService
|
||||
@@ -41,10 +37,10 @@ logger.add(sys.stderr, level="DEBUG")
|
||||
current_language = "English"
|
||||
|
||||
|
||||
async def switch_language(llm, args):
|
||||
async def switch_language(function_name, tool_call_id, args, llm, context, result_callback):
|
||||
global current_language
|
||||
current_language = args["language"]
|
||||
return {"voice": f"Your answers from now on should be in {current_language}."}
|
||||
await result_callback({"voice": f"Your answers from now on should be in {current_language}."})
|
||||
|
||||
|
||||
async def english_filter(frame) -> bool:
|
||||
@@ -117,20 +113,19 @@ async def main():
|
||||
]
|
||||
|
||||
context = OpenAILLMContext(messages, tools)
|
||||
tma_in = LLMUserContextAggregator(context)
|
||||
tma_out = LLMAssistantContextAggregator(context)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
pipeline = Pipeline([
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
tma_in, # User responses
|
||||
context_aggregator.user(), # User responses
|
||||
llm, # LLM
|
||||
ParallelPipeline( # TTS (bot will speak the chosen language)
|
||||
[FunctionFilter(english_filter), english_tts], # English
|
||||
[FunctionFilter(spanish_filter), spanish_tts], # Spanish
|
||||
),
|
||||
transport.output(), # Transport bot output
|
||||
tma_out # Assistant spoken responses
|
||||
context_aggregator.assistant() # Assistant spoken responses
|
||||
])
|
||||
|
||||
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True))
|
||||
|
||||
@@ -9,18 +9,16 @@ import aiohttp
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pipecat.frames.frames import LLMMessagesFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.services.cartesia import CartesiaTTSService
|
||||
|
||||
from pipecat.services.anthropic import AnthropicLLMService, AnthropicUserContextAggregator, AnthropicAssistantContextAggregator
|
||||
from pipecat.services.anthropic import AnthropicLLMService
|
||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||
from pipecat.vad.silero import SileroVADAnalyzer
|
||||
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
|
||||
from runner import configure
|
||||
@@ -34,7 +32,7 @@ logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
|
||||
async def get_weather(function_name, tool_call_id, arguments, context, result_callback):
|
||||
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||
location = arguments["location"]
|
||||
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
|
||||
|
||||
@@ -98,7 +96,7 @@ async def main():
|
||||
|
||||
pipeline = Pipeline([
|
||||
transport.input(), # Transport user input
|
||||
context_aggregator.user(), # User speech to text
|
||||
context_aggregator.user(), # User spoken responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
|
||||
@@ -32,21 +32,16 @@ load_dotenv(override=True)
|
||||
|
||||
logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
# logger.add(sys.stderr, level="TRACE")
|
||||
|
||||
video_participant_id = None
|
||||
|
||||
# globally declare llm so that we can access it in the get_image function
|
||||
llm = None
|
||||
|
||||
|
||||
async def get_weather(function_name, tool_call_id, arguments, context, result_callback):
|
||||
async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||
location = arguments["location"]
|
||||
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
|
||||
|
||||
|
||||
async def get_image(function_name, tool_call_id, arguments, context, result_callback):
|
||||
global llm
|
||||
async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback):
|
||||
question = arguments["question"]
|
||||
await llm.request_image_frame(user_id=video_participant_id, text_content=question)
|
||||
|
||||
|
||||
@@ -35,7 +35,13 @@ logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
|
||||
async def get_current_weather(function_name, tool_call_id, arguments, context, result_callback):
|
||||
async def get_current_weather(
|
||||
function_name,
|
||||
tool_call_id,
|
||||
arguments,
|
||||
llm,
|
||||
context,
|
||||
result_callback):
|
||||
logger.debug("IN get_current_weather")
|
||||
location = arguments["location"]
|
||||
await result_callback(f"The weather in {location} is currently 72 degrees and sunny.")
|
||||
|
||||
@@ -10,24 +10,14 @@ import os
|
||||
import sys
|
||||
import wave
|
||||
|
||||
from typing import List
|
||||
|
||||
from openai._types import NotGiven, NOT_GIVEN
|
||||
|
||||
from openai.types.chat import (
|
||||
ChatCompletionToolParam,
|
||||
)
|
||||
|
||||
from pipecat.frames.frames import AudioRawFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator, LLMAssistantContextAggregator
|
||||
from pipecat.processors.logger import FrameLogger
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.services.elevenlabs import ElevenLabsTTSService
|
||||
from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService
|
||||
from pipecat.services.ai_services import AIService
|
||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||
from pipecat.vad.silero import SileroVADAnalyzer
|
||||
|
||||
@@ -64,20 +54,11 @@ for file in sound_files:
|
||||
|
||||
|
||||
class IntakeProcessor:
|
||||
def __init__(
|
||||
self,
|
||||
context: OpenAILLMContext,
|
||||
llm: AIService,
|
||||
tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._context: OpenAILLMContext = context
|
||||
self._llm = llm
|
||||
|
||||
def __init__(self, context: OpenAILLMContext):
|
||||
print(f"Initializing context from IntakeProcessor")
|
||||
self._context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."})
|
||||
self._context.set_tools([
|
||||
context.add_message({"role": "system", "content": "You are Jessica, an agent for a company called Tri-County Health Services. Your job is to collect important information from the user before their doctor visit. You're talking to Chad Bailey. You should address the user by their first name and be polite and professional. You're not a medical professional, so you shouldn't provide any advice. Keep your responses short. Your job is to collect information to give to a doctor. Don't make assumptions about what values to plug into functions. Ask for clarification if a user response is ambiguous. Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function."})
|
||||
context.set_tools([
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
@@ -93,18 +74,17 @@ class IntakeProcessor:
|
||||
},
|
||||
},
|
||||
}])
|
||||
# Create an allowlist of functions that the LLM can call
|
||||
self._functions = [
|
||||
"verify_birthday",
|
||||
"list_prescriptions",
|
||||
"list_allergies",
|
||||
"list_conditions",
|
||||
"list_visit_reasons",
|
||||
]
|
||||
|
||||
async def verify_birthday(self, llm, args):
|
||||
async def verify_birthday(
|
||||
self,
|
||||
function_name,
|
||||
tool_call_id,
|
||||
args,
|
||||
llm,
|
||||
context,
|
||||
result_callback):
|
||||
if args["birthday"] == "1983-01-01":
|
||||
self._context.set_tools(
|
||||
context.set_tools(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
@@ -134,18 +114,18 @@ class IntakeProcessor:
|
||||
},
|
||||
}])
|
||||
# It's a bit weird to push this to the LLM, but it gets it into the pipeline
|
||||
await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM)
|
||||
# await llm.push_frame(sounds["ding2.wav"], FrameDirection.DOWNSTREAM)
|
||||
# We don't need the function call in the context, so just return a new
|
||||
# system message and let the framework re-prompt
|
||||
return [{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}]
|
||||
await result_callback([{"role": "system", "content": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages."}])
|
||||
else:
|
||||
# The user provided an incorrect birthday; ask them to try again
|
||||
return [{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}]
|
||||
await result_callback([{"role": "system", "content": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function."}])
|
||||
|
||||
async def start_prescriptions(self, llm):
|
||||
async def start_prescriptions(self, llm, context, function_name):
|
||||
print(f"!!! doing start prescriptions")
|
||||
# Move on to allergies
|
||||
self._context.set_tools(
|
||||
context.set_tools(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
@@ -169,18 +149,18 @@ class IntakeProcessor:
|
||||
},
|
||||
},
|
||||
}])
|
||||
self._context.add_message(
|
||||
context.add_message(
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function."})
|
||||
print(f"!!! about to await llm process frame in start prescrpitions")
|
||||
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
|
||||
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
|
||||
print(f"!!! past await process frame in start prescriptions")
|
||||
|
||||
async def start_allergies(self, llm):
|
||||
async def start_allergies(self, llm, context, function_name):
|
||||
print("!!! doing start allergies")
|
||||
# Move on to conditions
|
||||
self._context.set_tools(
|
||||
context.set_tools(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
@@ -205,16 +185,16 @@ class IntakeProcessor:
|
||||
},
|
||||
},
|
||||
])
|
||||
self._context.add_message(
|
||||
context.add_message(
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function."})
|
||||
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
|
||||
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
|
||||
|
||||
async def start_conditions(self, llm):
|
||||
async def start_conditions(self, llm, context, function_name):
|
||||
print("!!! doing start conditions")
|
||||
# Move on to visit reasons
|
||||
self._context.set_tools(
|
||||
context.set_tools(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
@@ -238,23 +218,25 @@ class IntakeProcessor:
|
||||
},
|
||||
},
|
||||
}])
|
||||
self._context.add_message(
|
||||
{"role": "system", "content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."})
|
||||
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
|
||||
context.add_message(
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Finally, ask the user the reason for their doctor visit today. Once they answer, call the list_visit_reasons function."})
|
||||
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
|
||||
|
||||
async def start_visit_reasons(self, llm):
|
||||
async def start_visit_reasons(self, llm, context, function_name):
|
||||
print("!!! doing start visit reasons")
|
||||
# move to finish call
|
||||
self._context.set_tools([])
|
||||
self._context.add_message({"role": "system",
|
||||
"content": "Now, thank the user and end the conversation."})
|
||||
await llm.process_frame(OpenAILLMContextFrame(self._context), FrameDirection.DOWNSTREAM)
|
||||
context.set_tools([])
|
||||
context.add_message({"role": "system",
|
||||
"content": "Now, thank the user and end the conversation."})
|
||||
await llm.process_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM)
|
||||
|
||||
async def save_data(self, llm, args):
|
||||
async def save_data(self, function_name, tool_call_id, args, llm, context, result_callback):
|
||||
logger.info(f"!!! Saving data: {args}")
|
||||
# Since this is supposed to be "async", returning None from the callback
|
||||
# will prevent adding anything to context or re-prompting
|
||||
return None
|
||||
await result_callback(None)
|
||||
|
||||
|
||||
async def main():
|
||||
@@ -302,10 +284,9 @@ async def main():
|
||||
|
||||
messages = []
|
||||
context = OpenAILLMContext(messages=messages)
|
||||
user_context = LLMUserContextAggregator(context)
|
||||
assistant_context = LLMAssistantContextAggregator(context)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
intake = IntakeProcessor(context, llm)
|
||||
intake = IntakeProcessor(context)
|
||||
llm.register_function("verify_birthday", intake.verify_birthday)
|
||||
llm.register_function(
|
||||
"list_prescriptions",
|
||||
@@ -328,12 +309,12 @@ async def main():
|
||||
|
||||
pipeline = Pipeline([
|
||||
transport.input(), # Transport input
|
||||
user_context, # User responses
|
||||
context_aggregator.user(), # User responses
|
||||
llm, # LLM
|
||||
fl, # Frame logger
|
||||
tts, # TTS
|
||||
transport.output(), # Transport output
|
||||
assistant_context, # Assistant responses
|
||||
context_aggregator.assistant(), # Assistant responses
|
||||
])
|
||||
|
||||
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False))
|
||||
|
||||
@@ -25,6 +25,7 @@ dependencies = [
|
||||
"loguru~=0.7.2",
|
||||
"Pillow~=10.4.0",
|
||||
"protobuf~=4.25.4",
|
||||
"pydantic~=2.8.2",
|
||||
"pyloudnorm~=0.1.1",
|
||||
]
|
||||
|
||||
@@ -42,6 +43,7 @@ examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ]
|
||||
fal = [ "fal-client~=0.4.1" ]
|
||||
gladia = [ "websockets~=12.0" ]
|
||||
google = [ "google-generativeai~=0.7.2" ]
|
||||
gstreamer = [ "pygobject~=3.48.2" ]
|
||||
fireworks = [ "openai~=1.37.2" ]
|
||||
langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ]
|
||||
local = [ "pyaudio~=0.2.14" ]
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
from typing import Any, List, Mapping, Tuple, Optional
|
||||
from typing import Any, List, Mapping, Optional, Tuple
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
@@ -419,7 +419,7 @@ class TTSStoppedFrame(ControlFrame):
|
||||
class UserImageRequestFrame(ControlFrame):
|
||||
"""A frame user to request an image from the given user."""
|
||||
user_id: str
|
||||
context: Optional[any]
|
||||
context: Optional[Any] = None
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.name}, user: {self.user_id}"
|
||||
|
||||
@@ -4,25 +4,33 @@
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
from dataclasses import dataclass
|
||||
import io
|
||||
import json
|
||||
|
||||
from typing import List
|
||||
from dataclasses import dataclass
|
||||
|
||||
from typing import Any, Awaitable, Callable, List
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from pipecat.frames.frames import Frame, VisionImageRawFrame, FunctionCallInProgressFrame, FunctionCallResultFrame
|
||||
from pipecat.processors.frame_processor import FrameProcessor
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from openai._types import NOT_GIVEN, NotGiven
|
||||
try:
|
||||
from openai._types import NOT_GIVEN, NotGiven
|
||||
|
||||
from openai.types.chat import (
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolChoiceOptionParam,
|
||||
ChatCompletionMessageParam
|
||||
)
|
||||
from openai.types.chat import (
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolChoiceOptionParam,
|
||||
ChatCompletionMessageParam
|
||||
)
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error(
|
||||
"In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable.")
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
# JSON custom encoder to handle bytes arrays so that we can log contexts
|
||||
# with images to the console.
|
||||
@@ -121,14 +129,20 @@ class OpenAILLMContext:
|
||||
tools = NOT_GIVEN
|
||||
self._tools = tools
|
||||
|
||||
async def call_function(
|
||||
self,
|
||||
f: callable,
|
||||
*,
|
||||
function_name: str,
|
||||
tool_call_id: str,
|
||||
arguments: str,
|
||||
llm: FrameProcessor) -> None:
|
||||
async def call_function(self,
|
||||
f: Callable[[str,
|
||||
str,
|
||||
Any,
|
||||
FrameProcessor,
|
||||
'OpenAILLMContext',
|
||||
Callable[[Any],
|
||||
Awaitable[None]]],
|
||||
Awaitable[None]],
|
||||
*,
|
||||
function_name: str,
|
||||
tool_call_id: str,
|
||||
arguments: str,
|
||||
llm: FrameProcessor) -> None:
|
||||
|
||||
# Push a SystemFrame downstream. This frame will let our assistant context aggregator
|
||||
# know that we are in the middle of a function call. Some contexts/aggregators may
|
||||
@@ -146,8 +160,7 @@ class OpenAILLMContext:
|
||||
tool_call_id=tool_call_id,
|
||||
arguments=arguments,
|
||||
result=result))
|
||||
await f(function_name=function_name, tool_call_id=tool_call_id, arguments=arguments,
|
||||
context=self, result_callback=function_call_result_callback)
|
||||
await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -25,6 +25,7 @@ from pipecat.frames.frames import (
|
||||
UserStartedSpeakingFrame,
|
||||
FunctionCallResultFrame,
|
||||
UserStoppedSpeakingFrame)
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.transports.base_transport import BaseTransport
|
||||
|
||||
@@ -310,7 +311,8 @@ class RTVIProcessor(FrameProcessor):
|
||||
function_name: str,
|
||||
tool_call_id: str,
|
||||
arguments: dict,
|
||||
context,
|
||||
llm: FrameProcessor,
|
||||
context: OpenAILLMContext,
|
||||
result_callback):
|
||||
fn = RTVILLMFunctionCallMessageData(
|
||||
function_name=function_name,
|
||||
@@ -319,7 +321,11 @@ class RTVIProcessor(FrameProcessor):
|
||||
message = RTVILLMFunctionCallMessage(data=fn)
|
||||
await self._push_transport_message(message, exclude_none=False)
|
||||
|
||||
async def handle_function_call_start(self, function_name: str):
|
||||
async def handle_function_call_start(
|
||||
self,
|
||||
llm: FrameProcessor,
|
||||
context: OpenAILLMContext,
|
||||
function_name: str):
|
||||
fn = RTVILLMFunctionCallStartMessageData(function_name=function_name)
|
||||
message = RTVILLMFunctionCallStartMessage(data=fn)
|
||||
await self._push_transport_message(message, exclude_none=False)
|
||||
|
||||
@@ -27,7 +27,8 @@ try:
|
||||
from gi.repository import Gst, GstApp
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error("In order to use GStreamer processors, you need to install GStreamer in your system`.")
|
||||
logger.error(
|
||||
"In order to use GStreamer, you need to `pip install pipecat-ai[gstreamer]`. Also, you need to install GStreamer in your system.")
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
|
||||
|
||||
@@ -137,11 +137,11 @@ class LLMService(AIService):
|
||||
llm=self)
|
||||
|
||||
# QUESTION FOR CB: maybe this isn't needed anymore?
|
||||
async def call_start_function(self, function_name: str):
|
||||
async def call_start_function(self, context: OpenAILLMContext, function_name: str):
|
||||
if function_name in self._start_callbacks.keys():
|
||||
await self._start_callbacks[function_name](self)
|
||||
await self._start_callbacks[function_name](self, context, function_name)
|
||||
elif None in self._start_callbacks.keys():
|
||||
return await self._start_callbacks[None](function_name)
|
||||
return await self._start_callbacks[None](self, context, function_name)
|
||||
|
||||
|
||||
class TTSService(AIService):
|
||||
|
||||
@@ -481,9 +481,6 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
|
||||
elif isinstance(frame, AnthropicImageMessageFrame):
|
||||
self._pending_image_frame_message = frame
|
||||
|
||||
def add_message(self, message):
|
||||
self._user_context_aggregator.add_message(message)
|
||||
|
||||
async def _push_aggregation(self):
|
||||
if not self._aggregation:
|
||||
return
|
||||
|
||||
@@ -167,7 +167,7 @@ class BaseOpenAILLMService(LLMService):
|
||||
if tool_call.function and tool_call.function.name:
|
||||
function_name += tool_call.function.name
|
||||
tool_call_id = tool_call.id
|
||||
await self.call_start_function(function_name)
|
||||
await self.call_start_function(context, function_name)
|
||||
if tool_call.function and tool_call.function.arguments:
|
||||
# Keep iterating through the response to collect all the argument fragments
|
||||
arguments += tool_call.function.arguments
|
||||
@@ -387,9 +387,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
|
||||
self._function_call_in_progress = None
|
||||
self._function_call_result = None
|
||||
|
||||
def add_message(self, message):
|
||||
self._user_context_aggregator.add_message(message)
|
||||
|
||||
async def _push_aggregation(self):
|
||||
if not (self._aggregation or self._function_call_result):
|
||||
return
|
||||
|
||||
@@ -24,7 +24,7 @@ try:
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error(
|
||||
"In order to use local audio, you need to `pip install pipecat-ai[audio]`. On MacOS, you also need to `brew install portaudio`.")
|
||||
"In order to use local audio, you need to `pip install pipecat-ai[local]`. On MacOS, you also need to `brew install portaudio`.")
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user