wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix LLM context tool conversion and audio content handling

Browse Source
This commit is contained in:
Hwuiwon Kim
2025-12-15 13:43:57 -05:00
parent 4cd971e4bd
commit 27647fc067
1 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/pipecat/processors/aggregators/llm_context.py
View File

@@ -87,10 +87,19 @@ class LLMContext:
# Convert tools to ToolsSchema if needed. # Convert tools to ToolsSchema if needed.
# If the tools are already a ToolsSchema, this is a no-op. # If the tools are already a ToolsSchema, this is a no-op.
# Otherwise, we wrap them in a shim ToolsSchema. # Otherwise, we wrap them in a shim ToolsSchema.
converted_tools = openai_context.tools converted_tools: ToolsSchema | NotGiven
if isinstance(converted_tools, list): raw_tools = openai_context.tools
if isinstance(raw_tools, list):
converted_tools = ToolsSchema( converted_tools = ToolsSchema(
standard_tools=[], custom_tools={AdapterType.SHIM: converted_tools} standard_tools=[], custom_tools={AdapterType.SHIM: raw_tools}
)
elif isinstance(raw_tools, ToolsSchema):
converted_tools = raw_tools
elif raw_tools is NOT_GIVEN:
converted_tools = NOT_GIVEN
else:
raise TypeError(
f"Unsupported tools type when converting OpenAI context: {type(raw_tools)}"
) )
return LLMContext( return LLMContext(
messages=openai_context.get_messages(), messages=openai_context.get_messages(),
@@ -179,13 +188,12 @@ class LLMContext:
audio_frames: List of audio frame objects to include. audio_frames: List of audio frame objects to include.
text: Optional text to include with the audio. text: Optional text to include with the audio.
""" """
content = [{"type": "text", "text": text}]
async def encode_audio(): async def encode_audio():
sample_rate = audio_frames[0].sample_rate sample_rate = audio_frames[0].sample_rate
num_channels = audio_frames[0].num_channels num_channels = audio_frames[0].num_channels
content = []
content.append({"type": "text", "text": text})
data = b"".join(frame.audio for frame in audio_frames) data = b"".join(frame.audio for frame in audio_frames)
with io.BytesIO() as buffer: with io.BytesIO() as buffer:
@@ -195,7 +203,7 @@ class LLMContext:
wf.setframerate(sample_rate) wf.setframerate(sample_rate)
wf.writeframes(data) wf.writeframes(data)
encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8")
return encoded_audio return encoded_audio
encoded_audio = await asyncio.to_thread(encode_audio) encoded_audio = await asyncio.to_thread(encode_audio)