From 27647fc067bf7f2b5f91826938dca76766c549b8 Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Mon, 15 Dec 2025 13:43:57 -0500 Subject: [PATCH 1/2] Fix LLM context tool conversion and audio content handling --- .../processors/aggregators/llm_context.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 99b9aeaa9..be2868122 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -87,10 +87,19 @@ class LLMContext: # Convert tools to ToolsSchema if needed. # If the tools are already a ToolsSchema, this is a no-op. # Otherwise, we wrap them in a shim ToolsSchema. - converted_tools = openai_context.tools - if isinstance(converted_tools, list): + converted_tools: ToolsSchema | NotGiven + raw_tools = openai_context.tools + if isinstance(raw_tools, list): converted_tools = ToolsSchema( - standard_tools=[], custom_tools={AdapterType.SHIM: converted_tools} + standard_tools=[], custom_tools={AdapterType.SHIM: raw_tools} + ) + elif isinstance(raw_tools, ToolsSchema): + converted_tools = raw_tools + elif raw_tools is NOT_GIVEN: + converted_tools = NOT_GIVEN + else: + raise TypeError( + f"Unsupported tools type when converting OpenAI context: {type(raw_tools)}" ) return LLMContext( messages=openai_context.get_messages(), @@ -179,13 +188,12 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ + content = [{"type": "text", "text": text}] async def encode_audio(): sample_rate = audio_frames[0].sample_rate num_channels = audio_frames[0].num_channels - content = [] - content.append({"type": "text", "text": text}) data = b"".join(frame.audio for frame in audio_frames) with io.BytesIO() as buffer: @@ -195,7 +203,7 @@ class LLMContext: wf.setframerate(sample_rate) wf.writeframes(data) - encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") + encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") return encoded_audio encoded_audio = await asyncio.to_thread(encode_audio) From 796f3aeff39a0d35da35c71df55a89b4bc9d3dcf Mon Sep 17 00:00:00 2001 From: Hwuiwon Kim Date: Tue, 16 Dec 2025 15:56:08 -0500 Subject: [PATCH 2/2] fix --- changelog/3234.fixed.md | 1 + src/pipecat/processors/aggregators/llm_context.py | 15 +++------------ 2 files changed, 4 insertions(+), 12 deletions(-) create mode 100644 changelog/3234.fixed.md diff --git a/changelog/3234.fixed.md b/changelog/3234.fixed.md new file mode 100644 index 000000000..022241279 --- /dev/null +++ b/changelog/3234.fixed.md @@ -0,0 +1 @@ +- Fix a bug in LLM context audio content handling \ No newline at end of file diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index be2868122..4bd4ad08a 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -87,19 +87,10 @@ class LLMContext: # Convert tools to ToolsSchema if needed. # If the tools are already a ToolsSchema, this is a no-op. # Otherwise, we wrap them in a shim ToolsSchema. - converted_tools: ToolsSchema | NotGiven - raw_tools = openai_context.tools - if isinstance(raw_tools, list): + converted_tools = openai_context.tools + if isinstance(converted_tools, list): converted_tools = ToolsSchema( - standard_tools=[], custom_tools={AdapterType.SHIM: raw_tools} - ) - elif isinstance(raw_tools, ToolsSchema): - converted_tools = raw_tools - elif raw_tools is NOT_GIVEN: - converted_tools = NOT_GIVEN - else: - raise TypeError( - f"Unsupported tools type when converting OpenAI context: {type(raw_tools)}" + standard_tools=[], custom_tools={AdapterType.SHIM: converted_tools} ) return LLMContext( messages=openai_context.get_messages(),