From fee2648ac0386bf15926a6646513bd01400627d1 Mon Sep 17 00:00:00 2001 From: Thu Nguyen <34468120+thsunkid@users.noreply.github.com> Date: Fri, 9 May 2025 18:20:02 +0800 Subject: [PATCH 1/2] Handle audio transcript from gpt-4o-audio and clean up logs --- src/pipecat/services/anthropic/llm.py | 2 +- src/pipecat/services/google/llm.py | 2 +- src/pipecat/services/openai/base_llm.py | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 43fe5ab6f..1ee344592 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -166,7 +166,7 @@ class AnthropicLLMService(LLMService): await self.start_processing_metrics() logger.debug( - f"{self}: Generating chat [{context.system}] | [{context.get_messages_for_logging()}]" + f"{self}: Generating chat [{context.system}] | {context.get_messages_for_logging()}" ) messages = context.messages diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index e5fcf56db..9aeda51a9 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -508,7 +508,7 @@ class GoogleLLMService(LLMService): try: logger.debug( # f"{self}: Generating chat [{self._system_instruction}] | [{context.get_messages_for_logging()}]" - f"{self}: Generating chat [{context.get_messages_for_logging()}]" + f"{self}: Generating chat {context.get_messages_for_logging()}" ) messages = context.messages diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index e24ad1d21..640717a15 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -154,7 +154,7 @@ class BaseOpenAILLMService(LLMService): async def _stream_chat_completions( self, context: OpenAILLMContext ) -> AsyncStream[ChatCompletionChunk]: - logger.debug(f"{self}: Generating chat [{context.get_messages_for_logging()}]") + logger.debug(f"{self}: Generating chat {context.get_messages_for_logging()}") messages: List[ChatCompletionMessageParam] = context.get_messages() @@ -240,6 +240,13 @@ class BaseOpenAILLMService(LLMService): elif chunk.choices[0].delta.content: await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) + # When gpt-4o-audio / gpt-4o-mini-audio is used for llm or stt+llm + # we need to get LLMTextFrame for the transcript + elif hasattr(chunk.choices[0].delta, "audio") and chunk.choices[0].delta.audio.get( + "transcript" + ): + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.audio["transcript"])) + # if we got a function name and arguments, check to see if it's a function with # a registered handler. If so, run the registered callback, save the result to # the context, and re-prompt to get a chat answer. If we don't have a registered From 15a5028d2343f092cc4e53f249fdaf48e0051bbd Mon Sep 17 00:00:00 2001 From: Thu Nguyen <34468120+thsunkid@users.noreply.github.com> Date: Wed, 14 May 2025 22:28:25 +0800 Subject: [PATCH 2/2] Revert log changes --- src/pipecat/services/anthropic/llm.py | 2 +- src/pipecat/services/google/llm.py | 2 +- src/pipecat/services/openai/base_llm.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 1ee344592..43fe5ab6f 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -166,7 +166,7 @@ class AnthropicLLMService(LLMService): await self.start_processing_metrics() logger.debug( - f"{self}: Generating chat [{context.system}] | {context.get_messages_for_logging()}" + f"{self}: Generating chat [{context.system}] | [{context.get_messages_for_logging()}]" ) messages = context.messages diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 9aeda51a9..e5fcf56db 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -508,7 +508,7 @@ class GoogleLLMService(LLMService): try: logger.debug( # f"{self}: Generating chat [{self._system_instruction}] | [{context.get_messages_for_logging()}]" - f"{self}: Generating chat {context.get_messages_for_logging()}" + f"{self}: Generating chat [{context.get_messages_for_logging()}]" ) messages = context.messages diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 640717a15..641c96f5e 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -154,7 +154,7 @@ class BaseOpenAILLMService(LLMService): async def _stream_chat_completions( self, context: OpenAILLMContext ) -> AsyncStream[ChatCompletionChunk]: - logger.debug(f"{self}: Generating chat {context.get_messages_for_logging()}") + logger.debug(f"{self}: Generating chat [{context.get_messages_for_logging()}]") messages: List[ChatCompletionMessageParam] = context.get_messages()