From 6fa797c8e48e7efe89f64b27da81cfbbc7860721 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 16 Jan 2026 22:01:39 -0500 Subject: [PATCH 1/4] Fix AWS Nova Sonic `reset_conversation()`, which would previously error out. Issues: - After disconnecting, we were prematurely sending audio messages using the new prompt and content names, before the new prompt and content were created - We weren't properly sending system instruction and conversation history messages to Nova Sonic with `"interactive": false` --- src/pipecat/services/aws/nova_sonic/llm.py | 29 ++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index fbcbe292e..e159ae9f6 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -296,6 +296,7 @@ class AWSNovaSonicLLMService(LLMService): self._user_text_buffer = "" self._assistant_text_buffer = "" self._completed_tool_calls = set() + self._audio_input_started = False file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav") with wave.open(file_path.open("rb"), "rb") as wav_file: @@ -533,9 +534,16 @@ class AWSNovaSonicLLMService(LLMService): await self._send_text_event(text=system_instruction, role=Role.SYSTEM) # Send conversation history - for message in llm_connection_params["messages"]: + messages = llm_connection_params["messages"] + for i, message in enumerate(messages): # logger.debug(f"Seeding conversation history with message: {message}") - await self._send_text_event(text=message.text, role=message.role) + # If last message is from user, mark it as interactive to trigger + # bot response + is_last_message = i == len(messages) - 1 + interactive = is_last_message and message.role == Role.USER + await self._send_text_event( + text=message.text, role=message.role, interactive=interactive + ) # Start audio input await self._send_audio_input_start_event() @@ -602,6 +610,7 @@ class AWSNovaSonicLLMService(LLMService): self._user_text_buffer = "" self._assistant_text_buffer = "" self._completed_tool_calls = set() + self._audio_input_started = False logger.info("Finished disconnecting") except Exception as e: @@ -727,8 +736,18 @@ class AWSNovaSonicLLMService(LLMService): }} ''' await self._send_client_event(audio_content_start) + self._audio_input_started = True - async def _send_text_event(self, text: str, role: Role): + async def _send_text_event(self, text: str, role: Role, interactive: bool = False): + """Send a text event to the LLM. + + Args: + text: The text content to send. + role: The role associated with the text (e.g., USER, ASSISTANT, SYSTEM). + interactive: Whether the content is interactive. Defaults to False. + False: conversation history or system instruction, sent prior to interactive audio + True: text input sent during (or at the start of) interactive audio + """ if not self._stream or not self._prompt_name or not text: return @@ -741,7 +760,7 @@ class AWSNovaSonicLLMService(LLMService): "promptName": "{self._prompt_name}", "contentName": "{content_name}", "type": "TEXT", - "interactive": true, + "interactive": {json.dumps(interactive)}, "role": "{role.value}", "textInputConfiguration": {{ "mediaType": "text/plain" @@ -779,7 +798,7 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(text_content_end) async def _send_user_audio_event(self, audio: bytes): - if not self._stream: + if not self._stream or not self._audio_input_started: return blob = base64.b64encode(audio) From c89083e72ec405445ed47a644bcd5ff16ca00442 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 16 Jan 2026 22:26:31 -0500 Subject: [PATCH 2/4] Improve 20e example to ask the bot to give a recap when loading a previous conversation from disk --- .../foundational/20e-persistent-context-aws-nova-sonic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index bd95fb7e7..ebd4afbf1 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -114,6 +114,14 @@ async def load_conversation(params: FunctionCallParams): # "content": f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}", # } # ) + # If the last message isn't from the user, add a message asking for a recap + if messages and messages[-1].get("role") != "user": + messages.append( + { + "role": "user", + "content": "Can you catch me up on what we were talking about?", + } + ) params.context.set_messages(messages) await params.llm.reset_conversation() # await params.llm.trigger_assistant_response() From b4d143e39b62c49542158cc392a3efa2702e7760 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 16 Jan 2026 22:36:35 -0500 Subject: [PATCH 3/4] Add CHANGELOG for fixing `AWSNovaSonicLLMService.reset_conversation()` --- changelog/3486.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/3486.fixed.md diff --git a/changelog/3486.fixed.md b/changelog/3486.fixed.md new file mode 100644 index 000000000..a02427e6e --- /dev/null +++ b/changelog/3486.fixed.md @@ -0,0 +1 @@ +- Fixed `AWSNovaSonicLLMService.reset_conversation()`, which would previously error out. Now it successfully reconnects and "rehydrates" from the context object. From 06b3ecd2d69d387a52a550672147e0d82468ba87 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 16 Jan 2026 22:43:49 -0500 Subject: [PATCH 4/4] In AWS Nova Sonic service, send the "interactive" user message (which triggers the bot response) only after sending the audio input start event, per the AWS team's recommendation --- src/pipecat/services/aws/nova_sonic/llm.py | 23 +++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index e159ae9f6..05baba2bd 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -533,21 +533,30 @@ class AWSNovaSonicLLMService(LLMService): if system_instruction: await self._send_text_event(text=system_instruction, role=Role.SYSTEM) - # Send conversation history + # Send conversation history (except for the last message if it's from the + # user, which we'll send as interactive after starting audio input) messages = llm_connection_params["messages"] + last_user_message = None for i, message in enumerate(messages): # logger.debug(f"Seeding conversation history with message: {message}") - # If last message is from user, mark it as interactive to trigger - # bot response is_last_message = i == len(messages) - 1 - interactive = is_last_message and message.role == Role.USER - await self._send_text_event( - text=message.text, role=message.role, interactive=interactive - ) + if is_last_message and message.role == Role.USER: + # Save for sending after audio input starts + last_user_message = message + else: + await self._send_text_event(text=message.text, role=message.role) # Start audio input await self._send_audio_input_start_event() + # Now send the last user message as interactive to trigger bot response + if last_user_message: + # logger.debug( + # f"Sending last user message as interactive to trigger bot response: {last_user_message}") + await self._send_text_event( + text=last_user_message.text, role=last_user_message.role, interactive=True + ) + # Start receiving events self._receive_task = self.create_task(self._receive_task_handler())