From 5e5626f04fe59d57fdc4982bcdc7fa48467f7c4f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 23 Apr 2025 11:40:36 -0400 Subject: [PATCH 01/55] [WIP] AWS Nova Sonic service --- examples/foundational/39-aws-nova-sonic.py | 115 ++++++++++++++++++ pyproject.toml | 2 +- .../services/aws_nova_sonic/__init__.py | 1 + src/pipecat/services/aws_nova_sonic/aws.py | 101 +++++++++++++++ 4 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 examples/foundational/39-aws-nova-sonic.py create mode 100644 src/pipecat/services/aws_nova_sonic/__init__.py create mode 100644 src/pipecat/services/aws_nova_sonic/aws.py diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py new file mode 100644 index 000000000..33fbbe477 --- /dev/null +++ b/examples/foundational/39-aws-nova-sonic.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMMessagesAppendFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.aws_nova_sonic import AWSNovaSonicService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +# Load environment variables +load_dotenv(override=True) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + # Initialize the SmallWebRTCTransport with the connection + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + camera_in_enabled=False, + vad_enabled=True, + vad_audio_passthrough=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + ) + + # Create the AWS Nova Sonic LLM service + # TODO: system instruction + # system_instruction = f""" + # You are a helpful AI assistant. + # Your goal is to demonstrate your capabilities in a helpful and engaging way. + # Your output will be converted to audio so don't include special characters in your answers. + # Respond to what the user said in a creative and helpful way. + # """ + + llm = AWSNovaSonicService( + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + region=os.getenv("AWS_REGION"), + ) + + # Build the pipeline + pipeline = Pipeline( + [ + transport.input(), + llm, + transport.output(), + ] + ) + + # Configure the pipeline task + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + # Handle client connection event + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames( + [ + LLMMessagesAppendFrame( + messages=[ + { + "role": "user", + "content": f"Greet the user and introduce yourself.", + } + ] + ) + ] + ) + + # Handle client disconnection events + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + # Run the pipeline + runner = PipelineRunner(handle_sigint=False) + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/pyproject.toml b/pyproject.toml index 13305933b..d6d05c00c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ Website = "https://pipecat.ai" [project.optional-dependencies] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "assemblyai~=0.37.0" ] -aws = [ "boto3~=1.37.16", "websockets~=13.1" ] +aws = [ "boto3~=1.37.16", "websockets~=13.1", "aws_sdk_bedrock_runtime~=0.0.2" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] cerebras = [] diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py new file mode 100644 index 000000000..b5559715a --- /dev/null +++ b/src/pipecat/services/aws_nova_sonic/__init__.py @@ -0,0 +1 @@ +from .aws import AWSNovaSonicService diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py new file mode 100644 index 000000000..3caf16761 --- /dev/null +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -0,0 +1,101 @@ +from aws_sdk_bedrock_runtime.client import ( + BedrockRuntimeClient, + InvokeModelWithBidirectionalStreamOperationInput, +) +from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme +from aws_sdk_bedrock_runtime.models import ( + BidirectionalInputPayloadPart, + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamInputChunk, + InvokeModelWithBidirectionalStreamOperationOutput, + InvokeModelWithBidirectionalStreamOutput, +) +from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver +from smithy_aws_core.identity import AWSCredentialsIdentity +from smithy_core.aio.eventstream import DuplexEventStream + +from pipecat.frames.frames import CancelFrame, EndFrame, StartFrame +from pipecat.services.llm_service import LLMService + + +class AWSNovaSonicService(LLMService): + def __init__( + self, + *, + secret_access_key: str, + access_key_id: str, + region: str, + model: str = "amazon.nova-sonic-v1:0", + **kwargs, + ): + super().__init__(**kwargs) + self._secret_access_key = secret_access_key + self._access_key_id = access_key_id + self._region = region + self._model = model + self._client: BedrockRuntimeClient = None + self._stream: DuplexEventStream[ + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamOutput, + InvokeModelWithBidirectionalStreamOperationOutput, + ] = None + self._receive_task = None + + # + # standard AIService frame handling + # + + async def start(self, frame: StartFrame): + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._disconnect() + + # + # communication + # + + async def _connect(self): + if self._client: + # Here we assume that if we have a client we are connected. + return + self._initialize_client() + self._stream = await self._client.invoke_model_with_bidirectional_stream( + InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) + ) + self._receive_task = self.create_task(self._receive_task_handler()) + pass + + async def _disconnect(self): + pass + + def _initialize_client(self) -> BedrockRuntimeClient: + config = Config( + endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", + region=self._region, + aws_credentials_identity_resolver=StaticCredentialsResolver( + credentials=AWSCredentialsIdentity( + access_key_id=self._access_key_id, + secret_access_key=self._secret_access_key, + # TODO: add additional stuff like aws_session_token + ) + ), + http_auth_scheme_resolver=HTTPAuthSchemeResolver(), + http_auth_schemes={"aws.auth#sigv4": SigV4AuthScheme()}, + ) + self._client = BedrockRuntimeClient(config=config) + + async def _send_client_event(self, event_json): + event = InvokeModelWithBidirectionalStreamInputChunk( + value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) + ) + await self._stream.input_stream.send(event) + + async def _receive_task_handler(self): + pass From a9e395b3660f873732cc89886ab3846a368be406 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 23 Apr 2025 14:05:04 -0400 Subject: [PATCH 02/55] [WIP] AWS Nova Sonic service --- examples/foundational/39-aws-nova-sonic.py | 14 +- src/pipecat/services/aws_nova_sonic/aws.py | 201 +++++++++++++++++++-- 2 files changed, 194 insertions(+), 21 deletions(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index 33fbbe477..266680542 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -43,15 +43,15 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): ) # Create the AWS Nova Sonic LLM service - # TODO: system instruction - # system_instruction = f""" - # You are a helpful AI assistant. - # Your goal is to demonstrate your capabilities in a helpful and engaging way. - # Your output will be converted to audio so don't include special characters in your answers. - # Respond to what the user said in a creative and helpful way. - # """ + system_instruction = f""" + You are a helpful AI assistant. + Your goal is to demonstrate your capabilities in a helpful and engaging way. + Your output will be converted to audio so don't include special characters in your answers. + Respond to what the user said in a creative and helpful way. + """ llm = AWSNovaSonicService( + instruction=system_instruction, secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), region=os.getenv("AWS_REGION"), diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 3caf16761..d94587879 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -1,3 +1,7 @@ +import base64 +import uuid +from enum import Enum + from aws_sdk_bedrock_runtime.client import ( BedrockRuntimeClient, InvokeModelWithBidirectionalStreamOperationInput, @@ -10,18 +14,26 @@ from aws_sdk_bedrock_runtime.models import ( InvokeModelWithBidirectionalStreamOperationOutput, InvokeModelWithBidirectionalStreamOutput, ) +from loguru import logger from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream -from pipecat.frames.frames import CancelFrame, EndFrame, StartFrame +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, InputAudioRawFrame, StartFrame +from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService +class Role(Enum): + SYSTEM = "SYSTEM" + USER = "USER" + + class AWSNovaSonicService(LLMService): def __init__( self, *, + instruction: str, secret_access_key: str, access_key_id: str, region: str, @@ -29,6 +41,7 @@ class AWSNovaSonicService(LLMService): **kwargs, ): super().__init__(**kwargs) + self._instruction = instruction self._secret_access_key = secret_access_key self._access_key_id = access_key_id self._region = region @@ -40,6 +53,8 @@ class AWSNovaSonicService(LLMService): InvokeModelWithBidirectionalStreamOperationOutput, ] = None self._receive_task = None + self._prompt_name = str(uuid.uuid4()) + self._input_audio_content_name = str(uuid.uuid4()) # # standard AIService frame handling @@ -58,24 +73,54 @@ class AWSNovaSonicService(LLMService): await self._disconnect() # - # communication + # frame processing + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, InputAudioRawFrame): + # TODO: check if _audio_input_paused? what causes that? + await self._send_user_audio(frame) + + await self.push_frame(frame, direction) + + # + # communication with LLM # async def _connect(self): - if self._client: - # Here we assume that if we have a client we are connected. - return - self._initialize_client() - self._stream = await self._client.invoke_model_with_bidirectional_stream( - InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) - ) - self._receive_task = self.create_task(self._receive_task_handler()) - pass + try: + if self._client: + # Here we assume that if we have a client we are connected + return + + # Create the client + self._client = self._create_client() + + # Start the bidirectional stream + self._stream = await self._client.invoke_model_with_bidirectional_stream( + InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) + ) + + # Send session start events + await self._send_session_start() + + # Send initial system instruction + await self._send_text(text=self._instruction, role=Role.SYSTEM) + + # Start audio input + await self._send_audio_input_start() + + self._receive_task = self.create_task(self._receive_task_handler()) + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._client = None async def _disconnect(self): pass - def _initialize_client(self) -> BedrockRuntimeClient: + def _create_client(self) -> BedrockRuntimeClient: config = Config( endpoint_uri=f"https://bedrock-runtime.{self._region}.amazonaws.com", region=self._region, @@ -89,9 +134,137 @@ class AWSNovaSonicService(LLMService): http_auth_scheme_resolver=HTTPAuthSchemeResolver(), http_auth_schemes={"aws.auth#sigv4": SigV4AuthScheme()}, ) - self._client = BedrockRuntimeClient(config=config) + return BedrockRuntimeClient(config=config) - async def _send_client_event(self, event_json): + # TODO: make params configurable? + async def _send_session_start(self): + session_start = """ + { + "event": { + "sessionStart": { + "inferenceConfiguration": { + "maxTokens": 1024, + "topP": 0.9, + "temperature": 0.7 + } + } + } + } + """ + await self._send_client_event(session_start) + + prompt_start = f''' + {{ + "event": {{ + "promptStart": {{ + "promptName": "{self._prompt_name}", + "textOutputConfiguration": {{ + "mediaType": "text/plain" + }}, + "audioOutputConfiguration": {{ + "mediaType": "audio/lpcm", + "sampleRateHertz": 24000, + "sampleSizeBits": 16, + "channelCount": 1, + "voiceId": "matthew", + "encoding": "base64", + "audioType": "SPEECH" + }} + }} + }} + }} + ''' + await self._send_client_event(prompt_start) + + async def _send_audio_input_start(self): + audio_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{self._input_audio_content_name}", + "type": "AUDIO", + "interactive": true, + "role": "USER", + "audioInputConfiguration": {{ + "mediaType": "audio/lpcm", + "sampleRateHertz": 16000, + "sampleSizeBits": 16, + "channelCount": 1, + "audioType": "SPEECH", + "encoding": "base64" + }} + }} + }} + }} + ''' + await self._send_client_event(audio_content_start) + + async def _send_text(self, text: str, role: Role): + content_name = str(uuid.uuid4()) + + text_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "type": "TEXT", + "interactive": true, + "role": "{role.value}", + "textInputConfiguration": {{ + "mediaType": "text/plain" + }} + }} + }} + }} + ''' + await self._send_client_event(text_content_start) + + text_input = f''' + {{ + "event": {{ + "textInput": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "content": "{text}" + }} + }} + }} + ''' + await self._send_client_event(text_input) + + text_content_end = f''' + {{ + "event": {{ + "contentEnd": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}" + }} + }} + }} + ''' + await self._send_client_event(text_content_end) + + async def _send_user_audio(self, frame: InputAudioRawFrame): + if not self._client: + return + + blob = base64.b64encode(frame.audio) + audio_event = f''' + {{ + "event": {{ + "audioInput": {{ + "promptName": "{self._prompt_name}", + "contentName": "{self._input_audio_content_name}", + "content": "{blob.decode("utf-8")}" + }} + }} + }} + ''' + await self._send_client_event(audio_event) + + async def _send_client_event(self, event_json: str): event = InvokeModelWithBidirectionalStreamInputChunk( value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) ) From 6d30f441e83d2a676a3c1210d6cc5127f28bd70f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 23 Apr 2025 15:00:04 -0400 Subject: [PATCH 03/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 55 +++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index d94587879..8bd2437bd 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -1,4 +1,5 @@ import base64 +import json import uuid from enum import Enum @@ -19,7 +20,14 @@ from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolv from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream -from pipecat.frames.frames import CancelFrame, EndFrame, Frame, InputAudioRawFrame, StartFrame +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InputAudioRawFrame, + StartFrame, + TTSAudioRawFrame, +) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService @@ -91,6 +99,8 @@ class AWSNovaSonicService(LLMService): async def _connect(self): try: + # TODO: remove after debugging + logger.debug("[pk] started connecting!") if self._client: # Here we assume that if we have a client we are connected return @@ -113,6 +123,8 @@ class AWSNovaSonicService(LLMService): await self._send_audio_input_start() self._receive_task = self.create_task(self._receive_task_handler()) + + logger.debug("[pk] finished connecting!") except Exception as e: logger.error(f"{self} initialization error: {e}") self._client = None @@ -271,4 +283,43 @@ class AWSNovaSonicService(LLMService): await self._stream.input_stream.send(event) async def _receive_task_handler(self): - pass + try: + while self._client: + # TODO: remove after debugging + logger.debug(f"[pk] awaiting output from server...") + + output = await self._stream.await_output() + + # TODO: remove after debugging + logger.debug(f"[pk] got output from server: {result}") + + result = await output[1].receive() + + # TODO: remove after debugging + logger.debug(f"[pk] got result from server: {result}") + + if result.value and result.value.bytes_: + response_data = result.value.bytes_.decode("utf-8") + json_data = json.loads(response_data) + + # TODO: remove after debugging + logger.debug(f"[pk] got JSON from server: {json_data}") + + if "audioOutput" in json_data["event"]: + self._handle_audio_output_event(json_data["event"]) + except Exception as e: + logger.error(f"{self} error processing responses: {e}") + + async def _handle_audio_output_event(self, event): + # TODO: remove after debugging + logger.debug("[pk] got output audio!") + audio_content = event["audioOutput"]["content"] + audio = base64.b64decode(audio_content) + # TODO: how is _current_audio_response used? + # TODO: make sample rate + channels (used in multiple places) consts + frame = TTSAudioRawFrame( + audio=audio, + sample_rate=24000, + num_channels=1, + ) + await self.push_frame(frame) From 7668b27fc0ac7019355d9e206f00ff4128a38905 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 23 Apr 2025 17:44:07 -0400 Subject: [PATCH 04/55] [WIP] AWS Nova Sonic service --- examples/foundational/39-aws-nova-sonic.py | 17 +++++++++++------ src/pipecat/services/aws_nova_sonic/aws.py | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index 266680542..fd7568d63 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -32,6 +32,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): webrtc_connection=webrtc_connection, params=TransportParams( audio_in_enabled=True, + audio_in_sample_rate=16000, audio_out_enabled=True, camera_in_enabled=False, vad_enabled=True, @@ -43,12 +44,16 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): ) # Create the AWS Nova Sonic LLM service - system_instruction = f""" - You are a helpful AI assistant. - Your goal is to demonstrate your capabilities in a helpful and engaging way. - Your output will be converted to audio so don't include special characters in your answers. - Respond to what the user said in a creative and helpful way. - """ + # system_instruction = f""" + # You are a helpful AI assistant. + # Your goal is to demonstrate your capabilities in a helpful and engaging way. + # Your output will be converted to audio so don't include special characters in your answers. + # Respond to what the user said in a creative and helpful way. + # """ + # TODO: looks like Nova Sonic can't handle new lines? + system_instruction = "You are a friendly assistant. The user and you will engage in a spoken dialog " \ + "exchanging the transcripts of a natural real-time conversation. Keep your responses short, " \ + "generally two or three sentences for chatty scenarios." llm = AWSNovaSonicService( instruction=system_instruction, diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 8bd2437bd..6cd953c3b 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -306,7 +306,7 @@ class AWSNovaSonicService(LLMService): logger.debug(f"[pk] got JSON from server: {json_data}") if "audioOutput" in json_data["event"]: - self._handle_audio_output_event(json_data["event"]) + await self._handle_audio_output_event(json_data["event"]) except Exception as e: logger.error(f"{self} error processing responses: {e}") From d789334a60e4fc7b3aa5c1952bcdfdb5bc43bbea Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 24 Apr 2025 10:29:27 -0400 Subject: [PATCH 05/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 23 ++-------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 6cd953c3b..aff0be2d2 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -99,8 +99,6 @@ class AWSNovaSonicService(LLMService): async def _connect(self): try: - # TODO: remove after debugging - logger.debug("[pk] started connecting!") if self._client: # Here we assume that if we have a client we are connected return @@ -123,8 +121,6 @@ class AWSNovaSonicService(LLMService): await self._send_audio_input_start() self._receive_task = self.create_task(self._receive_task_handler()) - - logger.debug("[pk] finished connecting!") except Exception as e: logger.error(f"{self} initialization error: {e}") self._client = None @@ -285,35 +281,20 @@ class AWSNovaSonicService(LLMService): async def _receive_task_handler(self): try: while self._client: - # TODO: remove after debugging - logger.debug(f"[pk] awaiting output from server...") - output = await self._stream.await_output() - - # TODO: remove after debugging - logger.debug(f"[pk] got output from server: {result}") - result = await output[1].receive() - # TODO: remove after debugging - logger.debug(f"[pk] got result from server: {result}") - if result.value and result.value.bytes_: response_data = result.value.bytes_.decode("utf-8") json_data = json.loads(response_data) - # TODO: remove after debugging - logger.debug(f"[pk] got JSON from server: {json_data}") - if "audioOutput" in json_data["event"]: await self._handle_audio_output_event(json_data["event"]) except Exception as e: logger.error(f"{self} error processing responses: {e}") - async def _handle_audio_output_event(self, event): - # TODO: remove after debugging - logger.debug("[pk] got output audio!") - audio_content = event["audioOutput"]["content"] + async def _handle_audio_output_event(self, event_json): + audio_content = event_json["audioOutput"]["content"] audio = base64.b64decode(audio_content) # TODO: how is _current_audio_response used? # TODO: make sample rate + channels (used in multiple places) consts From 13569a5a5a634a7ff65041d08ce12c410dd721bf Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 24 Apr 2025 13:55:09 -0400 Subject: [PATCH 06/55] [WIP] AWS Nova Sonic service --- examples/foundational/39-aws-nova-sonic.py | 14 +++-- src/pipecat/services/aws_nova_sonic/aws.py | 63 ++++++++++++++++++++-- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index fd7568d63..fffaee686 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -9,6 +9,7 @@ import os from dotenv import load_dotenv from loguru import logger +# import logging from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import LLMMessagesAppendFrame @@ -23,6 +24,11 @@ from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection # Load environment variables load_dotenv(override=True) +# logging.basicConfig( +# level=logging.DEBUG, +# format='%(asctime)s - %(levelname)s - %(message)s' +# ) + async def run_bot(webrtc_connection: SmallWebRTCConnection): logger.info(f"Starting bot") @@ -51,9 +57,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): # Respond to what the user said in a creative and helpful way. # """ # TODO: looks like Nova Sonic can't handle new lines? - system_instruction = "You are a friendly assistant. The user and you will engage in a spoken dialog " \ - "exchanging the transcripts of a natural real-time conversation. Keep your responses short, " \ - "generally two or three sentences for chatty scenarios." + system_instruction = ( + "You are a friendly assistant. The user and you will engage in a spoken dialog " + "exchanging the transcripts of a natural real-time conversation. Keep your responses short, " + "generally two or three sentences for chatty scenarios." + ) llm = AWSNovaSonicService( instruction=system_instruction, diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index aff0be2d2..f8517023e 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -94,7 +94,7 @@ class AWSNovaSonicService(LLMService): await self.push_frame(frame, direction) # - # communication with LLM + # LLM communication: lifecycle # async def _connect(self): @@ -144,6 +144,10 @@ class AWSNovaSonicService(LLMService): ) return BedrockRuntimeClient(config=config) + # + # LLM communication: input events (pipecat -> LLM) + # + # TODO: make params configurable? async def _send_session_start(self): session_start = """ @@ -278,6 +282,18 @@ class AWSNovaSonicService(LLMService): ) await self._stream.input_stream.send(event) + # + # LLM communication: output events (LLM -> pipecat) + # + + # Receive LLM responses ("completions"). + # Each response contains up to four pieces of content, delivered sequentially: + # - User transcription + # - Tool use (optional) + # - Text response + # - Audio response + # Each piece of content is wrapped by "contentStart" and "contentEnd" events. + # Each overall response is wrapped by "completionStart" and "completionEnd" events. async def _receive_task_handler(self): try: while self._client: @@ -288,13 +304,46 @@ class AWSNovaSonicService(LLMService): response_data = result.value.bytes_.decode("utf-8") json_data = json.loads(response_data) - if "audioOutput" in json_data["event"]: - await self._handle_audio_output_event(json_data["event"]) + if "event" in json_data: + event_json = json_data["event"] + if "completionStart" in event_json: + # Handle the LLM response starting + await self._handle_completion_start_event(event_json) + elif "contentStart" in event_json: + # Handle a piece of content starting + await self._handle_content_start_event(event_json) + elif "textOutput" in event_json: + # Handle text output content + await self._handle_text_output_event(event_json) + elif "audioOutput" in event_json: + # Handle audio output content + await self._handle_audio_output_event(event_json) + elif "contentEnd" in event_json: + # Handle a piece of content ending + await self._handle_content_end_event(event_json) + elif "completionStart" in event_json: + # Handle the LLM response ending + await self._handle_completion_end_event(event_json) + except Exception as e: logger.error(f"{self} error processing responses: {e}") + async def _handle_completion_start_event(self, event_json): + print("[pk] completion start") + + async def _handle_content_start_event(self, event_json): + content_start = event_json["contentStart"] + type = content_start["type"] + role = content_start["role"] + print(f"[pk] content start. type: {type}, role: {role}") + + async def _handle_text_output_event(self, event_json): + text_content = event_json["textOutput"]["content"] + print(f"[pk] text output. content: {text_content}") + async def _handle_audio_output_event(self, event_json): audio_content = event_json["audioOutput"]["content"] + print(f"[pk] audio output. content: {len(audio_content)}") audio = base64.b64decode(audio_content) # TODO: how is _current_audio_response used? # TODO: make sample rate + channels (used in multiple places) consts @@ -304,3 +353,11 @@ class AWSNovaSonicService(LLMService): num_channels=1, ) await self.push_frame(frame) + + async def _handle_content_end_event(self, event_json): + content_end = event_json["contentEnd"] + type = content_end["type"] + print(f"[pk] content end. type: {type}") + + async def _handle_completion_end_event(self, event_json): + print("[pk] completion end") From 8cbad070ade59d21df5f0c109545af197376f1cd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 24 Apr 2025 14:21:43 -0400 Subject: [PATCH 07/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index f8517023e..f28fe4360 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -89,7 +89,7 @@ class AWSNovaSonicService(LLMService): if isinstance(frame, InputAudioRawFrame): # TODO: check if _audio_input_paused? what causes that? - await self._send_user_audio(frame) + await self._send_user_audio_event(frame) await self.push_frame(frame, direction) @@ -112,13 +112,13 @@ class AWSNovaSonicService(LLMService): ) # Send session start events - await self._send_session_start() + await self._send_session_start_event() # Send initial system instruction - await self._send_text(text=self._instruction, role=Role.SYSTEM) + await self._send_text_event(text=self._instruction, role=Role.SYSTEM) # Start audio input - await self._send_audio_input_start() + await self._send_audio_input_start_event() self._receive_task = self.create_task(self._receive_task_handler()) except Exception as e: @@ -149,7 +149,7 @@ class AWSNovaSonicService(LLMService): # # TODO: make params configurable? - async def _send_session_start(self): + async def _send_session_start_event(self): session_start = """ { "event": { @@ -188,7 +188,7 @@ class AWSNovaSonicService(LLMService): ''' await self._send_client_event(prompt_start) - async def _send_audio_input_start(self): + async def _send_audio_input_start_event(self): audio_content_start = f''' {{ "event": {{ @@ -212,7 +212,7 @@ class AWSNovaSonicService(LLMService): ''' await self._send_client_event(audio_content_start) - async def _send_text(self, text: str, role: Role): + async def _send_text_event(self, text: str, role: Role): content_name = str(uuid.uuid4()) text_content_start = f''' @@ -258,7 +258,7 @@ class AWSNovaSonicService(LLMService): ''' await self._send_client_event(text_content_end) - async def _send_user_audio(self, frame: InputAudioRawFrame): + async def _send_user_audio_event(self, frame: InputAudioRawFrame): if not self._client: return @@ -357,7 +357,8 @@ class AWSNovaSonicService(LLMService): async def _handle_content_end_event(self, event_json): content_end = event_json["contentEnd"] type = content_end["type"] - print(f"[pk] content end. type: {type}") + stop_reason = content_end["stopReason"] + print(f"[pk] content end. type: {type}, stop_reason: {stop_reason}") async def _handle_completion_end_event(self, event_json): print("[pk] completion end") From b1d413b9be63779766aaa164114cdbc69f010cb1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 24 Apr 2025 15:23:01 -0400 Subject: [PATCH 08/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 28 +++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index f28fe4360..c271f49c5 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -286,14 +286,18 @@ class AWSNovaSonicService(LLMService): # LLM communication: output events (LLM -> pipecat) # - # Receive LLM responses ("completions"). - # Each response contains up to four pieces of content, delivered sequentially: - # - User transcription - # - Tool use (optional) - # - Text response - # - Audio response - # Each piece of content is wrapped by "contentStart" and "contentEnd" events. - # Each overall response is wrapped by "completionStart" and "completionEnd" events. + # Receive the ongoing LLM "completion". + # There is generally a single completion per session. + # In a completion, a few different kinds of content can be delivered: + # - Transcription of user audio + # - Tool use + # - Text preview of planned response speech before audio delivered + # - User interruption notification + # - Text of response speech that whose audio was actually delivered + # - Audio of response speech + # Each piece of content is wrapped by "contentStart" and "contentEnd" events. The content is + # delivered sequentially: one piece of content will end before another starts. + # The overall completion is wrapped by "completionStart" and "completionEnd" events. async def _receive_task_handler(self): try: while self._client: @@ -335,7 +339,13 @@ class AWSNovaSonicService(LLMService): content_start = event_json["contentStart"] type = content_start["type"] role = content_start["role"] - print(f"[pk] content start. type: {type}, role: {role}") + generation_stage = None + if "additionalModelFields" in content_start: + additional_model_fields = json.loads(content_start["additionalModelFields"]) + generation_stage = additional_model_fields.get("generationStage") + print( + f"[pk] content start. type: {type}, role: {role}, generation_stage: {generation_stage}" + ) async def _handle_text_output_event(self, event_json): text_content = event_json["textOutput"]["content"] From e40aa4f99a5b3e95fb8546c8fbe7b59749e64164 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 24 Apr 2025 15:56:28 -0400 Subject: [PATCH 09/55] [WIP] AWS Nova Sonic service - added TTSStartedFrame and TTSStoppedFrame --- src/pipecat/services/aws_nova_sonic/aws.py | 34 +++++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index c271f49c5..2e875f96f 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -27,6 +27,8 @@ from pipecat.frames.frames import ( InputAudioRawFrame, StartFrame, TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService @@ -63,6 +65,7 @@ class AWSNovaSonicService(LLMService): self._receive_task = None self._prompt_name = str(uuid.uuid4()) self._input_audio_content_name = str(uuid.uuid4()) + self._audio_response_ongoing = False # # standard AIService frame handling @@ -333,7 +336,8 @@ class AWSNovaSonicService(LLMService): logger.error(f"{self} error processing responses: {e}") async def _handle_completion_start_event(self, event_json): - print("[pk] completion start") + # print("[pk] completion start") + pass async def _handle_content_start_event(self, event_json): content_start = event_json["contentStart"] @@ -343,19 +347,26 @@ class AWSNovaSonicService(LLMService): if "additionalModelFields" in content_start: additional_model_fields = json.loads(content_start["additionalModelFields"]) generation_stage = additional_model_fields.get("generationStage") - print( - f"[pk] content start. type: {type}, role: {role}, generation_stage: {generation_stage}" - ) + # print( + # f"[pk] content start. type: {type}, role: {role}, generation_stage: {generation_stage}" + # ) async def _handle_text_output_event(self, event_json): text_content = event_json["textOutput"]["content"] - print(f"[pk] text output. content: {text_content}") + # print(f"[pk] text output. content: {text_content}") async def _handle_audio_output_event(self, event_json): audio_content = event_json["audioOutput"]["content"] print(f"[pk] audio output. content: {len(audio_content)}") + + # Report that *equivalent* of TTS (this is a speech-to-speech model) started + if not self._audio_response_ongoing: + self._audio_response_ongoing = True + # print("[pk] starting TTS") + await self.push_frame(TTSStartedFrame()) + + # Push audio frame audio = base64.b64decode(audio_content) - # TODO: how is _current_audio_response used? # TODO: make sample rate + channels (used in multiple places) consts frame = TTSAudioRawFrame( audio=audio, @@ -368,7 +379,14 @@ class AWSNovaSonicService(LLMService): content_end = event_json["contentEnd"] type = content_end["type"] stop_reason = content_end["stopReason"] - print(f"[pk] content end. type: {type}, stop_reason: {stop_reason}") + # print(f"[pk] content end. type: {type}, stop_reason: {stop_reason}") + + # Report that *equivalent* of TTS (this is a speech-to-speech model) stopped + if type == "AUDIO" and self._audio_response_ongoing: + print("[pk] stopping TTS") + self._audio_response_ongoing = False + await self.push_frame(TTSStoppedFrame()) async def _handle_completion_end_event(self, event_json): - print("[pk] completion end") + # print("[pk] completion end") + pass From de294caed9aaf2d3b37d6c2f0ec0ba7382b9d3f7 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 25 Apr 2025 15:12:37 -0400 Subject: [PATCH 10/55] [WIP] AWS Nova Sonic service - added LLMFullResponseStartFrame, LLMTextFrame, and LLMFullResponseEndFrame --- src/pipecat/services/aws_nova_sonic/aws.py | 124 +++++++++++++++++---- 1 file changed, 102 insertions(+), 22 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 2e875f96f..a2437b9dd 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -1,6 +1,7 @@ import base64 import json import uuid +from dataclasses import dataclass from enum import Enum from aws_sdk_bedrock_runtime.client import ( @@ -25,6 +26,9 @@ from pipecat.frames.frames import ( EndFrame, Frame, InputAudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, StartFrame, TTSAudioRawFrame, TTSStartedFrame, @@ -37,6 +41,36 @@ from pipecat.services.llm_service import LLMService class Role(Enum): SYSTEM = "SYSTEM" USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +class ContentType(Enum): + AUDIO = "AUDIO" + TEXT = "TEXT" + TOOL = "TOOL" + + +class TextStage(Enum): + FINAL = "FINAL" # what has been said + SPECULATIVE = "SPECULATIVE" # what's planned to be said + + +@dataclass +class CurrentContent: + type: ContentType + role: Role + text_stage: TextStage # None if not text + text_content: str # starts as None, then fills in if text + + def __str__(self): + return ( + f"CurrentContent(\n" + f" type={self.type.name},\n" + f" role={self.role.name},\n" + f" text_stage={self.text_stage.name if self.text_stage else 'None'}\n" + f")" + ) class AWSNovaSonicService(LLMService): @@ -65,7 +99,8 @@ class AWSNovaSonicService(LLMService): self._receive_task = None self._prompt_name = str(uuid.uuid4()) self._input_audio_content_name = str(uuid.uuid4()) - self._audio_response_ongoing = False + self._content_being_received = None # TODO: clean this up on error or when finished + self._assistant_is_responding = False # # standard AIService frame handling @@ -314,7 +349,7 @@ class AWSNovaSonicService(LLMService): if "event" in json_data: event_json = json_data["event"] if "completionStart" in event_json: - # Handle the LLM response starting + # Handle the LLM completion starting await self._handle_completion_start_event(event_json) elif "contentStart" in event_json: # Handle a piece of content starting @@ -329,7 +364,7 @@ class AWSNovaSonicService(LLMService): # Handle a piece of content ending await self._handle_content_end_event(event_json) elif "completionStart" in event_json: - # Handle the LLM response ending + # Handle the LLM completion ending await self._handle_completion_end_event(event_json) except Exception as e: @@ -347,24 +382,35 @@ class AWSNovaSonicService(LLMService): if "additionalModelFields" in content_start: additional_model_fields = json.loads(content_start["additionalModelFields"]) generation_stage = additional_model_fields.get("generationStage") - # print( - # f"[pk] content start. type: {type}, role: {role}, generation_stage: {generation_stage}" - # ) + + # Bookkeeping: track current content being received + content = CurrentContent( + type=ContentType(type), + role=Role(role), + text_stage=TextStage(generation_stage) if generation_stage else None, + text_content=None + ) + self._content_being_received = content + + if content.role == Role.ASSISTANT: + if content.type == ContentType.AUDIO: + # Report that *equivalent* of TTS (this is a speech-to-speech model) started + # print("[pk] TTS started") + await self.push_frame(TTSStartedFrame()) + + print(f"[pk] content start: {self._content_being_received}") async def _handle_text_output_event(self, event_json): text_content = event_json["textOutput"]["content"] - # print(f"[pk] text output. content: {text_content}") + print(f"[pk] text output. content: {text_content}") + + # Bookkeeping: augment the current content being received with text + content = self._content_being_received + content.text_content = text_content async def _handle_audio_output_event(self, event_json): audio_content = event_json["audioOutput"]["content"] - print(f"[pk] audio output. content: {len(audio_content)}") - - # Report that *equivalent* of TTS (this is a speech-to-speech model) started - if not self._audio_response_ongoing: - self._audio_response_ongoing = True - # print("[pk] starting TTS") - await self.push_frame(TTSStartedFrame()) - + # print(f"[pk] audio output. content: {len(audio_content)}") # Push audio frame audio = base64.b64decode(audio_content) # TODO: make sample rate + channels (used in multiple places) consts @@ -377,15 +423,49 @@ class AWSNovaSonicService(LLMService): async def _handle_content_end_event(self, event_json): content_end = event_json["contentEnd"] - type = content_end["type"] stop_reason = content_end["stopReason"] - # print(f"[pk] content end. type: {type}, stop_reason: {stop_reason}") + # print( + # f"[pk] content end: {self._content_being_received}.\n" + # f" stop_reason: {stop_reason}" + # ) - # Report that *equivalent* of TTS (this is a speech-to-speech model) stopped - if type == "AUDIO" and self._audio_response_ongoing: - print("[pk] stopping TTS") - self._audio_response_ongoing = False - await self.push_frame(TTSStoppedFrame()) + # Bookkeeping: clear current content being received + content = self._content_being_received + self._content_being_received = None + + if content and content.role == Role.ASSISTANT: + if content.type == ContentType.AUDIO: + # We got to the end of a chunk of the assistant's audio. + # Report that *equivalent* of TTS (this is a speech-to-speech model) stopped. + # print("[pk] TTS stopped") + await self.push_frame(TTSStoppedFrame()) + elif content.type == ContentType.TEXT: + # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) + if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": + # TODO: the way we're tracking the start and stop of the assistant response here + # is rather busted, and results in way too many "responses" being put into the + # context (every final text content block is treated as its own response). + # We *should* only record that an assistant response has ended when: + # - the assistant truly finished its turn (stop_reason is END_TURN) + # - when this is the next text content block after an INTERRUPTED has occurred + # BUT it seems like there's a bug where, if there are multiple assistant text + # content blocks, the *first* one gets marked END_TURN rather than the last. + print("[pk] LLM full response started") + self._assistant_is_responding = True + await self.push_frame(LLMFullResponseStartFrame()) + + if self._assistant_is_responding: + # Add text to the ongoing reported assistant response + print(f"[pk] LLM text: {content.text_content}") + await self.push_frame(LLMTextFrame(content.text_content)) + + # Report that the assistant has finished their response. + # TODO: kinda busted. see TODO comment above. + print("[pk] LLM full response ended") + await self.push_frame(LLMFullResponseEndFrame()) + self._assistant_is_responding = False + + self._content_being_received = False async def _handle_completion_end_event(self, event_json): # print("[pk] completion end") From 260f7c9b85f9c38b6d00dfb79c8f6ebbe021dcbf Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 25 Apr 2025 15:19:45 -0400 Subject: [PATCH 11/55] [WIP] AWS Nova Sonic service - format --- src/pipecat/services/aws_nova_sonic/aws.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index a2437b9dd..e9ce2013d 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -61,7 +61,7 @@ class CurrentContent: type: ContentType role: Role text_stage: TextStage # None if not text - text_content: str # starts as None, then fills in if text + text_content: str # starts as None, then fills in if text def __str__(self): return ( @@ -388,7 +388,7 @@ class AWSNovaSonicService(LLMService): type=ContentType(type), role=Role(role), text_stage=TextStage(generation_stage) if generation_stage else None, - text_content=None + text_content=None, ) self._content_being_received = content @@ -396,7 +396,7 @@ class AWSNovaSonicService(LLMService): if content.type == ContentType.AUDIO: # Report that *equivalent* of TTS (this is a speech-to-speech model) started # print("[pk] TTS started") - await self.push_frame(TTSStartedFrame()) + await self.push_frame(TTSStartedFrame()) print(f"[pk] content start: {self._content_being_received}") @@ -424,10 +424,7 @@ class AWSNovaSonicService(LLMService): async def _handle_content_end_event(self, event_json): content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] - # print( - # f"[pk] content end: {self._content_being_received}.\n" - # f" stop_reason: {stop_reason}" - # ) + print(f"[pk] content end: {self._content_being_received}.\n stop_reason: {stop_reason}") # Bookkeeping: clear current content being received content = self._content_being_received @@ -443,25 +440,25 @@ class AWSNovaSonicService(LLMService): # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": # TODO: the way we're tracking the start and stop of the assistant response here - # is rather busted, and results in way too many "responses" being put into the + # is rather busted, and results in way too many "responses" being put into the # context (every final text content block is treated as its own response). # We *should* only record that an assistant response has ended when: # - the assistant truly finished its turn (stop_reason is END_TURN) # - when this is the next text content block after an INTERRUPTED has occurred - # BUT it seems like there's a bug where, if there are multiple assistant text + # BUT it seems like there's a bug where, if there are multiple assistant text # content blocks, the *first* one gets marked END_TURN rather than the last. - print("[pk] LLM full response started") + # print("[pk] LLM full response started") self._assistant_is_responding = True await self.push_frame(LLMFullResponseStartFrame()) if self._assistant_is_responding: # Add text to the ongoing reported assistant response - print(f"[pk] LLM text: {content.text_content}") + # print(f"[pk] LLM text: {content.text_content}") await self.push_frame(LLMTextFrame(content.text_content)) # Report that the assistant has finished their response. # TODO: kinda busted. see TODO comment above. - print("[pk] LLM full response ended") + # print("[pk] LLM full response ended") await self.push_frame(LLMFullResponseEndFrame()) self._assistant_is_responding = False From a38206de9caf91d29364c72a879b303e76bad51d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 25 Apr 2025 15:33:45 -0400 Subject: [PATCH 12/55] [WIP] AWS Nova Sonic service - added TranscriptionFrame --- src/pipecat/services/aws_nova_sonic/aws.py | 41 ++++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index e9ce2013d..5ded76b81 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -30,12 +30,14 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, StartFrame, + TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService +from pipecat.utils.time import time_now_iso8601 class Role(Enum): @@ -398,19 +400,30 @@ class AWSNovaSonicService(LLMService): # print("[pk] TTS started") await self.push_frame(TTSStartedFrame()) - print(f"[pk] content start: {self._content_being_received}") + # print(f"[pk] content start: {self._content_being_received}") async def _handle_text_output_event(self, event_json): + # This should never happen + if not self._content_being_received: + return + text_content = event_json["textOutput"]["content"] - print(f"[pk] text output. content: {text_content}") + # print(f"[pk] text output. content: {text_content}") # Bookkeeping: augment the current content being received with text + # Assumption: only one text content per content block content = self._content_being_received content.text_content = text_content async def _handle_audio_output_event(self, event_json): + # This should never happen + if not self._content_being_received: + return + + # Get audio audio_content = event_json["audioOutput"]["content"] # print(f"[pk] audio output. content: {len(audio_content)}") + # Push audio frame audio = base64.b64decode(audio_content) # TODO: make sample rate + channels (used in multiple places) consts @@ -422,15 +435,19 @@ class AWSNovaSonicService(LLMService): await self.push_frame(frame) async def _handle_content_end_event(self, event_json): + # This should never happen + if not self._content_being_received: + return + content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] - print(f"[pk] content end: {self._content_being_received}.\n stop_reason: {stop_reason}") + # print(f"[pk] content end: {self._content_being_received}.\n stop_reason: {stop_reason}") # Bookkeeping: clear current content being received content = self._content_being_received self._content_being_received = None - if content and content.role == Role.ASSISTANT: + if content.role == Role.ASSISTANT: if content.type == ContentType.AUDIO: # We got to the end of a chunk of the assistant's audio. # Report that *equivalent* of TTS (this is a speech-to-speech model) stopped. @@ -447,20 +464,30 @@ class AWSNovaSonicService(LLMService): # - when this is the next text content block after an INTERRUPTED has occurred # BUT it seems like there's a bug where, if there are multiple assistant text # content blocks, the *first* one gets marked END_TURN rather than the last. - # print("[pk] LLM full response started") + print("[pk] LLM full response started") self._assistant_is_responding = True await self.push_frame(LLMFullResponseStartFrame()) if self._assistant_is_responding: # Add text to the ongoing reported assistant response - # print(f"[pk] LLM text: {content.text_content}") + print(f"[pk] LLM text: {content.text_content}") await self.push_frame(LLMTextFrame(content.text_content)) # Report that the assistant has finished their response. # TODO: kinda busted. see TODO comment above. - # print("[pk] LLM full response ended") + print("[pk] LLM full response ended") await self.push_frame(LLMFullResponseEndFrame()) self._assistant_is_responding = False + elif content.role == Role.USER: + if content.type == ContentType.TEXT: + if content.text_stage == TextStage.FINAL: + # Report a bit of user transcription + print(f"[pk] transcription: {content.text_content}") + await self.push_frame( + TranscriptionFrame( + text=content.text_content, user_id="", timestamp=time_now_iso8601() + ) + ) self._content_being_received = False From 0c255d26183d988c82111e007807f19d89105111 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 25 Apr 2025 16:49:59 -0400 Subject: [PATCH 13/55] [WIP] AWS Nova Sonic service - added TTSTextFrame and reworked/cleaned up some bookkeeping logic --- src/pipecat/services/aws_nova_sonic/aws.py | 101 +++++++++++++-------- 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 5ded76b81..4e5619f52 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -34,6 +34,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService @@ -394,13 +395,14 @@ class AWSNovaSonicService(LLMService): ) self._content_being_received = content + # print(f"[pk] content start: {self._content_being_received}") + if content.role == Role.ASSISTANT: if content.type == ContentType.AUDIO: - # Report that *equivalent* of TTS (this is a speech-to-speech model) started - # print("[pk] TTS started") - await self.push_frame(TTSStartedFrame()) - - # print(f"[pk] content start: {self._content_being_received}") + if not self._assistant_is_responding: + # The assistant has started responding. + self._assistant_is_responding = True + await self._report_assistant_started_responding() async def _handle_text_output_event(self, event_json): # This should never happen @@ -448,49 +450,76 @@ class AWSNovaSonicService(LLMService): self._content_being_received = None if content.role == Role.ASSISTANT: - if content.type == ContentType.AUDIO: - # We got to the end of a chunk of the assistant's audio. - # Report that *equivalent* of TTS (this is a speech-to-speech model) stopped. - # print("[pk] TTS stopped") - await self.push_frame(TTSStoppedFrame()) - elif content.type == ContentType.TEXT: + if content.type == ContentType.TEXT: # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": - # TODO: the way we're tracking the start and stop of the assistant response here - # is rather busted, and results in way too many "responses" being put into the - # context (every final text content block is treated as its own response). - # We *should* only record that an assistant response has ended when: - # - the assistant truly finished its turn (stop_reason is END_TURN) - # - when this is the next text content block after an INTERRUPTED has occurred - # BUT it seems like there's a bug where, if there are multiple assistant text - # content blocks, the *first* one gets marked END_TURN rather than the last. - print("[pk] LLM full response started") - self._assistant_is_responding = True - await self.push_frame(LLMFullResponseStartFrame()) + # TODO: shoot, for now we may need to "restart" the assistant responding because + # every FINAL text block has to be treated as its own response. See below TODO + # for more information. + if not self._assistant_is_responding: + self._assistant_is_responding = True + await self._report_assistant_started_responding() if self._assistant_is_responding: - # Add text to the ongoing reported assistant response - print(f"[pk] LLM text: {content.text_content}") - await self.push_frame(LLMTextFrame(content.text_content)) + # Text added to the ongoing assistant response + await self._report_assistant_response_text_added(content.text_content) - # Report that the assistant has finished their response. - # TODO: kinda busted. see TODO comment above. - print("[pk] LLM full response ended") - await self.push_frame(LLMFullResponseEndFrame()) + # Consider the assistant finished with their response. + # TODO: the way we're tracking the start/stop of the assistant response + # is rather busted, and results in way too many "responses" being put into + # the context (every FINAL text content block is treated as its own + # response). We *should* only record that an assistant response has ended + # when: + # - the assistant truly finished its turn (stop_reason is END_TURN) + # - when the assistant has been interrupted, and outputs what's actually + # been said + # BUT it seems like there's a bug where, if there are multiple assistant + # text content blocks, the *first* one gets marked END_TURN rather than the + # last. It's similarly unclear how to determine what the last text content + # block will be after an interruption. self._assistant_is_responding = False + await self._report_assistant_stopped_responding() elif content.role == Role.USER: if content.type == ContentType.TEXT: if content.text_stage == TextStage.FINAL: - # Report a bit of user transcription - print(f"[pk] transcription: {content.text_content}") - await self.push_frame( - TranscriptionFrame( - text=content.text_content, user_id="", timestamp=time_now_iso8601() - ) - ) + # User transcription text added + await self._report_user_transcription_text_added(content.text_content) self._content_being_received = False async def _handle_completion_end_event(self, event_json): # print("[pk] completion end") pass + + async def _report_assistant_started_responding(self): + # Report that the assistant has started their response. + print("[pk] LLM full response started") + await self.push_frame(LLMFullResponseStartFrame()) + + # Report that equivalent of TTS (this is a speech-to-speech model) started + print("[pk] TTS started") + await self.push_frame(TTSStartedFrame()) + + async def _report_assistant_response_text_added(self, text): + # Report some text added to the ongoing assistant response + print(f"[pk] LLM text: {text}") + await self.push_frame(LLMTextFrame(text)) + + # Report some text added to the *equivalent* of TTS (this is a speech-to-speech model) + print(f"[pk] TTS text: {text}") + await self.push_frame(TTSTextFrame(text)) + + async def _report_assistant_stopped_responding(self): + # Report that the assistant has finished their response. + print("[pk] LLM full response ended") + await self.push_frame(LLMFullResponseEndFrame()) + + # Report that equivalent of TTS (this is a speech-to-speech model) stopped. + print("[pk] TTS stopped") + await self.push_frame(TTSStoppedFrame()) + + async def _report_user_transcription_text_added(self, text): + print(f"[pk] transcription: {text}") + await self.push_frame( + TranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) + ) From 1f9baefba8bab6a13c46ca1af129544f520e2a5a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Sun, 27 Apr 2025 06:50:28 -0400 Subject: [PATCH 14/55] [WIP] AWS Nova Sonic service - added stubs for handling interruption and user-started-speaking frames --- src/pipecat/services/aws_nova_sonic/aws.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 4e5619f52..737bc82fb 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -30,11 +30,15 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, StartFrame, + StartInterruptionFrame, + StopInterruptionFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import LLMService @@ -131,6 +135,15 @@ class AWSNovaSonicService(LLMService): if isinstance(frame, InputAudioRawFrame): # TODO: check if _audio_input_paused? what causes that? await self._send_user_audio_event(frame) + # TODO: do we need to do anything for these? + elif isinstance(frame, StartInterruptionFrame): + print("[pk] StartInterruptionFrame") + elif isinstance(frame, UserStartedSpeakingFrame): + print("[pk] UserStartedSpeakingFrame") + elif isinstance(frame, StopInterruptionFrame): + print("[pk] StopInterruptionFrame") + elif isinstance(frame, UserStoppedSpeakingFrame): + print("[pk] UserStoppedSpeakingFrame") await self.push_frame(frame, direction) From 5b64613f65a3c6d28923bb2396a58c4909a05f4d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 28 Apr 2025 09:16:10 -0400 Subject: [PATCH 15/55] [WIP] AWS Nova Sonic service --- examples/foundational/39-aws-nova-sonic.py | 4 +--- src/pipecat/services/aws_nova_sonic/aws.py | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index fffaee686..567655002 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -43,9 +43,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): camera_in_enabled=False, vad_enabled=True, vad_audio_passthrough=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), ), ) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 737bc82fb..facf84a49 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -137,13 +137,17 @@ class AWSNovaSonicService(LLMService): await self._send_user_audio_event(frame) # TODO: do we need to do anything for these? elif isinstance(frame, StartInterruptionFrame): - print("[pk] StartInterruptionFrame") + # print("[pk] StartInterruptionFrame") + pass elif isinstance(frame, UserStartedSpeakingFrame): - print("[pk] UserStartedSpeakingFrame") + # print("[pk] UserStartedSpeakingFrame") + pass elif isinstance(frame, StopInterruptionFrame): - print("[pk] StopInterruptionFrame") + # print("[pk] StopInterruptionFrame") + pass elif isinstance(frame, UserStoppedSpeakingFrame): - print("[pk] UserStoppedSpeakingFrame") + # print("[pk] UserStoppedSpeakingFrame") + pass await self.push_frame(frame, direction) @@ -415,7 +419,7 @@ class AWSNovaSonicService(LLMService): if not self._assistant_is_responding: # The assistant has started responding. self._assistant_is_responding = True - await self._report_assistant_started_responding() + await self._report_assistant_response_started() async def _handle_text_output_event(self, event_json): # This should never happen @@ -471,7 +475,7 @@ class AWSNovaSonicService(LLMService): # for more information. if not self._assistant_is_responding: self._assistant_is_responding = True - await self._report_assistant_started_responding() + await self._report_assistant_response_started() if self._assistant_is_responding: # Text added to the ongoing assistant response @@ -491,7 +495,7 @@ class AWSNovaSonicService(LLMService): # last. It's similarly unclear how to determine what the last text content # block will be after an interruption. self._assistant_is_responding = False - await self._report_assistant_stopped_responding() + await self._report_assistant_response_ended() elif content.role == Role.USER: if content.type == ContentType.TEXT: if content.text_stage == TextStage.FINAL: @@ -504,7 +508,7 @@ class AWSNovaSonicService(LLMService): # print("[pk] completion end") pass - async def _report_assistant_started_responding(self): + async def _report_assistant_response_started(self): # Report that the assistant has started their response. print("[pk] LLM full response started") await self.push_frame(LLMFullResponseStartFrame()) @@ -522,7 +526,7 @@ class AWSNovaSonicService(LLMService): print(f"[pk] TTS text: {text}") await self.push_frame(TTSTextFrame(text)) - async def _report_assistant_stopped_responding(self): + async def _report_assistant_response_ended(self): # Report that the assistant has finished their response. print("[pk] LLM full response ended") await self.push_frame(LLMFullResponseEndFrame()) From 68c1069548c110dbad409bee788a2f571854bea0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 28 Apr 2025 10:37:11 -0400 Subject: [PATCH 16/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index facf84a49..dd7fd702b 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -412,7 +412,9 @@ class AWSNovaSonicService(LLMService): ) self._content_being_received = content - # print(f"[pk] content start: {self._content_being_received}") + # print(f"[pk] content start: {content}") + if content.role == Role.ASSISTANT: + print(f"[pk] assistant content start: {content}") if content.role == Role.ASSISTANT: if content.type == ContentType.AUDIO: @@ -425,13 +427,15 @@ class AWSNovaSonicService(LLMService): # This should never happen if not self._content_being_received: return + content = self._content_being_received text_content = event_json["textOutput"]["content"] # print(f"[pk] text output. content: {text_content}") + if content.role == Role.ASSISTANT: + print(f"[pk] assistant text output. content: {text_content}") # Bookkeeping: augment the current content being received with text # Assumption: only one text content per content block - content = self._content_being_received content.text_content = text_content async def _handle_audio_output_event(self, event_json): @@ -457,13 +461,15 @@ class AWSNovaSonicService(LLMService): # This should never happen if not self._content_being_received: return + content = self._content_being_received content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] - # print(f"[pk] content end: {self._content_being_received}.\n stop_reason: {stop_reason}") + # print(f"[pk] content end: {content}.\n stop_reason: {stop_reason}") + if content.role == Role.ASSISTANT: + print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") # Bookkeeping: clear current content being received - content = self._content_being_received self._content_being_received = None if content.role == Role.ASSISTANT: From 96d05e12fcd7fa821657b10b74ef6d564c4e279d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 28 Apr 2025 11:15:51 -0400 Subject: [PATCH 17/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 57 +++++++++++----------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index dd7fd702b..640658ca9 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -22,6 +22,7 @@ from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, CancelFrame, EndFrame, Frame, @@ -148,9 +149,29 @@ class AWSNovaSonicService(LLMService): elif isinstance(frame, UserStoppedSpeakingFrame): # print("[pk] UserStoppedSpeakingFrame") pass + elif isinstance(frame, BotStoppedSpeakingFrame): + await self._handle_bot_stopped_speaking() await self.push_frame(frame, direction) + async def _handle_bot_stopped_speaking(self): + if self._assistant_is_responding: + # Consider the assistant finished with their response. + # + # TODO: ideally we could base this solely on the LLM output events, but I couldn't + # figure out a reliable way to determine when we've gotten our last FINAL text block + # after the LLM is done talking. + # + # First I looked at stopReason, but it doesn't seem like the last FINAL text block is + # reliably marked END_TURN (sometimes the *first* one is, but not the last...bug?) + # + # Then I considered schemes where we tally or match up SPECULATIVE text blocks with + # FINAL text blocks to know how many or which FINAL blocks to expect, but user + # interruptions throw a wrench in these schemes: depending on the exact timing of the + # interruption, we should or shouldn't expect some FINAL blocks. + self._assistant_is_responding = False + await self._report_assistant_response_ended() + # # LLM communication: lifecycle # @@ -413,11 +434,12 @@ class AWSNovaSonicService(LLMService): self._content_being_received = content # print(f"[pk] content start: {content}") - if content.role == Role.ASSISTANT: - print(f"[pk] assistant content start: {content}") + # if content.role == Role.ASSISTANT: + # print(f"[pk] assistant content start: {content}") if content.role == Role.ASSISTANT: if content.type == ContentType.AUDIO: + # Note that an assistant response can comprise of multiple audio blocks if not self._assistant_is_responding: # The assistant has started responding. self._assistant_is_responding = True @@ -431,8 +453,8 @@ class AWSNovaSonicService(LLMService): text_content = event_json["textOutput"]["content"] # print(f"[pk] text output. content: {text_content}") - if content.role == Role.ASSISTANT: - print(f"[pk] assistant text output. content: {text_content}") + # if content.role == Role.ASSISTANT: + # print(f"[pk] assistant text output. content: {text_content}") # Bookkeeping: augment the current content being received with text # Assumption: only one text content per content block @@ -466,8 +488,8 @@ class AWSNovaSonicService(LLMService): content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] # print(f"[pk] content end: {content}.\n stop_reason: {stop_reason}") - if content.role == Role.ASSISTANT: - print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") + # if content.role == Role.ASSISTANT: + # print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") # Bookkeeping: clear current content being received self._content_being_received = None @@ -476,32 +498,9 @@ class AWSNovaSonicService(LLMService): if content.type == ContentType.TEXT: # Ignore non-final text, and the "interrupted" message (which isn't meaningful text) if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED": - # TODO: shoot, for now we may need to "restart" the assistant responding because - # every FINAL text block has to be treated as its own response. See below TODO - # for more information. - if not self._assistant_is_responding: - self._assistant_is_responding = True - await self._report_assistant_response_started() - if self._assistant_is_responding: # Text added to the ongoing assistant response await self._report_assistant_response_text_added(content.text_content) - - # Consider the assistant finished with their response. - # TODO: the way we're tracking the start/stop of the assistant response - # is rather busted, and results in way too many "responses" being put into - # the context (every FINAL text content block is treated as its own - # response). We *should* only record that an assistant response has ended - # when: - # - the assistant truly finished its turn (stop_reason is END_TURN) - # - when the assistant has been interrupted, and outputs what's actually - # been said - # BUT it seems like there's a bug where, if there are multiple assistant - # text content blocks, the *first* one gets marked END_TURN rather than the - # last. It's similarly unclear how to determine what the last text content - # block will be after an interruption. - self._assistant_is_responding = False - await self._report_assistant_response_ended() elif content.role == Role.USER: if content.type == ContentType.TEXT: if content.text_stage == TextStage.FINAL: From 9b8bce1914810971ef18bc8a01d4ab41287fc2e5 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 28 Apr 2025 13:18:09 -0400 Subject: [PATCH 18/55] [WIP] AWS Nova Sonic service - add voice_id --- examples/foundational/39-aws-nova-sonic.py | 1 + src/pipecat/services/aws_nova_sonic/aws.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index 567655002..445464957 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -66,6 +66,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), region=os.getenv("AWS_REGION"), + voice_id="tiffany", # matthew, tiffany, amy ) # Build the pipeline diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 640658ca9..eb57d9b80 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -90,6 +90,7 @@ class AWSNovaSonicService(LLMService): access_key_id: str, region: str, model: str = "amazon.nova-sonic-v1:0", + voice_id: str = "matthew", # matthew, tiffany, amy **kwargs, ): super().__init__(**kwargs) @@ -99,6 +100,7 @@ class AWSNovaSonicService(LLMService): self._region = region self._model = model self._client: BedrockRuntimeClient = None + self._voice_id = voice_id self._stream: DuplexEventStream[ InvokeModelWithBidirectionalStreamInput, InvokeModelWithBidirectionalStreamOutput, @@ -257,7 +259,7 @@ class AWSNovaSonicService(LLMService): "sampleRateHertz": 24000, "sampleSizeBits": 16, "channelCount": 1, - "voiceId": "matthew", + "voiceId": "{self._voice_id}", "encoding": "base64", "audioType": "SPEECH" }} From 9f7f42e885db0d68cfa619ad5720abfd536ce690 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 28 Apr 2025 13:41:55 -0400 Subject: [PATCH 19/55] [WIP] AWS Nova Sonic service --- src/pipecat/services/aws_nova_sonic/aws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index eb57d9b80..563d35422 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -193,7 +193,7 @@ class AWSNovaSonicService(LLMService): ) # Send session start events - await self._send_session_start_event() + await self._send_session_start_events() # Send initial system instruction await self._send_text_event(text=self._instruction, role=Role.SYSTEM) @@ -230,7 +230,7 @@ class AWSNovaSonicService(LLMService): # # TODO: make params configurable? - async def _send_session_start_event(self): + async def _send_session_start_events(self): session_start = """ { "event": { From f182eafb40dd320c874f963e7549ba64a4dfac8b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 29 Apr 2025 11:39:32 -0400 Subject: [PATCH 20/55] [WIP] AWS Nova Sonic service - add ability to pass in OpenAILLMContext --- examples/foundational/39-aws-nova-sonic.py | 47 ++--- .../services/aws_nova_sonic/__init__.py | 2 +- src/pipecat/services/aws_nova_sonic/aws.py | 187 +++++++++++++++--- .../services/aws_nova_sonic/context.py | 121 ++++++++++++ 4 files changed, 303 insertions(+), 54 deletions(-) create mode 100644 src/pipecat/services/aws_nova_sonic/context.py diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index 445464957..c44f85a48 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -16,7 +16,8 @@ from pipecat.frames.frames import LLMMessagesAppendFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.services.aws_nova_sonic import AWSNovaSonicService +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.aws_nova_sonic import AWSNovaSonicLLMService from pipecat.transports.base_transport import TransportParams from pipecat.transports.network.small_webrtc import SmallWebRTCTransport from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection @@ -47,13 +48,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): ), ) - # Create the AWS Nova Sonic LLM service - # system_instruction = f""" - # You are a helpful AI assistant. - # Your goal is to demonstrate your capabilities in a helpful and engaging way. - # Your output will be converted to audio so don't include special characters in your answers. - # Respond to what the user said in a creative and helpful way. - # """ + # Specify initial system instruction # TODO: looks like Nova Sonic can't handle new lines? system_instruction = ( "You are a friendly assistant. The user and you will engage in a spoken dialog " @@ -61,20 +56,37 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): "generally two or three sentences for chatty scenarios." ) - llm = AWSNovaSonicService( - instruction=system_instruction, + # Create the AWS Nova Sonic LLM service + llm = AWSNovaSonicLLMService( secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), region=os.getenv("AWS_REGION"), - voice_id="tiffany", # matthew, tiffany, amy + voice_id="tiffany", # matthew, tiffany, amy + # instruction=system_instruction # could pass instruction here rather than context, below ) + # Set up context and context management. + # AWSNovaSonicService will adapt OpenAI LLM context objects with standard message format to + # what's expected by Nova Sonic. + context = OpenAILLMContext( + messages=[ + {"role": "system", "content": f"{system_instruction}"}, + { + "role": "user", + "content": "Tell me hello! Don't wait for me to say anything else first!", + }, + ] + ) + context_aggregator = llm.create_context_aggregator(context) + # Build the pipeline pipeline = Pipeline( [ transport.input(), + context_aggregator.user(), llm, transport.output(), + context_aggregator.assistant(), ] ) @@ -93,18 +105,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): async def on_client_connected(transport, client): logger.info(f"Client connected") # Kick off the conversation. - await task.queue_frames( - [ - LLMMessagesAppendFrame( - messages=[ - { - "role": "user", - "content": f"Greet the user and introduce yourself.", - } - ] - ) - ] - ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) # Handle client disconnection events @transport.event_handler("on_client_disconnected") diff --git a/src/pipecat/services/aws_nova_sonic/__init__.py b/src/pipecat/services/aws_nova_sonic/__init__.py index b5559715a..e14c44f8a 100644 --- a/src/pipecat/services/aws_nova_sonic/__init__.py +++ b/src/pipecat/services/aws_nova_sonic/__init__.py @@ -1 +1 @@ -from .aws import AWSNovaSonicService +from .aws import AWSNovaSonicLLMService diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 563d35422..cc07e5463 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -3,6 +3,7 @@ import json import uuid from dataclasses import dataclass from enum import Enum +from typing import Any from aws_sdk_bedrock_runtime.client import ( BedrockRuntimeClient, @@ -41,18 +42,26 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.processors.aggregators.llm_response import ( + LLMAssistantAggregatorParams, + LLMUserAggregatorParams, +) +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws_nova_sonic.context import ( + AWSNovaSonicAssistantContextAggregator, + AWSNovaSonicContextAggregatorPair, + AWSNovaSonicLLMContext, + AWSNovaSonicUserContextAggregator, + Role, +) from pipecat.services.llm_service import LLMService from pipecat.utils.time import time_now_iso8601 -class Role(Enum): - SYSTEM = "SYSTEM" - USER = "USER" - ASSISTANT = "ASSISTANT" - TOOL = "TOOL" - - class ContentType(Enum): AUDIO = "AUDIO" TEXT = "TEXT" @@ -81,36 +90,40 @@ class CurrentContent: ) -class AWSNovaSonicService(LLMService): +class AWSNovaSonicLLMService(LLMService): def __init__( self, *, - instruction: str, + # TODO: if we have instruction here as an alternative to using context, we should do the same for tools...right? secret_access_key: str, access_key_id: str, region: str, model: str = "amazon.nova-sonic-v1:0", voice_id: str = "matthew", # matthew, tiffany, amy + instruction: str = None, **kwargs, ): super().__init__(**kwargs) - self._instruction = instruction self._secret_access_key = secret_access_key self._access_key_id = access_key_id self._region = region self._model = model self._client: BedrockRuntimeClient = None self._voice_id = voice_id + self._instruction = instruction + self._context: AWSNovaSonicLLMContext = None self._stream: DuplexEventStream[ InvokeModelWithBidirectionalStreamInput, InvokeModelWithBidirectionalStreamOutput, InvokeModelWithBidirectionalStreamOperationOutput, ] = None self._receive_task = None - self._prompt_name = str(uuid.uuid4()) - self._input_audio_content_name = str(uuid.uuid4()) - self._content_being_received = None # TODO: clean this up on error or when finished + self._prompt_name = None + self._input_audio_content_name = None + self._content_being_received = None self._assistant_is_responding = False + self._context_available = False + self._ready_to_send_context = False # # standard AIService frame handling @@ -118,7 +131,14 @@ class AWSNovaSonicService(LLMService): async def start(self, frame: StartFrame): await super().start(frame) - await self._connect() + # TODO: maybe connect but don't send history until we get all of our settings? + # how do we know how long to wait? + # ah, i think we'll *always* get at least one OpenAILLMContextFrame which kicks things off + # so we need to send the initial history when: + # - we're connected + # - we've gotten the first context + # i *think* this is what's controlled by _api_session_ready/_run_llm_when_api_session_ready + await self._start_connecting() async def stop(self, frame: EndFrame): await super().stop(frame) @@ -135,10 +155,14 @@ class AWSNovaSonicService(LLMService): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, InputAudioRawFrame): + if isinstance(frame, OpenAILLMContextFrame): + await self._handle_context(frame.context) + elif isinstance(frame, InputAudioRawFrame): # TODO: check if _audio_input_paused? what causes that? await self._send_user_audio_event(frame) - # TODO: do we need to do anything for these? + elif isinstance(frame, BotStoppedSpeakingFrame): + await self._handle_bot_stopped_speaking() + # TODO: do we need to do anything for the below four frame types? elif isinstance(frame, StartInterruptionFrame): # print("[pk] StartInterruptionFrame") pass @@ -151,11 +175,19 @@ class AWSNovaSonicService(LLMService): elif isinstance(frame, UserStoppedSpeakingFrame): # print("[pk] UserStoppedSpeakingFrame") pass - elif isinstance(frame, BotStoppedSpeakingFrame): - await self._handle_bot_stopped_speaking() await self.push_frame(frame, direction) + async def _handle_context(self, context: OpenAILLMContext): + # TODO: if context has changed, reconnect + # TODO: remove + print(f"[pk] _handle_context: {context.get_messages_for_initializing_history()}") + if not self._context: + # We got our initial context - try to finish connecting + self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic(context) + self._context_available = True + await self._finish_connecting_if_context_available() + async def _handle_bot_stopped_speaking(self): if self._assistant_is_responding: # Consider the assistant finished with their response. @@ -178,12 +210,16 @@ class AWSNovaSonicService(LLMService): # LLM communication: lifecycle # - async def _connect(self): + async def _start_connecting(self): try: if self._client: - # Here we assume that if we have a client we are connected + # Here we assume that if we have a client we are connected or connecting return + # Set IDs for the connection + self._prompt_name = str(uuid.uuid4()) + self._input_audio_content_name = str(uuid.uuid4()) + # Create the client self._client = self._create_client() @@ -195,19 +231,71 @@ class AWSNovaSonicService(LLMService): # Send session start events await self._send_session_start_events() - # Send initial system instruction - await self._send_text_event(text=self._instruction, role=Role.SYSTEM) - - # Start audio input - await self._send_audio_input_start_event() - - self._receive_task = self.create_task(self._receive_task_handler()) + # Finish connecting + self._ready_to_send_context = True + await self._finish_connecting_if_context_available() except Exception as e: logger.error(f"{self} initialization error: {e}") - self._client = None + self._disconnect() + + async def _finish_connecting_if_context_available(self): + # We can only finish connecting once we've gotten our initial context and we're ready to + # send it + if not (self._context_available and self._ready_to_send_context): + return + + # Read context + history = self._context.get_messages_for_initializing_history() + + # Send system instruction + # Instruction from context takes priority + instruction = history.instruction if history.instruction else self._instruction + if instruction: + await self._send_text_event(text=instruction, role=Role.SYSTEM) + + # Send conversation history + for message in history.messages: + await self._send_text_event(text=message.text, role=message.role) + + # Send initial context (system instruction and conversation history) + # TODO: finish implementing + # - pass additional message(s) + # - merge init-passed system instruction + context instruction (latter takes precedence) + # - merge init-passed tools + context tools (latter takes precedence) + await self._send_text_event(text=self._instruction, role=Role.SYSTEM) + + # Start audio input + await self._send_audio_input_start_event() + + # Start receiving events + self._receive_task = self.create_task(self._receive_task_handler()) async def _disconnect(self): - pass + try: + # Clean up receive task + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=1.0) + self._receive_task = None + + # Clean up client + if self._client: + await self._send_session_end_events() + self._client = None + + # Clean up stream + if self._stream: + await self._stream.input_stream.close() + self._stream = None + + # Reset remaining connection-specific state + self._prompt_name = None + self._input_audio_content_name = None + self._content_being_received = None + self._assistant_is_responding = False + self._context_available = False + self._ready_to_send_context = False + except Exception as e: + logger.error(f"{self} error disconnecting: {e}") def _create_client(self) -> BedrockRuntimeClient: config = Config( @@ -340,7 +428,7 @@ class AWSNovaSonicService(LLMService): await self._send_client_event(text_content_end) async def _send_user_audio_event(self, frame: InputAudioRawFrame): - if not self._client: + if not self._stream: return blob = base64.b64encode(frame.audio) @@ -357,6 +445,30 @@ class AWSNovaSonicService(LLMService): ''' await self._send_client_event(audio_event) + async def _send_session_end_events(self): + if not self._stream: + return + + prompt_end = f''' + {{ + "event": {{ + "promptEnd": {{ + "promptName": "{self._prompt_name}" + }} + }} + }} + ''' + await self._send_client_event(prompt_end) + + session_end = """ + { + "event": { + "sessionEnd": {} + } + } + """ + await self._send_client_event(session_end) + async def _send_client_event(self, event_json: str): event = InvokeModelWithBidirectionalStreamInputChunk( value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) @@ -547,3 +659,18 @@ class AWSNovaSonicService(LLMService): await self.push_frame( TranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) ) + + # + # Context + # + + def create_context_aggregator( + self, + context: OpenAILLMContext, + *, + user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), + assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), + ) -> AWSNovaSonicContextAggregatorPair: + user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) + assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) + return AWSNovaSonicContextAggregatorPair(user, assistant) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py new file mode 100644 index 000000000..331ecc13e --- /dev/null +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -0,0 +1,121 @@ +import copy +from dataclasses import dataclass, field +from enum import Enum + +from loguru import logger + +from pipecat.frames.frames import DataFrame, Frame, LLMMessagesUpdateFrame, LLMSetToolsFrame +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + + +class Role(Enum): + SYSTEM = "SYSTEM" + USER = "USER" + ASSISTANT = "ASSISTANT" + TOOL = "TOOL" + + +@dataclass +class AWSNovaSonicConversationHistoryMessage: + role: Role # only USER and ASSISTANT + text: str + + +@dataclass +class AWSNovaSonicConversationHistory: + instruction: str = None + messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) + + +@dataclass +class AWSNovaSonicLLMContext(OpenAILLMContext): + @staticmethod + def upgrade_to_nova_sonic(obj: OpenAILLMContext) -> "AWSNovaSonicLLMContext": + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): + obj.__class__ = AWSNovaSonicLLMContext + return obj + + def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: + history = AWSNovaSonicConversationHistory() + + # Bail if there are no messages + if not self.messages: + return history + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into "instruction" + if messages[0].get("role") == "system": + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + history.instruction = content + elif isinstance(content, list): + history.instruction = content[0].get("text") + + # Process remaining messages to fill out conversation history. + # Nova Sonic supports "user" and "assistant" messages in history. + for message in messages: + history_message = self.from_standard_message(message) + if history_message: + history.messages.append(history_message) + + return history + + def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + role = message.get("role") + if message.get("role") == "user" or message.get("role") == "assistant": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + logger.error(f"Unhandled message type in from_standard_message: {message}") + + +@dataclass +class AWSNovaSonicMessagesUpdateFrame(DataFrame): + context: AWSNovaSonicLLMContext + + +class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + await super().process_frame(frame, direction) + + # Parent does not push LLMMessagesUpdateFrame + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) + + # Parent also doesn't push the LLMSetToolsFrame + # TODO: this + # if isinstance(frame, LLMSetToolsFrame): + # await self.push_frame(frame, direction) + + +class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): + pass + + +@dataclass +class AWSNovaSonicContextAggregatorPair: + _user: AWSNovaSonicUserContextAggregator + _assistant: AWSNovaSonicAssistantContextAggregator + + def user(self) -> AWSNovaSonicUserContextAggregator: + return self._user + + def assistant(self) -> AWSNovaSonicAssistantContextAggregator: + return self._assistant From 2b7e1cb5b1fdae4a7b1ecf27ade3d8bafd0fefd8 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 29 Apr 2025 16:38:02 -0400 Subject: [PATCH 21/55] [WIP] AWS Nova Sonic service - add tool calling --- examples/foundational/39-aws-nova-sonic.py | 50 +++++- .../services/aws_nova_sonic_adapter.py | 40 +++++ src/pipecat/services/aws_nova_sonic/aws.py | 149 +++++++++++++++++- .../services/aws_nova_sonic/context.py | 25 ++- src/pipecat/services/aws_nova_sonic/frames.py | 14 ++ 5 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 src/pipecat/adapters/services/aws_nova_sonic_adapter.py create mode 100644 src/pipecat/services/aws_nova_sonic/frames.py diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index c44f85a48..c9bef1fed 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -5,14 +5,16 @@ # import os +from datetime import datetime from dotenv import load_dotenv from loguru import logger # import logging +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMMessagesAppendFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -31,6 +33,39 @@ load_dotenv(override=True) # ) +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + temperature = 75 if args["format"] == "fahrenheit" else 24 + await result_callback( + { + "conditions": "nice", + "temperature": temperature, + "format": args["format"], + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + } + ) + + +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) + + async def run_bot(webrtc_connection: SmallWebRTCConnection): logger.info(f"Starting bot") @@ -62,20 +97,27 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), region=os.getenv("AWS_REGION"), voice_id="tiffany", # matthew, tiffany, amy - # instruction=system_instruction # could pass instruction here rather than context, below + # instruction=system_instruction # you could pass instruction here rather than in context ) + # Register function for function calls + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) + # Set up context and context management. # AWSNovaSonicService will adapt OpenAI LLM context objects with standard message format to # what's expected by Nova Sonic. + # TODO: since we can't trigger a response upon joining, this isn't particularly useful context = OpenAILLMContext( messages=[ {"role": "system", "content": f"{system_instruction}"}, { "role": "user", - "content": "Tell me hello! Don't wait for me to say anything else first!", + "content": "Say hello!", }, - ] + ], + tools=tools, ) context_aggregator = llm.create_context_aggregator(context) diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py new file mode 100644 index 000000000..b96980046 --- /dev/null +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +import json +from typing import Any, Dict, List, Union + +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema + + +class AWSNovaSonicLLMAdapter(BaseLLMAdapter): + @staticmethod + def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]: + return { + "toolSpec": { + "name": function.name, + "description": function.description, + "inputSchema": { + "json": json.dumps( + { + "type": "object", + "properties": function.properties, + "required": function.required, + } + ) + }, + } + } + + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]: + """Converts function schemas to Openai Realtime function-calling format. + + :return: Openai Realtime formatted function call definition. + """ + + functions_schema = tools_schema.standard_tools + return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema] diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index cc07e5463..8b6dab3ed 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -1,9 +1,15 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import base64 import json import uuid from dataclasses import dataclass from enum import Enum -from typing import Any +from typing import Any, List from aws_sdk_bedrock_runtime.client import ( BedrockRuntimeClient, @@ -22,6 +28,7 @@ from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolv from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream +from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -58,10 +65,15 @@ from pipecat.services.aws_nova_sonic.context import ( AWSNovaSonicUserContextAggregator, Role, ) +from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame from pipecat.services.llm_service import LLMService from pipecat.utils.time import time_now_iso8601 +class AWSNovaSonicUnhandledFunctionException(Exception): + pass + + class ContentType(Enum): AUDIO = "AUDIO" TEXT = "TEXT" @@ -91,6 +103,9 @@ class CurrentContent: class AWSNovaSonicLLMService(LLMService): + # Override the default adapter to use the AWSNovaSonicLLMAdapter one + adapter_class = AWSNovaSonicLLMAdapter + def __init__( self, *, @@ -162,6 +177,8 @@ class AWSNovaSonicLLMService(LLMService): await self._send_user_audio_event(frame) elif isinstance(frame, BotStoppedSpeakingFrame): await self._handle_bot_stopped_speaking() + elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): + await self._handle_function_call_result(frame) # TODO: do we need to do anything for the below four frame types? elif isinstance(frame, StartInterruptionFrame): # print("[pk] StartInterruptionFrame") @@ -206,6 +223,10 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_is_responding = False await self._report_assistant_response_ended() + async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): + result = frame.result_frame + await self._send_tool_result(tool_call_id=result.tool_call_id, result=result.result) + # # LLM communication: lifecycle # @@ -228,8 +249,8 @@ class AWSNovaSonicLLMService(LLMService): InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model) ) - # Send session start events - await self._send_session_start_events() + # Send session start event + await self._send_session_start_event() # Finish connecting self._ready_to_send_context = True @@ -247,6 +268,10 @@ class AWSNovaSonicLLMService(LLMService): # Read context history = self._context.get_messages_for_initializing_history() + # Send prompt start event, specifying tools + tools = self._context.tools + await self._send_prompt_start_event(tools) + # Send system instruction # Instruction from context takes priority instruction = history.instruction if history.instruction else self._instruction @@ -318,7 +343,7 @@ class AWSNovaSonicLLMService(LLMService): # # TODO: make params configurable? - async def _send_session_start_events(self): + async def _send_session_start_event(self): session_start = """ { "event": { @@ -334,6 +359,20 @@ class AWSNovaSonicLLMService(LLMService): """ await self._send_client_event(session_start) + async def _send_prompt_start_event(self, tools: List[Any]): + tools_config = ( + f""", + "toolUseOutputConfiguration": {{ + "mediaType": "application/json" + }}, + "toolConfiguration": {{ + "tools": {json.dumps(tools)} + }} + """ + if tools + else "" + ) + prompt_start = f''' {{ "event": {{ @@ -350,7 +389,7 @@ class AWSNovaSonicLLMService(LLMService): "voiceId": "{self._voice_id}", "encoding": "base64", "audioType": "SPEECH" - }} + }}{tools_config} }} }} }} @@ -382,6 +421,9 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(audio_content_start) async def _send_text_event(self, text: str, role: Role): + if not self._stream: + return + content_name = str(uuid.uuid4()) text_content_start = f''' @@ -469,6 +511,61 @@ class AWSNovaSonicLLMService(LLMService): """ await self._send_client_event(session_end) + async def _send_tool_result(self, tool_call_id, result): + if not self._stream: + return + + # print(f"[pk] sending tool result. tool call ID: {tool_call_id}, result: {result}") + + content_name = str(uuid.uuid4()) + + result_content_start = f''' + {{ + "event": {{ + "contentStart": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}", + "interactive": false, + "type": "TOOL", + "role": "TOOL", + "toolResultInputConfiguration": {{ + "toolUseId": "{tool_call_id}", + "type": "TEXT", + "textInputConfiguration": {{ + "mediaType": "text/plain" + }} + }} + }} + }} + }} + ''' + await self._send_client_event(result_content_start) + + result_content = json.dumps( + { + "event": { + "toolResult": { + "promptName": self._prompt_name, + "contentName": content_name, + "content": json.dumps(result) if isinstance(result, dict) else result, + } + } + } + ) + await self._send_client_event(result_content) + + result_content_end = f""" + {{ + "event": {{ + "contentEnd": {{ + "promptName": "{self._prompt_name}", + "contentName": "{content_name}" + }} + }} + }} + """ + await self._send_client_event(result_content_end) + async def _send_client_event(self, event_json: str): event = InvokeModelWithBidirectionalStreamInputChunk( value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) @@ -515,6 +612,9 @@ class AWSNovaSonicLLMService(LLMService): elif "audioOutput" in event_json: # Handle audio output content await self._handle_audio_output_event(event_json) + elif "toolUse" in event_json: + # Handle tool use + await self._handle_tool_use_event(event_json) elif "contentEnd" in event_json: # Handle a piece of content ending await self._handle_content_end_event(event_json) @@ -593,6 +693,42 @@ class AWSNovaSonicLLMService(LLMService): ) await self.push_frame(frame) + async def _handle_tool_use_event(self, event_json): + # This should never happen + if not self._content_being_received: + return + + # Get tool use details + tool_use = event_json["toolUse"] + function_name = tool_use["toolName"] + tool_call_id = tool_use["toolUseId"] + arguments = json.loads(tool_use["content"]) + + # print( + # f"[pk] tool use - function_name: {function_name}, tool_call_id: {tool_call_id}, arguments: {arguments}" + # ) + + # Call tool function + if self.has_function(function_name): + if function_name in self._functions.keys(): + await self.call_function( + context=self._context, + tool_call_id=tool_call_id, + function_name=function_name, + arguments=arguments, + ) + elif None in self._functions.keys(): + await self.call_function( + context=self._context, + tool_call_id=tool_call_id, + function_name=function_name, + arguments=arguments, + ) + else: + raise AWSNovaSonicUnhandledFunctionException( + f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function." + ) + async def _handle_content_end_event(self, event_json): # This should never happen if not self._content_being_received: @@ -671,6 +807,9 @@ class AWSNovaSonicLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> AWSNovaSonicContextAggregatorPair: + context.set_llm_adapter(self.get_llm_adapter()) + user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) + return AWSNovaSonicContextAggregatorPair(user, assistant) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 331ecc13e..820254cfb 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -1,12 +1,25 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import copy from dataclasses import dataclass, field from enum import Enum from loguru import logger -from pipecat.frames.frames import DataFrame, Frame, LLMMessagesUpdateFrame, LLMSetToolsFrame +from pipecat.frames.frames import ( + DataFrame, + Frame, + FunctionCallResultFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, +) from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, @@ -106,7 +119,15 @@ class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): - pass + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, + # so we didn't have a chance to add the result to the openai realtime api context. Let's push a + # special frame to do that. + await self.push_frame( + AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) @dataclass diff --git a/src/pipecat/services/aws_nova_sonic/frames.py b/src/pipecat/services/aws_nova_sonic/frames.py new file mode 100644 index 000000000..94d410f22 --- /dev/null +++ b/src/pipecat/services/aws_nova_sonic/frames.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from dataclasses import dataclass + +from pipecat.frames.frames import DataFrame, FunctionCallResultFrame + + +@dataclass +class AWSNovaSonicFunctionCallResultFrame(DataFrame): + result_frame: FunctionCallResultFrame From da5c4953d5c793d25d7ab1d523928d48550b27b2 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 10:51:06 -0400 Subject: [PATCH 22/55] [WIP] AWS Nova Sonic service - allow passing in tools into initializer --- examples/foundational/39-aws-nova-sonic.py | 5 ++++- src/pipecat/services/aws_nova_sonic/aws.py | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index c9bef1fed..f08cfad04 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -97,7 +97,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), region=os.getenv("AWS_REGION"), voice_id="tiffany", # matthew, tiffany, amy - # instruction=system_instruction # you could pass instruction here rather than in context + # you could choose to pass instruction here rather than via context + # instruction=system_instruction + # you could choose to pass tools here rather than via context + # tools=tools ) # Register function for function calls diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 8b6dab3ed..586f759c6 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -9,7 +9,7 @@ import json import uuid from dataclasses import dataclass from enum import Enum -from typing import Any, List +from typing import Any, List, Optional from aws_sdk_bedrock_runtime.client import ( BedrockRuntimeClient, @@ -28,6 +28,7 @@ from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolv from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -115,7 +116,8 @@ class AWSNovaSonicLLMService(LLMService): region: str, model: str = "amazon.nova-sonic-v1:0", voice_id: str = "matthew", # matthew, tiffany, amy - instruction: str = None, + instruction: Optional[str] = None, + tools: Optional[ToolsSchema] = None, **kwargs, ): super().__init__(**kwargs) @@ -126,6 +128,7 @@ class AWSNovaSonicLLMService(LLMService): self._client: BedrockRuntimeClient = None self._voice_id = voice_id self._instruction = instruction + self._tools = tools self._context: AWSNovaSonicLLMContext = None self._stream: DuplexEventStream[ InvokeModelWithBidirectionalStreamInput, @@ -269,11 +272,16 @@ class AWSNovaSonicLLMService(LLMService): history = self._context.get_messages_for_initializing_history() # Send prompt start event, specifying tools - tools = self._context.tools + # Tools from context take priority over tools from __init__() + tools = ( + self._context.tools + if self._context.tools + else self.get_llm_adapter().from_standard_tools(self._tools) + ) await self._send_prompt_start_event(tools) # Send system instruction - # Instruction from context takes priority + # Instruction from context takes priority over instruction from __init__() instruction = history.instruction if history.instruction else self._instruction if instruction: await self._send_text_event(text=instruction, role=Role.SYSTEM) From 394648f1c9a7a6e1cb6d7865fae31498696185a2 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 11:04:47 -0400 Subject: [PATCH 23/55] [WIP] AWS Nova Sonic service - fix user utterances not making it into the context --- src/pipecat/services/aws_nova_sonic/aws.py | 9 ++++++--- src/pipecat/services/aws_nova_sonic/context.py | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 586f759c6..1b2937f83 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -199,9 +199,8 @@ class AWSNovaSonicLLMService(LLMService): await self.push_frame(frame, direction) async def _handle_context(self, context: OpenAILLMContext): - # TODO: if context has changed, reconnect - # TODO: remove - print(f"[pk] _handle_context: {context.get_messages_for_initializing_history()}") + # TODO: reset connection if needed (if entirely new context object provided, for instance) + print(f"[pk] receive updated context: {context.get_messages_for_initializing_history()}") if not self._context: # We got our initial context - try to finish connecting self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic(context) @@ -800,6 +799,10 @@ class AWSNovaSonicLLMService(LLMService): async def _report_user_transcription_text_added(self, text): print(f"[pk] transcription: {text}") + # Manually add new user transcription text to context. + # We can't rely on the user context aggregator to do this since it's upstream from the LLM. + self._context.add_user_transcription_text_as_message(text) + # Report that some new user transcription text is available. await self.push_frame( TranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) ) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 820254cfb..92cd313cb 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -96,6 +96,13 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) logger.error(f"Unhandled message type in from_standard_message: {message}") + def add_user_transcription_text_as_message(self, text): + message = { + "role": "user", + "content": [{"type": "text", "text": text}], + } + self.add_message(message) + @dataclass class AWSNovaSonicMessagesUpdateFrame(DataFrame): From 3960c604a4ab53a45269855cd976605c262c6261 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 11:20:23 -0400 Subject: [PATCH 24/55] [WIP] AWS Nova Sonic service - fix empty assistant conversation history item in the context after tool use --- src/pipecat/services/aws_nova_sonic/context.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 92cd313cb..206c1fd2b 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -93,7 +93,13 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): logger.error( f"Unhandled content type in context message: {c.get('type')} - {message}" ) - return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # There won't be content if this is an assistant tool call entry. + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # history + if content: + return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) + # We're ignoring messages with role "tool" since they can't be loaded into AWS Nova Sonic + # conversation history logger.error(f"Unhandled message type in from_standard_message: {message}") def add_user_transcription_text_as_message(self, text): From 5e0803479ea04fbb40f7f80d398165f6e2b50380 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 14:53:22 -0400 Subject: [PATCH 25/55] [WIP] AWS Nova Sonic service - add send_transcription_frames option --- src/pipecat/services/aws_nova_sonic/aws.py | 10 +++++++--- src/pipecat/services/aws_nova_sonic/context.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 1b2937f83..3c6de7ad7 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -118,6 +118,7 @@ class AWSNovaSonicLLMService(LLMService): voice_id: str = "matthew", # matthew, tiffany, amy instruction: Optional[str] = None, tools: Optional[ToolsSchema] = None, + send_transcription_frames: bool = True, **kwargs, ): super().__init__(**kwargs) @@ -129,6 +130,7 @@ class AWSNovaSonicLLMService(LLMService): self._voice_id = voice_id self._instruction = instruction self._tools = tools + self._send_transcription_frames = send_transcription_frames self._context: AWSNovaSonicLLMContext = None self._stream: DuplexEventStream[ InvokeModelWithBidirectionalStreamInput, @@ -802,10 +804,12 @@ class AWSNovaSonicLLMService(LLMService): # Manually add new user transcription text to context. # We can't rely on the user context aggregator to do this since it's upstream from the LLM. self._context.add_user_transcription_text_as_message(text) + # Report that some new user transcription text is available. - await self.push_frame( - TranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) - ) + if self._send_transcription_frames: + await self.push_frame( + TranscriptionFrame(text=text, user_id="", timestamp=time_now_iso8601()) + ) # # Context diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 206c1fd2b..e4662ee57 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -94,11 +94,11 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): f"Unhandled content type in context message: {c.get('type')} - {message}" ) # There won't be content if this is an assistant tool call entry. - # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation + # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation # history if content: return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) - # We're ignoring messages with role "tool" since they can't be loaded into AWS Nova Sonic + # We're ignoring messages with role "tool" since they can't be loaded into AWS Nova Sonic # conversation history logger.error(f"Unhandled message type in from_standard_message: {message}") From 2154db07f085e92aab4ef99b45a438dc316088cd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 15:10:10 -0400 Subject: [PATCH 26/55] [WIP] AWS Nova Sonic service - remove unnecessary error log --- src/pipecat/services/aws_nova_sonic/context.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index e4662ee57..4b41b53b3 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -98,9 +98,8 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # history if content: return AWSNovaSonicConversationHistoryMessage(role=Role[role.upper()], text=content) - # We're ignoring messages with role "tool" since they can't be loaded into AWS Nova Sonic - # conversation history - logger.error(f"Unhandled message type in from_standard_message: {message}") + # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova + # Sonic conversation history def add_user_transcription_text_as_message(self, text): message = { From 6938152db67947e09da37b7d2e3d649fa0b5a939 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 15:15:49 -0400 Subject: [PATCH 27/55] [WIP] AWS Nova Sonic service - fix comment --- src/pipecat/services/aws_nova_sonic/context.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 4b41b53b3..5d9bafec5 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -73,6 +73,7 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # Process remaining messages to fill out conversation history. # Nova Sonic supports "user" and "assistant" messages in history. + print(f"[pk] standard messages: {messages}") for message in messages: history_message = self.from_standard_message(message) if history_message: @@ -134,9 +135,9 @@ class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): async def handle_function_call_result(self, frame: FunctionCallResultFrame): await super().handle_function_call_result(frame) - # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, - # so we didn't have a chance to add the result to the openai realtime api context. Let's push a - # special frame to do that. + # The standard function callback code path pushes the FunctionCallResultFrame from the LLM + # itself, so we didn't have a chance to add the result to the AWS Nova Sonic server-side + # context. Let's push a special frame to do that. await self.push_frame( AWSNovaSonicFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM ) From d6ef3d64ace855238e0ee60c5e5d92247b8a7448 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 21:40:40 -0400 Subject: [PATCH 28/55] [WIP] AWS Nova Sonic service - fix context problems of double-counting LLM text, and mis-categorizing user text as LLM text --- .../services/aws_nova_sonic/context.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 5d9bafec5..4e2a4fcc1 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -16,6 +16,8 @@ from pipecat.frames.frames import ( FunctionCallResultFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, + LLMTextFrame, + TranscriptionFrame, ) from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection @@ -132,6 +134,23 @@ class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): + # AWS Nova Sonic is a speech-to-speech model. + # It behaves like a combined STT + LLM + TTS service, emitting all of: + # - TranscriptionFrame (for user text) + # - LLMTextFrame (for assistant text) + # - TTSTextFrame (for assistant text) + # In a "standard" pipeline (with separate STT + LLM + TTS services): + # - The TranscriptionFrame is swallowed by the LLMUserContextAggregator + # - The LLMTextFrame is swallowed by the TTS service + # Meaning the LLMAssistantContextAggregator only receives the TTSTextFrames. It actually + # implicitly assumes it will receive only *non-duplicate* *assistant-related* text frames, and + # will misbehave otherwise (double-counting assistant text, or mis-categorizing user text as + # assistant text). + # So, let's override process_frame here to ignore TranscriptionFrames and LLMTextFrames. + async def process_frame(self, frame: Frame, direction: FrameDirection): + if not isinstance(frame, (LLMTextFrame, TranscriptionFrame)): + await super().process_frame(frame, direction) + async def handle_function_call_result(self, frame: FunctionCallResultFrame): await super().handle_function_call_result(frame) From c47703995406be98ff717dc9bb4f9943b1bc1faa Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 30 Apr 2025 22:29:36 -0400 Subject: [PATCH 29/55] [WIP] AWS Nova Sonic service - just for safety, add a short delay after BotStoppedSpeaking before sending LLMFullResponseEndFrame + TTSStoppedFrame, to give a bit of leeway for the LLM to deliver the "FINAL" text block describing what was said --- src/pipecat/services/aws_nova_sonic/aws.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 3c6de7ad7..1ef171750 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import asyncio import base64 import json import uuid @@ -211,7 +212,8 @@ class AWSNovaSonicLLMService(LLMService): async def _handle_bot_stopped_speaking(self): if self._assistant_is_responding: - # Consider the assistant finished with their response. + # Consider the assistant finished with their response (after a short delay, to allow for + # any FINAL text block to come in). # # TODO: ideally we could base this solely on the LLM output events, but I couldn't # figure out a reliable way to determine when we've gotten our last FINAL text block @@ -224,6 +226,7 @@ class AWSNovaSonicLLMService(LLMService): # FINAL text blocks to know how many or which FINAL blocks to expect, but user # interruptions throw a wrench in these schemes: depending on the exact timing of the # interruption, we should or shouldn't expect some FINAL blocks. + await asyncio.sleep(0.25) self._assistant_is_responding = False await self._report_assistant_response_ended() From 38c9fa681a3a0678e58805bcca5ff833f8a1a9e3 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 1 May 2025 17:50:29 -0400 Subject: [PATCH 30/55] [WIP] AWS Nova Sonic service - Protect against back-to-back BotStoppedSpeaking calls, which I've observed --- src/pipecat/services/aws_nova_sonic/aws.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 1ef171750..e8da485a8 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -145,6 +145,8 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_is_responding = False self._context_available = False self._ready_to_send_context = False + self._handling_bot_stopped_speaking = False + # # standard AIService frame handling @@ -211,6 +213,11 @@ class AWSNovaSonicLLMService(LLMService): await self._finish_connecting_if_context_available() async def _handle_bot_stopped_speaking(self): + # Protect against back-to-back BotStoppedSpeaking calls, which I've observed + if self._handling_bot_stopped_speaking: + return + self._handling_bot_stopped_speaking = True + if self._assistant_is_responding: # Consider the assistant finished with their response (after a short delay, to allow for # any FINAL text block to come in). @@ -229,6 +236,9 @@ class AWSNovaSonicLLMService(LLMService): await asyncio.sleep(0.25) self._assistant_is_responding = False await self._report_assistant_response_ended() + self._handling_bot_stopped_speaking = False + + self._handling_bot_stopped_speaking = False async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): result = frame.result_frame From 4ffdc3b77ceed4168f747384ac71ae1acd0ae941 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 1 May 2025 21:54:36 -0400 Subject: [PATCH 31/55] [WIP] AWS Nova Sonic service - do hacky direct manipulation of the context for now, since I can't seem to get assistant context aggregation working properly with frames, grr --- src/pipecat/services/aws_nova_sonic/aws.py | 19 +++++-- .../services/aws_nova_sonic/context.py | 54 +++++++++++++------ 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index e8da485a8..35f312a0e 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -147,7 +147,6 @@ class AWSNovaSonicLLMService(LLMService): self._ready_to_send_context = False self._handling_bot_stopped_speaking = False - # # standard AIService frame handling # @@ -760,8 +759,10 @@ class AWSNovaSonicLLMService(LLMService): content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] # print(f"[pk] content end: {content}.\n stop_reason: {stop_reason}") - # if content.role == Role.ASSISTANT: - # print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") + if content.role == Role.ASSISTANT: + # print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") + if content.text_stage == TextStage.FINAL: + print(f"[pk] assistant FINAL text: {content.text_content}") # Bookkeeping: clear current content being received self._content_being_received = None @@ -803,6 +804,18 @@ class AWSNovaSonicLLMService(LLMService): print(f"[pk] TTS text: {text}") await self.push_frame(TTSTextFrame(text)) + # TODO: this is a (hopefully temporary) HACK. Here we directly manipulate the context rather + # than relying on the frames pushed to the assistant context aggregator. The pattern of + # receiving full-sentence text after the assistant has spoken does not easily fit with the + # Pipecat expectation of chunks of text streaming in while the assistant is speaking. + # Interruption handling was especially challenging. Rather than spend days trying to fit a + # square peg in a round hole, I decided on this hack for the time being. We can most cleanly + # abandon this hack if/when AWS Nova Sonic implements streaming smaller text chunks + # interspersed with audio. Note that when we move away from this hack, we need to make sure + # that on an interruption we avoid sending LLMFullResponseEndFrame, which gets the + # LLMAssistantContextAggregator into a bad state. + self._context.add_assistant_text_as_message(text) + async def _report_assistant_response_ended(self): # Report that the assistant has finished their response. print("[pk] LLM full response ended") diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 4e2a4fcc1..647e40ae6 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -11,13 +11,19 @@ from enum import Enum from loguru import logger from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, DataFrame, Frame, FunctionCallResultFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesAppendFrame, LLMMessagesUpdateFrame, + LLMSetToolChoiceFrame, LLMSetToolsFrame, - LLMTextFrame, - TranscriptionFrame, + StartInterruptionFrame, + TextFrame, + UserImageRawFrame, ) from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection @@ -110,6 +116,15 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): "content": [{"type": "text", "text": text}], } self.add_message(message) + # print(f"[pk] context updated (user): {self.get_messages_for_logging()}") + + def add_assistant_text_as_message(self, text): + message = { + "role": "assistant", + "content": [{"type": "text", "text": text}], + } + self.add_message(message) + # print(f"[pk] context updated (assistant): {self.get_messages_for_logging()}") @dataclass @@ -134,21 +149,28 @@ class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): - # AWS Nova Sonic is a speech-to-speech model. - # It behaves like a combined STT + LLM + TTS service, emitting all of: - # - TranscriptionFrame (for user text) - # - LLMTextFrame (for assistant text) - # - TTSTextFrame (for assistant text) - # In a "standard" pipeline (with separate STT + LLM + TTS services): - # - The TranscriptionFrame is swallowed by the LLMUserContextAggregator - # - The LLMTextFrame is swallowed by the TTS service - # Meaning the LLMAssistantContextAggregator only receives the TTSTextFrames. It actually - # implicitly assumes it will receive only *non-duplicate* *assistant-related* text frames, and - # will misbehave otherwise (double-counting assistant text, or mis-categorizing user text as - # assistant text). - # So, let's override process_frame here to ignore TranscriptionFrames and LLMTextFrames. async def process_frame(self, frame: Frame, direction: FrameDirection): - if not isinstance(frame, (LLMTextFrame, TranscriptionFrame)): + # HACK: For now, disable the context aggregator by making it just pass through all frames + # that the parent handles (except the function call stuff, which we still need). + # For an explanation of this hack, see + # AWSNovaSonicLLMService._report_assistant_response_text_added. + if isinstance( + frame, + ( + StartInterruptionFrame, + LLMFullResponseStartFrame, + LLMFullResponseEndFrame, + TextFrame, + LLMMessagesAppendFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMSetToolChoiceFrame, + UserImageRawFrame, + BotStoppedSpeakingFrame, + ), + ): + await self.push_frame(frame, direction) + else: await super().process_frame(frame, direction) async def handle_function_call_result(self, frame: FunctionCallResultFrame): From 3784bdbd27ef06e857537b9e08f74a74f865ffeb Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 2 May 2025 10:42:52 -0400 Subject: [PATCH 32/55] [WIP] AWS Nova Sonic service - in our hacky direct manipulation of the context, aggregate assistant text rather than recording every chunk as a separate message --- src/pipecat/services/aws_nova_sonic/aws.py | 13 ++++++------ .../services/aws_nova_sonic/context.py | 20 +++++++++++++++---- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 35f312a0e..e7b1fd8e6 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -759,10 +759,8 @@ class AWSNovaSonicLLMService(LLMService): content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] # print(f"[pk] content end: {content}.\n stop_reason: {stop_reason}") - if content.role == Role.ASSISTANT: - # print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") - if content.text_stage == TextStage.FINAL: - print(f"[pk] assistant FINAL text: {content.text_content}") + # if content.role == Role.ASSISTANT: + # print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") # Bookkeeping: clear current content being received self._content_being_received = None @@ -814,7 +812,7 @@ class AWSNovaSonicLLMService(LLMService): # interspersed with audio. Note that when we move away from this hack, we need to make sure # that on an interruption we avoid sending LLMFullResponseEndFrame, which gets the # LLMAssistantContextAggregator into a bad state. - self._context.add_assistant_text_as_message(text) + self._context.buffer_assistant_text(text) async def _report_assistant_response_ended(self): # Report that the assistant has finished their response. @@ -825,11 +823,14 @@ class AWSNovaSonicLLMService(LLMService): print("[pk] TTS stopped") await self.push_frame(TTSStoppedFrame()) + # For an explanation of this hack, see _report_assistant_response_text_added. + self._context.flush_aggregated_assistant_text() + async def _report_user_transcription_text_added(self, text): print(f"[pk] transcription: {text}") # Manually add new user transcription text to context. # We can't rely on the user context aggregator to do this since it's upstream from the LLM. - self._context.add_user_transcription_text_as_message(text) + self._context.add_user_transcription_text(text) # Report that some new user transcription text is available. if self._send_transcription_frames: diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 647e40ae6..3fac65a72 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -53,12 +53,19 @@ class AWSNovaSonicConversationHistory: messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) -@dataclass class AWSNovaSonicLLMContext(OpenAILLMContext): + def __init__(self, messages=None, tools=None, **kwargs): + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self): + self._assistant_text = "" + @staticmethod def upgrade_to_nova_sonic(obj: OpenAILLMContext) -> "AWSNovaSonicLLMContext": if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): obj.__class__ = AWSNovaSonicLLMContext + obj.__setup_local() return obj def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: @@ -110,7 +117,7 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # NOTE: we're ignoring messages with role "tool" since they can't be loaded into AWS Nova # Sonic conversation history - def add_user_transcription_text_as_message(self, text): + def add_user_transcription_text(self, text): message = { "role": "user", "content": [{"type": "text", "text": text}], @@ -118,11 +125,16 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): self.add_message(message) # print(f"[pk] context updated (user): {self.get_messages_for_logging()}") - def add_assistant_text_as_message(self, text): + def buffer_assistant_text(self, text): + self._assistant_text += text # TODO: determine if we need to add space or something + # print(f"[pk] assistant text buffered: {self._assistant_text}") + + def flush_aggregated_assistant_text(self): message = { "role": "assistant", - "content": [{"type": "text", "text": text}], + "content": [{"type": "text", "text": self._assistant_text}], } + self._assistant_text = "" self.add_message(message) # print(f"[pk] context updated (assistant): {self.get_messages_for_logging()}") From cc1f4ba81c24ff0928eb336507ee20b800d5946f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Fri, 2 May 2025 11:31:56 -0400 Subject: [PATCH 33/55] [WIP] AWS Nova Sonic service - add a hacky way of programmatically triggering an assistant response --- examples/foundational/39-aws-nova-sonic.py | 18 +++- pyproject.toml | 1 + src/pipecat/services/aws_nova_sonic/aws.py | 92 ++++++++++++++++-- src/pipecat/services/aws_nova_sonic/ready.wav | Bin 0 -> 23484 bytes 4 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 src/pipecat/services/aws_nova_sonic/ready.wav diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index f08cfad04..07670f75a 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -83,12 +83,16 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): ), ) - # Specify initial system instruction + # Specify initial system instruction. + # HACK: note that, for now, we need to inject a special bit of text into this instruction to + # allow the first assistant response to be programmatically triggered (which happens in the + # on_client_connected handler, below) # TODO: looks like Nova Sonic can't handle new lines? system_instruction = ( - "You are a friendly assistant. The user and you will engage in a spoken dialog " - "exchanging the transcripts of a natural real-time conversation. Keep your responses short, " - "generally two or three sentences for chatty scenarios." + "You are a friendly assistant. The user and you will engage in a spoken dialog exchanging " + "the transcripts of a natural real-time conversation. Keep your responses short, generally " + "two or three sentences for chatty scenarios. " + f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}" ) # Create the AWS Nova Sonic LLM service @@ -117,7 +121,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): {"role": "system", "content": f"{system_instruction}"}, { "role": "user", - "content": "Say hello!", + "content": "Tell me a fun fact!", }, ], tools=tools, @@ -151,6 +155,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): logger.info(f"Client connected") # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) + # HACK: for now, we need this special way of triggering the first assistant response in AWS + # Nova Sonic. Note that this trigger requires a special corresponding bit of text in the + # system instruction. In the future, simply queueing the context frame should be sufficient. + await llm.trigger_assistant_response() # Handle client disconnection events @transport.event_handler("on_client_disconnected") diff --git a/pyproject.toml b/pyproject.toml index d6d05c00c..7ce167d77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ where = ["src"] [tool.setuptools.package-data] "pipecat" = ["py.typed"] +"pipecat.services.aws_nova_sonic" = ["src/pipecat/services/aws_nova_sonic/ready.wav"] [tool.pytest.ini_options] addopts = "--verbose" diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index e7b1fd8e6..5b69810f3 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -8,8 +8,10 @@ import asyncio import base64 import json import uuid +import wave from dataclasses import dataclass from enum import Enum +from importlib.resources import files from typing import Any, List, Optional from aws_sdk_bedrock_runtime.client import ( @@ -146,6 +148,8 @@ class AWSNovaSonicLLMService(LLMService): self._context_available = False self._ready_to_send_context = False self._handling_bot_stopped_speaking = False + self._triggering_assistant_response = False + self._assistant_response_trigger_audio: bytes = None # Not cleared on _disconnect() # # standard AIService frame handling @@ -180,8 +184,7 @@ class AWSNovaSonicLLMService(LLMService): if isinstance(frame, OpenAILLMContextFrame): await self._handle_context(frame.context) elif isinstance(frame, InputAudioRawFrame): - # TODO: check if _audio_input_paused? what causes that? - await self._send_user_audio_event(frame) + await self._handle_input_audio_frame(frame) elif isinstance(frame, BotStoppedSpeakingFrame): await self._handle_bot_stopped_speaking() elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): @@ -211,6 +214,15 @@ class AWSNovaSonicLLMService(LLMService): self._context_available = True await self._finish_connecting_if_context_available() + async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): + # Wait until we're done sending the assistant response trigger audio before sending audio + # from the user's mic + if self._triggering_assistant_response: + return + + # TODO: check if _audio_input_paused? what causes that? + await self._send_user_audio_event(frame.audio) + async def _handle_bot_stopped_speaking(self): # Protect against back-to-back BotStoppedSpeaking calls, which I've observed if self._handling_bot_stopped_speaking: @@ -316,6 +328,14 @@ class AWSNovaSonicLLMService(LLMService): # Start receiving events self._receive_task = self.create_task(self._receive_task_handler()) + # If we need to, send assistant response trigger + if self._triggering_assistant_response: + # If the trigger was the first audio chunk sent on this connection it'd be ignored (I'm + # guessing the LLM can't quite "hear" the first little bit of audio sent). So send a bit + # of leading blank audio first. + await self._send_assistant_response_trigger(lead_with_blank_audio=True) + self._triggering_assistant_response = False + async def _disconnect(self): try: # Clean up receive task @@ -340,6 +360,8 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_is_responding = False self._context_available = False self._ready_to_send_context = False + self._handling_bot_stopped_speaking = False + self._triggering_assistant_response = False except Exception as e: logger.error(f"{self} error disconnecting: {e}") @@ -490,11 +512,11 @@ class AWSNovaSonicLLMService(LLMService): ''' await self._send_client_event(text_content_end) - async def _send_user_audio_event(self, frame: InputAudioRawFrame): + async def _send_user_audio_event(self, audio: bytes): if not self._stream: return - blob = base64.b64encode(frame.audio) + blob = base64.b64encode(audio) audio_event = f''' {{ "event": {{ @@ -639,7 +661,7 @@ class AWSNovaSonicLLMService(LLMService): elif "contentEnd" in event_json: # Handle a piece of content ending await self._handle_content_end_event(event_json) - elif "completionStart" in event_json: + elif "completionEnd" in event_json: # Handle the LLM completion ending await self._handle_completion_end_event(event_json) @@ -839,7 +861,7 @@ class AWSNovaSonicLLMService(LLMService): ) # - # Context + # context # def create_context_aggregator( @@ -855,3 +877,61 @@ class AWSNovaSonicLLMService(LLMService): assistant = AWSNovaSonicAssistantContextAggregator(context=context, params=assistant_params) return AWSNovaSonicContextAggregatorPair(user, assistant) + + # + # assistant response trigger (HACK) + # + + # Class variable + AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION = ( + "Start speaking when you hear the user say 'ready', but don't consider that 'ready' to be " + "a meaningful part of the conversation other than as a trigger for you to start speaking." + ) + + async def trigger_assistant_response(self): + if self._triggering_assistant_response: + return False + + self._triggering_assistant_response = True + + # Read audio bytes, if we don't already have them cached + if not self._assistant_response_trigger_audio: + file_path = files("pipecat.services.aws_nova_sonic").joinpath("ready.wav") + with wave.open(file_path.open("rb"), "rb") as wav_file: + self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) + + # Send the trigger audio, if we're fully connected and set up + # NOTE: maybe there's a better way to determine whether we're done setting up? + if self._receive_task: + await self._send_assistant_response_trigger() + self._triggering_assistant_response = False + + async def _send_assistant_response_trigger(self, lead_with_blank_audio=False): + # TODO: if/when we make bitrate, etc configurable, avoid hard-coding this + chunk_size = 640 # equivalent to what we get from InputAudioRawFrame + chunk_duration = 640 / ( + 16000 * 2 + ) # 640 bytes of 16-bit (2-byte) PCM mono audio at 16kHz corresponds to 0.02 seconds + + # Lead with blank audio, if needed + if lead_with_blank_audio: + blank_audio_duration = 0.5 # much less than this and it doesn't reliably work + blank_audio_chunk = b"\x00" * chunk_size + num_chunks = int(blank_audio_duration / chunk_duration) + for _ in range(num_chunks): + await self._send_user_audio_event(blank_audio_chunk) + await asyncio.sleep(chunk_duration) + + # Send trigger audio + # NOTE: this audio *will* be transcribed and eventually make it into the context. That's OK: + # if we ever need to seed this service again with context it would make sense to include it + # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the + # context as well. + # print(f"[pk] sending trigger audio! {len(self._assistant_response_trigger_audio)}") + audio_chunks = [ + self._assistant_response_trigger_audio[i : i + chunk_size] + for i in range(0, len(self._assistant_response_trigger_audio), chunk_size) + ] + for chunk in audio_chunks: + await self._send_user_audio_event(chunk) + await asyncio.sleep(chunk_duration) diff --git a/src/pipecat/services/aws_nova_sonic/ready.wav b/src/pipecat/services/aws_nova_sonic/ready.wav new file mode 100644 index 0000000000000000000000000000000000000000..ca932afa66d69dcf3626a54f9b171c767eb4d0e9 GIT binary patch literal 23484 zcmeHvXLuCHvgqX5q}^4PKoTMfM9!FGvdIQa&KQgV1IB<2CK?lr$=N2EWWWY&a?TkH z#$a+3LP7||jkB|p`qgOe++7Lzob&GY-oLy3&2)98uCDH$&{MtL+O=tOs2_&)YTmQ; z@G%n;?HGpPkiKaOz;p~FFgw<<-ACO^K(SrtHXpY8s7-e(z#WHtHN0MpYBlRst68H4 zHf+emA!9+y|Nj2h1OMxRf8_xZBVkUdKO7DOKNVKstd`Z8M?g|@9a76hld472q-xXQ z0GEpWf0HyTbX{7Vdg(vO$^UA{f5!WtaQ)x#6!k-(9Dr2+={f_DjwzZK&ruXx*lWQ$ zG)htaV2=Me{O_~|TNOOQ0$-ZBqK!~;P6xa zD`}Vl;UXthckoZED=;gxa`9)q_M%kPZnZ6%jls~n_+U%`4#;S@;Bufgt$zKiwjwZ> z16(?!c~iZ!2JF*jiOQj3H4a^ehSRmGx)D+j^~#}3X!5E~m0Pt>21zZfsQE;dl)N7FpeJ3M75)hAkkh}$l8_pi&LR`V&Ex>X}1RkNFzt!uR zYLQwm4cb}Ef|=m3LJGrdSU98zNRe0!oTISl0w@w7?T7|0l^zPu9ATIlB#e+$Js8-o z+J+E;NnoEJYDG}5YM~Ec57;6hCty6t4JjApp&XQf^3iLQjozY6z^G6TBnR;1LsHMG zw4}Nc8Nq4;*cwt$uNf?|VXpq(7aV0FPsGY&30P&UKH#fkwXw3mVJq-0 ziK+hbKu!11pXd&{hAyM4=sY@&j-rF;Bszi)qf-DMMF-FcIG+OCDd4z@9-^D*Z}bS= zLJ!b$uv7Id34Sob5e9bDg_hUHlCdPHuM*Y@>x9+9x?^py-dH270oDO~5330F)PlSW z9E}UCwt-bLN&_!sLk)kROK|K%zoUy#`$4ef09bbz9Y=ow^&+~BUV=Z~f_HFeH3mHx z3;j|8D~Z*>s=?6+YYp`@#6HAEV7;)x*d%NQ_6;@?n}DsvzQ8`krePDY(Ln74R+TR3 zJsfkQ`{*&$@f{k6TA>;!1UVJAa#Ojc99Di+eo_`I8k;jjzY&;zRNG@S1oC zj<7e_73?_n6ZSdys2Q}wh`GSO$I*H;4fRIvp$e!3G9V+eB0HiGuc&splxS26)r6Y+ zqw#1I8jeN)_m}8Xv=Qw84SaVQ`X&u}mxlgN!un%#ut!)f_96Zhzkw?_Lmb5yyZU-L^`QUR16(KH>2Js58~JH@%U7%EA|U=D<|ZAk}P$R zdP-J#f;>bXDE%P*A*}NE?m==2SCoxny2!u3>&=3O9KTpBjE*>oNw~LWPiT z@jcK_Q_wEumYgVWlun9&3)O_N{w=)6x5wAlcZd6q>%%2-r@YO*-MvG+d%PLmhTJv| zaqE3``K5eyzr%l7*eBkSDk_ywB-R>VKzvJ%rbf{%nYTf3L)We^RtG0k8w z@&?Y3U`#S58_OBr7=AajHE`?}mS&eSx%4dhF2zub$O_~kVk%LE2qOq$E?yo_h0*>L zB_Rn$%@*Ys<(|SR!%!tO2`xZXQ8HSin3NynvhuG|ed!Bnm1L2-$=UJBkVSe_UdSkk}Y?Y*GO5SNwf+>{6F!Fe1CGH*WmrcbJLyeYU7G` zSzM{kRA)CA>)!5e=_%n&;v#&Pe82Kn{fC7u;wWjZJVjZ8oES@VC+AYz=;cg(_7vOI zFwd~taN3Y){J_}M*vMGQxXvK5No*gcEj^1`PfjOB;RCVn&}wC!+)*aw)l$49ix0)m z#B)L^!QuD#%Lz#ML~5j9*kr;-b!Sf+W%I6(XxmoXQQIZkWZPToB5SPmZpe<1i6Lb} zT87jKIb-P<^4bz^S!rHr9cIX38W|>%vvGnHrl?{e2A(;y73%tgY%xV zdR~d_so9O*I4kF5LHbmB7VfG%O5k!kdAY-9F1{S)Dm>-w`i3mk`Y zDrEUH+rOEW@#&j}S?RemouOQuSYFw$tW#zxZRI+0j$9wRNVPO(hl~tc6nJe>%D93;h~qukqf*vCaXz*_9{Z$2s^^U%t9$nL+%KK({HnPo=S}wnkBj?T znof>3^tD!qyjikg!u9xu@hP!#_!HYm>pCkIdOLDjiR8p7WkM4@C3i+9g^VB{_`7@e zxx2c?dEXavu{i1_HJoaJ@AhXnKhC?BGdB0Bvzfmo(bM>|H6si~{uZ$&yj^(L(0$f3 zAzy-r?po%Cth7H09UeN-cGU7Ey9ED2E+y5J>dEslAJKy>OIq;r=u6onrYetdnK{a~ zH}p1^WympR?PB^G*KK8)>3-Ok5hE-Wgqzv3 zQz3>ZTLNMjDLOZ-t_({gHyYv9nOwaZW`BwQc+}yp7GnGV&j^E*@0g!0RYJxahZ5C=Gx=*W+q}un zZszGE*_mUORd$b^vgMfnGHihnG|1NvYDC9f4!}iKBz&IT%t(+Dr zde*q5oV>hhj3k$CT2FaM@Y z!C30|_?HRm;-|(fiTl0;AC(+aHZ~?Nhd5xDw(^bj-=I&?9W=2*E_e7W3i)a4*q8SvpTPCxet&fqt_)IuCOGLu(apv=Xf&r z=C=}mHIE8AW9z{#^nR9h{eJ%4At}$^c=PY^>*NM9%TIS-bRXeH3B6Gq^~Urh{MWd& zvL!24s5H7@FPGUKY2`XlzG1(PYq?aXCa?OpZdxrVXk1ICl|$7r3f zLOM;p&#IRW3R*rlf_@q9=fWjWk|C1vycadQOsp3l?OD5Un_)X=M(xsz1D<5V5{+i49)Ok?e zKnwTXTL@#Rw(VoX$O`RRNjS6MxWtnV#X`IV!V-K36LfuhaW6wojj`D^t z3ab&mBz#$z(OQO{B-6fl_c_--Z!dm;@4Ba|<89W6x4&lp=I$$>B^FcTD3*FdCXvz9 z8d{-CutstY{+)TWq&j8V)tPuzIZ_R(HHHx<*1QA$T@F0WrRHqp6NtgsRjy1{*OZPAqV7+6 zQ6c-XTj8FFZ}8sK&qNpcB{hY7P6+g$*2yv6#I&Sif+C)`y$V_Zudee#~?#yU7pC-HAAgZiF+2CZLBH)B^Bce16JnIulWC94sy<#y70 zKGL(>u_nKjC*M6m*d|=ycX9LGz1(fYc%d$lhRe!Sl*@1ToaHYGccpYsYR)HOW=QF1 zW1JFo(At-{o%bg7^R$uKn|1}Z@dr1zQ{!984sDK8jsLT z$u)GAxqd{gxS6E~ByNfA7_t(b;O6+k(Lm$Z)=}0BV*~7lw}tb9=dkpKiLh-CiwbLM z|1IR8mAAeObwsX-{vbLgvQ^k!>l9NHs4Ta%NR45LG)y$t+s*Gzpj_crY5AE&Y0|67lV zy`98NQZb#fOkpc1H$4gYEghHn71ZL8D`5x2cZ3eGPql^GlfpYjej4kEeH+yv!eeh| zsmWecX7J-Z=RNh|IXBhE`tG=!I$q`U%w6X>Bh4T;GxKN%l}e4LBB{AlfA*~TJ4++; zV#Y!XvQ;3x6P$mz5Awr3rMU9mGfv7og=^s6E%X*z2%Dt}zN4;ip}cgAtWV5xAIk5I zS)$67x>qthysJ4}Zjih6)!@u0jwRv>d86lS#_uTuUPvjvl-93r^_;#xiW>p_&u^VyASTw z)lBy-eQkn$Md&Z~-eGUU?NMKp;G*|hXS1b<=4iCi4V57|yr&$_J;@KxYndPKx#=Gx zb(Xs;wNX7Z7_Uq9X6(js#;Ndz(Vn?NO(H%)rKRrjK(PXU(fhHpqUS^JS^g1c_qOn4 zyPCT&k3YWzpDu?IF6BeVn!HGJbjc4YnM(Bw?LdV4-)4QA6Xn>zpAzng&+|8=m3!7A zrDs}|j3uxCcr!59>HggJL|lb0rVcPe*<|*X`H#qH38xdYOH2*xWB%OGgKkUv3}-Ce zEIZg6@?q~qxK>z24&LN4um{8sbS3sj!(CHy$ZPA((7O>K(RggN5@TYIhR24iViwa3 zU6;*d;~5IS$U7ZfazpZJIgfZG-ypuRza#JDWucDpA?iU?rdG4_4Q|r{<1O|IHW4k8 zNn*cp32Q0;DLRC<{&B83E-Y`5V~gj2Yq6)XceuZ_P}RR9=W13@u|yUnoDLEYuyZhAl#;@pz_P zNQdyfF<~Wl#Qqgl#q!uN+jz^;&3-+Mv&ORz<=fmM_ef7g?*VQ%_qBJv`!Cl_*Gzb~ z$@JFX)5K(SAKsQTl)FkC@?wL@z03!uP39iPSn?0~zON?l_v2C{;jNIydxY)Y)A@ta zXJqvB_)T1-G5V~*D*E%*noX@lFX3xkun>Qzat!pyJ3FGB? z;!AH;=T6s2p#>dh>l0o!{IE67c$%ulKC�+!;GJwq@jJ)|Up4p|`1q`D>$}E=P5s zPSQ8v&3P)(2qjBTq%tUhUTvIgK41zpyrrIzLFQ@e1;1tG)ABc@b zK?oBs!4+wo;1Z1DX3-%ollDjkDPLG2%oT4+D`m4nD>IZSN*9>lcDtqOsa%*X&_%$R(6hXID93lQH-W1=6?WLabJS9t6sU#_* zl@HJhOvbkm2Z^1;4&pgMkw?j)R2*$#4l$M4-7ICOWvF7H4ZpFtfiPN3YfKYNXN(&S zEe)Lv&kb9QeT^Fo?bx|YI1|m>r%zI?Niz|GFNM{`+NgqZPx?*F5f%zvg!hEQ{wDtG zd<}jfzl}f7ALgg?FMI>|r~F0#Sz)61NVwzw#J|ShTuhUeD&x@$bVli_dsBOk4B@3iXySnZjlq$NFTzxJ_f4>^`x`@%lvu&ui}2Gx|AuD7bl3XrD5_jd87QJ zG6z;E&ZD{LCaeVgLViwpsDAWQritOM@sOzrtOsl}PB+ao@ur=o{iXHm^W(Ak=(V^*s4a{W z`=C4YU~`&nT12@LPvfVRbcFA-&Is!h+asY-VnVzVW1qU?cwtt8y=n!PnVzj{JFVLlrd=l|HV-qVin=ld~pM_9Mv|aTzIMQCQ(CT$A(u6dk|HoOhmaxrGJgyXggwT1S?s`ZD%8W zq4k*a!e@LFaezEeCeQV0bR!16HAFjl-1Cm z8AzNUt{Y!iaocOtDZ^FPK|LaxFkV9yrWt-f`APKod;5Fx*Z7NkB`(Ev)79GjgZHL4 z-_yuF#(l>7J-65UwRfp6nZKYg^3P&ZrM_Sj`Y2`&}kT8>*DbuWB#?q^pJ#XMXx|turZ{d1#`}|9Uu0kvM zeWkJTJMv3Iq(*X6X|S@394YL|eQaJH{#B?cOp?|*a)o5nnQ&WkLOziroo90<=O0wk z$m_mx?ilBjobB2B-hP!;&%KWu>)htr>Dlet>qvGE^-Uvkj0+7_s4vKdc)8@Ms&z&6dY-0&?Kg$(ld*bTZZHCDRi z`^~qNf8?v|A1RGcb_=Vy(cDa-rZQhSrX-<-7>-9{AESZzcKi_D0Uv;E#BZT<(p0g9 zd;#WhS^p_+o^Jy`-oFUieV0q|u%1hvcE0ES95K^BpLck=`D%z2<+jp1$u0I*epkMb zPs2KHvT{M1YrN?^;JRtK7TzrMxUIFXop+G3hb+ssvQM+D=*d?(ouOhJeyoos^KHCaOIeM0KI?}{<|m=JwcC@cJfeB z61T(scf23t7x5i^jkq_SCEQqkoiCM}>2-Q*a+|#Uef@-Xq8+X!Ck0uypr^bfw3KlC z6LJySC|4H`NlQ@=REvA$&bCouhwR76&F<6g8{{P8Nz0ekr*wNh(Rqt|iFdU9VZ6v! z%N^ou3Zra=KhM|HciqS4FLRFf-1g1zZ}Kk{(v_?71;Of1!hbZJv`(}z#^r2jX0xeA zXzjRfap@#0-6N*i^Vln76S@p(AV%QLh)^O` z+U8@q7+NNFb1@+cUG_qcE>Dx{2gp80~Ta1+o~%4U3Jx^JkA z(z)Kg2J$HOvN;q#;$H3Oa~yn{X7VzE34dl<7n=5UlL?5zD*`0DVI5o;o@Msy8bZkl6Q zW-b#l-_na+L(V1{d^p+xPeRktDlyL|a*2Ej{~_1kGoPEonY@GDcK1l{9^YZ#SNupH z!JXxP_RSM6NzbJj!a08rakcUpd}*tR&MBXvL0Db<0B$Ef!8c-mp=NkP;x)bw>xmMS z+6n_-4R)Z?7?P9aZ=|IXFOHUM*ar9pby{Y`%JKs28>OF84Rx1m$sa1~#3W^e_*xk+ zua+CR-;=wVY8opVGNirySz!^p;axHONH50v^NsoU6dzl`cz~Gh%5r|l50&aGP33L; zN^iEyocqxGwfwpGId{VKr?Wio=c9yC;#uVnY^>B)TC0qsW*c@JPnr6dT}FqIH%azg zQSQk4p`)x$TjTI8ktZXUMLNQ(+0#u+%@eG6=uX=Y#yeCxIfht(m%=NO+c8X<>TBma z#kb{4@b7bB+i=yb9fK) zRD3G_8P|r3ktO3uJe$nIcMC_|jF?Z@*`!pg0w!6YtR;{37uMJK0dvaDZuR4zHOUA`=TDTgUF?clLhL4-7Jio(`i$}! ze3{`9<3byFud^}QOMXOEpu>nvUjw)z%s@Aod2E<8+tJ3E!Os*wQ=;Y0BIA1IsOV_n z?uadyPjlZmmU+5)clrCvZ55BWKq)O8_tznQro+iH#0S&^<^ubTVFFW&O|r}}T{k4K zaZojSSCm5vR;&52)dMM6PaQrdW1K)%{z;EFs zxr_>-9q{$DJpC>0rDBo z=`Ktw_AI-IUC+#7&QkHrC3t&S!y1?$sj^HrdN1<>-JV`VPX+B0=*o0&CWWfTj9?d# z<(O$Cf_C>~o>4c67!+k-sgg_!>OK5CX*TsH#?Xi13-@Rug1CuQpiaSa*#_}1=^7e< zI!oPsKY2)gz5f_biqCw9Tq*7??xyZMUf}w{(|J4heOG5+Gv6KlsL#ih@jvi=Bvh9^ zm2OL$lq9(__BCpcy-$$D6QU}45$_M{qm7BZutu7VA16|=viLKiGO>-QLrlcY*h16@ zO~5{bFYd?T>o_i*6n2S>bXD9c*!kJ;O?;?-qCbs)?W@i8@uqoK`99@W@@IXAxF_BO z{v)`%MGF`Fr~JPOMkz|(tt6mtmHSFGRsoMEGKdajEH#RHLxt0G>60|aaLiI>E4z(} zWzR8n*-2~#!x;7xb^vo3Ji34lHS}i@`+{k}_F`pb7?aFIG85Pp&<}m#UED%dVNTEs znI=ppW(S=|_N8x73jLhvz|?g$yeEEwC%#qY$k;#V+}_efWz(Xt{}Q)Vm6l+qCY zvKjjxs}B(m-{AF$i^OqqB( z#8BPvC!5EvgVnlJSX=JFrn4{DPnnkNm(1_XLFObwF%{W~^k0;XK0_Mm=Cqq$Lye(c zlaEM_d`b)>X5jaU2lynS1~G(qACJQ!Dg*V#-VlEf3&}EgdF(OVC$B^M-opC(3HeX? zh15|>5%YyKVY84TRFxhJoH$47vlVn z`J?`9{|D-v<1)nAsiOFiEAZBYAV%|K9JI-H0f8Vx7@p6r669J3{fvfmGg?E zw1QY48vJt#+lWoZzQtMsycIi#{ej)a;vi-r4@-g79R^l;;xP|IoYcfGK+Mk_h-R6J zj-iqe5v0Hx;68|lXa~_MQaIJZczr<@39-p zG^|36k{U}%QVVIJv`e}pCChE(fpSl|g4|YqEHg@5rMc1oo}BDTOT~;T!rGQssfmU_ zELKYhe^11A!Tl-%`wpJD2qKnviT{NEh_Av^@lc`%@eAQ1lF2Z#D_qHT5qpTbWM?uN zo+37r$H?JiU-C1uHu)*JhrTx@y>Lj&>T0;Fq?WFwFb!sGBGjieE(kd9+17UQJA&-#Z)FbftL~;dbfNu$t zi3>zG;x4`rENg@x#O}hZehRbrV~A6Fjbb29W)r-Je5`CzzEUK4zI;`_Dc6%jRKN#Ma_Mv5uG}d;;rGm&M;;P3k#Z3;q=U5*2Zh)JiHPjg>A*r=?EtW>+3O zUqxOkQ}F(=MYhAb-~w2E+oTv#M~Kea1<`!3U<_mMUa%Znh&{!sz|;IZ@cBzfF+>|8 z2SOaz6ZeU~h%)41av-^pr~;ndPJRoA5B%Mg{F6+EFRkB@<;Y&-RdN@M_7LhE*@ApX zM3XzowPY7^15goYSV7()+dyPPHL?_}7>^-mkb}v_WHWLh^j~-KJJLgtWJhuU*_<3l zjwct8H;KbU7vTM#I6~ZnJ5^btD)_zuaRa{sv1oO18{PsRh)XbgXF@z&DTt)23^Vrv zT8wJLEWnT#-kHzCYG^rFUwtXN<%{wS`M5kt9u1?iw_HlTEu~0V(ne{wv{zan&4;T| z2Y8C^BsGU?$S0B_CP1kNTw&`=gQd@<_AsW8O9!N1q(@SHh=GcLC)O3<(~0l?h;-U5|1OtRx+!y&UU04iPo9j@UzrDKnX*IKt6YJ5;{zoOR(k88_7JIe5u*0) zplaA4xQ;A^w~5_wZCVQvbq8TIO~qbgBk>p<$M0ducpJPS%!p_3J~9xlGC$)aQ5SsD zgJ?=ngo&6&q!71>h0tfi0iH$(L>!q*%pm3xM~SnG)uL1k^YJZ;8j_Y1kbs1A7c< z7d8dr4?nB zs6r?_ta4XVN-3dmJ;;aq!E@*%T*(0sPM3d@f00kgX|hRq3hRDf%2T0qQGO|t%3b-6 z`~tq+*x)|NLtYK8CGC_6%4&#k{92gxJ=8vF@f2J%|aTRreT@P6_!U?$_E@b(Zh`8`1M zU%E<-C2N3<)84X3Bz*rxFra+9~Hi!~D zhJJ-~0;NNgumpCiQPdbj+tz`5Ya@t1?1pv3hQU2RjdvXdGhjBfeI2$Pjs@6aYyvaH!X@kuY}#VSOQu#KuBI>oBYzM9mI`V=OiV;7Q=k zZ{Y4f7j%9Dv8B7Q1K5us`5Sfw;62ze7~9LR)!07l5Vj4VMZh-+dSNzUZGS2h`ueTfd^MFqhVW=YB>%qMy(Pv;r+eYoKqJp(SW7 zJeMqlF+CSBi_vo6{0X=gqb;C$FWLd}=L_PXuc2oUPx}z1K%NCxd^O(L0P8;-vV#2~ z5W8FsVvh;PE5p~VaG1+#)NLq072&=X3pDlWYK3Q_WZ(>ksBII(lb3=c0a7GTqJggr zL@*~pFIEBRWUMB{NY}*bL7%<{zG?|$wk7m#b2!?==xqt(_XDgQ|t8#4w#!z{<9#W{1JSs zdI-_wFG0&|csEJ~UFiV5fpR9?HFMy|1b^m0oU$63?*P0LqT4Z;k?Qh=id7H=(d2%J z@24TkTaDLO_XAK6wPY^vqB?Rc;L!+(4i1A{jna<i;~RVWfPsj_MwRiN7reXH&dP`zgb3sfuASbq~(qDJ{EFe22Q zQtF-p3G7vOl&E_RN;V?vPM( z5$scUPN<$%z-D#Vi@IY$ZHc-kL*2hX6|{(ic4~VhRKL^Ubt9Y&kQ<==hJsO~_JN8w zftS?zp!SWrkHQXfzzSv6Yqo-1#aRnrb@zfgb8H1ZR7<7;U8)|{uZI8juUfyl-$AQS z9S2kaP3;AB_lUM@MB6{2_JWF2cblkE+Wr&u8d@M*^N%buM-$~`ylUPuPwo64Ws9m*1wL`U}AcI_m z0*87{5ByY$ic!aZa8lc)CbiAMdvWw#IXX_uRlfuvt)zNS^_-e?4{EpoR^?Fl<>*{$ zS(6H!HK?$py01o))A_aD(KM-+sQjw+YCE(xXclTXt##UNA>DG#e(kKaT$c~{MboC2 z)sp(t%W57d7k~vc&BGc+TM2KtzxtkT&pS{;L){ddC{}3vnXCIt3P%A zX>f7YYEnh(3bd&(rEp2}U~%;Z+O1Jled_OB-ohH+rE7gv7#}PXfP=Mb*x++v4SFe9 zM#Ba4={x~U08(lHUc1Vp%e-5YuD@trm{XT8$`vT9{3_SK=M2c`&_DHHz;2cEpEL(^ z6~|K?POtet&x5rD;9xJQ*x<6NTPqhwDOz(;$)a>kS8-hbDXw^p!Pvr@f?KJ#K<5b1 zg601`TvSfiS6uG~&r}swdA0Mq^#t_jn%=Fc@F>%?shWai1DXq}0&3CpF88~bV3~hS z4RGpwdS2MB09KO-w5}*cZ@_P!EjNzf1(w|YvrO?Ra-EwXq$@G7F^ck zf+d1sjq_cMt|9nZ8Ng}Qs8RvR;y87BfJ@g=xIM*T1Nw@S4oJKUX`>+^uW; z%{oPY+S1FqWU$VF=ECDi$7{KI)-l0QVU2nzz!|{kI9-!gn~p8c!T_hPMePw?B7iF% zQaN=Ey2bCx7R3}TYr2cWYT9+pYOZ05xJ*%a{w0G+W&~<(9VsuL3Jkai7 zd|_#|R6Jj>Wx?_qtjQI|2ijE_A82QwCfy1>4`?katCtIN>ezR4jqjhh0~TnK!F&ad z0DlB1!PIyCp;5I~>r%QmgR#MIfGQCxB5;_C|L)NR%CV44d5YkW~^(el66rg8sM zJO7<7)iz!B-}4r=^olC=Meh%OQkV-FH(#nCJ(X|$BkB%>ztJ*YP%@^H zQUS@ru+Fba>u}M$sCB6{RZdFfVs;)U;&;K7tv!bXiT3xzD zDz$iye{ZF(BVc{;ngVx8@172nDd|8RDACRTOJG-&mE|%Fz>tE8oel2a65x(!LZ7s)>9Z)xD>1> z0P8$ju9yBNngTlCg#vnYntm11%kSPB0u&tz&ULT8t2vnK{~rEN^cBaW+f#gv?^55z z6qO9-&}&rZr2f-a%+;@bff24lf&U6r^#&kM^JZMCjP_u9Xs>->7I zjRn1=d-tEU1UTP?3QK7g6~?_QQy8PSLd9y50eQ{R;?LT5Ts0m*iviHT{;MVR|0VUP OiQ=XI7yGY0@P7cz=1kiF literal 0 HcmV?d00001 From 9fe265ea6489fe3c9cba72dc94b22463d96829a0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 5 May 2025 13:41:30 -0400 Subject: [PATCH 34/55] [WIP] AWS Nova Sonic service - implement ability to persist and load conversations --- .../20e-persistent-context-aws-nova-sonic.py | 256 ++++++++++++++++++ examples/foundational/39-aws-nova-sonic.py | 2 +- src/pipecat/services/aws_nova_sonic/aws.py | 102 +++++-- .../services/aws_nova_sonic/context.py | 29 +- 4 files changed, 350 insertions(+), 39 deletions(-) create mode 100644 examples/foundational/20e-persistent-context-aws-nova-sonic.py diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py new file mode 100644 index 000000000..8a95f54b9 --- /dev/null +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -0,0 +1,256 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import glob +import json +import os +from datetime import datetime + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.aws_nova_sonic.aws import AWSNovaSonicLLMService +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.network.small_webrtc import SmallWebRTCTransport +from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection + +load_dotenv(override=True) + +BASE_FILENAME = "/tmp/pipecat_conversation_" + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + temperature = 75 if args["format"] == "fahrenheit" else 24 + await result_callback( + { + "conditions": "nice", + "temperature": temperature, + "format": args["format"], + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + } + ) + + +async def get_saved_conversation_filenames( + function_name, tool_call_id, args, llm, context, result_callback +): + # Construct the full pattern including the BASE_FILENAME + full_pattern = f"{BASE_FILENAME}*.json" + + # Use glob to find all matching files + matching_files = glob.glob(full_pattern) + logger.debug(f"matching files: {matching_files}") + + await result_callback({"filenames": matching_files}) + + +# async def get_saved_conversation_filenames( +# function_name, tool_call_id, args, llm, context, result_callback +# ): +# pattern = re.compile(re.escape(BASE_FILENAME) + "\\d{8}_\\d{6}\\.json$") +# matching_files = [] + +# for filename in os.listdir("."): +# if pattern.match(filename): +# matching_files.append(filename) + +# await result_callback({"filenames": matching_files}) + + +async def save_conversation(function_name, tool_call_id, args, llm, context, result_callback): + timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + filename = f"{BASE_FILENAME}{timestamp}.json" + logger.debug( + f"writing conversation to {filename}\n{json.dumps(context.get_messages_for_persistent_storage(), indent=4)}" + ) + try: + with open(filename, "w") as file: + messages = context.get_messages_for_persistent_storage() + # remove the last message, which is the instruction we just gave to save the conversation + messages.pop() + json.dump(messages, file, indent=2) + await result_callback({"success": True}) + except Exception as e: + await result_callback({"success": False, "error": str(e)}) + + +async def load_conversation(function_name, tool_call_id, args, llm, context, result_callback): + async def _reset(): + filename = args["filename"] + logger.debug(f"loading conversation from {filename}") + try: + with open(filename, "r") as file: + messages = json.load(file) + messages.append( + { + "role": "user", + "content": f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}", + } + ) + context.set_messages(messages) + await llm.reset_conversation() + await llm.trigger_assistant_response() + except Exception as e: + await result_callback({"success": False, "error": str(e)}) + + asyncio.create_task(_reset()) + + +get_current_weather_tool = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], +) + +save_conversation_tool = FunctionSchema( + name="save_conversation", + description="Save the current conversation. Use this function to persist the current conversation to external storage.", + properties={}, + required=[], +) + +get_saved_conversation_filenames_tool = FunctionSchema( + name="get_saved_conversation_filenames", + description="Get a list of saved conversation histories. Returns a list of filenames. Each filename includes a date and timestamp. Each file is conversation history that can be loaded into this session.", + properties={}, + required=[], +) + +load_conversation_tool = FunctionSchema( + name="load_conversation", + description="Load a conversation history. Use this function to load a conversation history into the current session.", + properties={ + "filename": { + "type": "string", + "description": "The filename of the conversation history to load.", + } + }, + required=["filename"], +) + +tools = ToolsSchema( + standard_tools=[ + get_current_weather_tool, + save_conversation_tool, + get_saved_conversation_filenames_tool, + load_conversation_tool, + ] +) + + +async def run_bot(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Starting bot") + + transport = SmallWebRTCTransport( + webrtc_connection=webrtc_connection, + params=TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) + + system_instruction = ( + "You are a friendly assistant. The user and you will engage in a spoken dialog exchanging " + "the transcripts of a natural real-time conversation. Keep your responses short, generally " + "two or three sentences for chatty scenarios. " + f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}" + ) + + llm = AWSNovaSonicLLMService( + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + region=os.getenv("AWS_REGION"), + voice_id="tiffany", # matthew, tiffany, amy + # you could choose to pass instruction here rather than via context + # system_instruction=system_instruction, + # you could choose to pass tools here rather than via context + # tools=tools + ) + + llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("save_conversation", save_conversation) + llm.register_function("get_saved_conversation_filenames", get_saved_conversation_filenames) + llm.register_function("load_conversation", load_conversation) + + context = OpenAILLMContext( + messages=[ + {"role": "system", "content": f"{system_instruction}"}, + ], + tools=tools, + ) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + # HACK: for now, we need this special way of triggering the first assistant response in AWS + # Nova Sonic. Note that this trigger requires a special corresponding bit of text in the + # system instruction. In the future, simply queueing the context frame should be sufficient. + await llm.trigger_assistant_response() + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from run import main + + main() diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index 07670f75a..c80626962 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -102,7 +102,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): region=os.getenv("AWS_REGION"), voice_id="tiffany", # matthew, tiffany, amy # you could choose to pass instruction here rather than via context - # instruction=system_instruction + # system_instruction=system_instruction # you could choose to pass tools here rather than via context # tools=tools ) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 5b69810f3..50b83d3e0 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -7,6 +7,7 @@ import asyncio import base64 import json +import time import uuid import wave from dataclasses import dataclass @@ -119,7 +120,7 @@ class AWSNovaSonicLLMService(LLMService): region: str, model: str = "amazon.nova-sonic-v1:0", voice_id: str = "matthew", # matthew, tiffany, amy - instruction: Optional[str] = None, + system_instruction: Optional[str] = None, tools: Optional[ToolsSchema] = None, send_transcription_frames: bool = True, **kwargs, @@ -131,7 +132,7 @@ class AWSNovaSonicLLMService(LLMService): self._model = model self._client: BedrockRuntimeClient = None self._voice_id = voice_id - self._instruction = instruction + self._system_instruction = system_instruction self._tools = tools self._send_transcription_frames = send_transcription_frames self._context: AWSNovaSonicLLMContext = None @@ -150,6 +151,8 @@ class AWSNovaSonicLLMService(LLMService): self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False self._assistant_response_trigger_audio: bytes = None # Not cleared on _disconnect() + self._disconnecting = False + self._connected_time: float = None # # standard AIService frame handling @@ -174,6 +177,18 @@ class AWSNovaSonicLLMService(LLMService): await super().cancel(frame) await self._disconnect() + # + # conversation resetting + # + + async def reset_conversation(self): + logger.debug("Resetting conversation") + await self._disconnect() + await self._start_connecting() + # Use existing context + self._context_available = True + await self._finish_connecting_if_context_available() + # # frame processing # @@ -207,10 +222,12 @@ class AWSNovaSonicLLMService(LLMService): async def _handle_context(self, context: OpenAILLMContext): # TODO: reset connection if needed (if entirely new context object provided, for instance) - print(f"[pk] receive updated context: {context.get_messages_for_initializing_history()}") + print(f"[pk] received updated context: {context.get_messages_for_initializing_history()}") if not self._context: # We got our initial context - try to finish connecting - self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic(context) + self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( + context, self._system_instruction + ) self._context_available = True await self._finish_connecting_if_context_available() @@ -296,8 +313,8 @@ class AWSNovaSonicLLMService(LLMService): # Read context history = self._context.get_messages_for_initializing_history() - # Send prompt start event, specifying tools - # Tools from context take priority over tools from __init__() + # Send prompt start event, specifying tools. + # Tools from context take priority over self._tools. tools = ( self._context.tools if self._context.tools @@ -305,11 +322,14 @@ class AWSNovaSonicLLMService(LLMService): ) await self._send_prompt_start_event(tools) - # Send system instruction - # Instruction from context takes priority over instruction from __init__() - instruction = history.instruction if history.instruction else self._instruction - if instruction: - await self._send_text_event(text=instruction, role=Role.SYSTEM) + # Send system instruction. + # Instruction from context takes priority over self._system_instruction. + # (NOTE: this prioritizing occurred automatically behind the scenes: the context was + # initialized with self._system_instruction and then updated itself from its messages when + # get_messages_for_initializing_history() was called). + # print(f"[pk] connecting, with system instruction: {history.system_instruction}") + if history.system_instruction: + await self._send_text_event(text=history.system_instruction, role=Role.SYSTEM) # Send conversation history for message in history.messages: @@ -320,7 +340,7 @@ class AWSNovaSonicLLMService(LLMService): # - pass additional message(s) # - merge init-passed system instruction + context instruction (latter takes precedence) # - merge init-passed tools + context tools (latter takes precedence) - await self._send_text_event(text=self._instruction, role=Role.SYSTEM) + await self._send_text_event(text=self._system_instruction, role=Role.SYSTEM) # Start audio input await self._send_audio_input_start_event() @@ -328,31 +348,43 @@ class AWSNovaSonicLLMService(LLMService): # Start receiving events self._receive_task = self.create_task(self._receive_task_handler()) - # If we need to, send assistant response trigger + # Record finished connecting time + self._connected_time = time.time() + + # If we need to, send assistant response trigger (depends on self._connected_time) if self._triggering_assistant_response: - # If the trigger was the first audio chunk sent on this connection it'd be ignored (I'm - # guessing the LLM can't quite "hear" the first little bit of audio sent). So send a bit - # of leading blank audio first. - await self._send_assistant_response_trigger(lead_with_blank_audio=True) + await self._send_assistant_response_trigger() self._triggering_assistant_response = False async def _disconnect(self): try: - # Clean up receive task - if self._receive_task: - await self.cancel_task(self._receive_task, timeout=1.0) - self._receive_task = None + # NOTE: see explanation of HACK, below + self._disconnecting = True # Clean up client if self._client: + print("[pk] Cleaning up client") await self._send_session_end_events() self._client = None # Clean up stream if self._stream: + print("[pk] Cleaning up stream") await self._stream.input_stream.close() self._stream = None + # NOTE: see explanation of HACK, below + await asyncio.sleep(1) + + # Clean up receive task + # HACK: we should ideally be able to cancel the receive task before stopping the input + # stream, above (meaning we wouldn't need self._disconnecting). But for some reason if + # we don't close the input stream and wait a second first, we're getting an error a lot + # like this one: https://github.com/awslabs/amazon-transcribe-streaming-sdk/issues/61. + if self._receive_task: + await self.cancel_task(self._receive_task, timeout=1.0) + self._receive_task = None + # Reset remaining connection-specific state self._prompt_name = None self._input_audio_content_name = None @@ -362,6 +394,8 @@ class AWSNovaSonicLLMService(LLMService): self._ready_to_send_context = False self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False + self._disconnecting = False + self._connected_time = None except Exception as e: logger.error(f"{self} error disconnecting: {e}") @@ -619,9 +653,8 @@ class AWSNovaSonicLLMService(LLMService): # LLM communication: output events (LLM -> pipecat) # - # Receive the ongoing LLM "completion". - # There is generally a single completion per session. - # In a completion, a few different kinds of content can be delivered: + # Receive events for the session. + # A few different kinds of content can be delivered: # - Transcription of user audio # - Tool use # - Text preview of planned response speech before audio delivered @@ -633,7 +666,7 @@ class AWSNovaSonicLLMService(LLMService): # The overall completion is wrapped by "completionStart" and "completionEnd" events. async def _receive_task_handler(self): try: - while self._client: + while self._client and not self._disconnecting: output = await self._stream.await_output() result = await output[1].receive() @@ -906,16 +939,25 @@ class AWSNovaSonicLLMService(LLMService): await self._send_assistant_response_trigger() self._triggering_assistant_response = False - async def _send_assistant_response_trigger(self, lead_with_blank_audio=False): + async def _send_assistant_response_trigger(self): # TODO: if/when we make bitrate, etc configurable, avoid hard-coding this chunk_size = 640 # equivalent to what we get from InputAudioRawFrame chunk_duration = 640 / ( 16000 * 2 ) # 640 bytes of 16-bit (2-byte) PCM mono audio at 16kHz corresponds to 0.02 seconds - # Lead with blank audio, if needed - if lead_with_blank_audio: - blank_audio_duration = 0.5 # much less than this and it doesn't reliably work + # Lead with a bit of blank audio, if needed. + # It seems like the LLM can't quite "hear" the first little bit of audio sent on a + # connection. + current_time = time.time() + max_blank_audio_duration = 0.5 + blank_audio_duration = ( + max_blank_audio_duration - (current_time - self._connected_time) + if self._connected_time is not None + and (current_time - self._connected_time) < max_blank_audio_duration + else None + ) + if blank_audio_duration: blank_audio_chunk = b"\x00" * chunk_size num_chunks = int(blank_audio_duration / chunk_duration) for _ in range(num_chunks): @@ -925,7 +967,7 @@ class AWSNovaSonicLLMService(LLMService): # Send trigger audio # NOTE: this audio *will* be transcribed and eventually make it into the context. That's OK: # if we ever need to seed this service again with context it would make sense to include it - # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the + # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the # context as well. # print(f"[pk] sending trigger audio! {len(self._assistant_response_trigger_audio)}") audio_chunks = [ diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 3fac65a72..b12061e1e 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -49,7 +49,7 @@ class AWSNovaSonicConversationHistoryMessage: @dataclass class AWSNovaSonicConversationHistory: - instruction: str = None + system_instruction: str = None messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) @@ -58,18 +58,22 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): super().__init__(messages=messages, tools=tools, **kwargs) self.__setup_local() - def __setup_local(self): + def __setup_local(self, system_instruction: str = ""): self._assistant_text = "" + self._system_instruction = system_instruction @staticmethod - def upgrade_to_nova_sonic(obj: OpenAILLMContext) -> "AWSNovaSonicLLMContext": + def upgrade_to_nova_sonic( + obj: OpenAILLMContext, system_instruction: str + ) -> "AWSNovaSonicLLMContext": if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): obj.__class__ = AWSNovaSonicLLMContext - obj.__setup_local() + obj.__setup_local(system_instruction) return obj + # NOTE: this method has the side-effect of updating _system_instruction from messages def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: - history = AWSNovaSonicConversationHistory() + history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) # Bail if there are no messages if not self.messages: @@ -82,13 +86,15 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): system = messages.pop(0) content = system.get("content") if isinstance(content, str): - history.instruction = content + history.system_instruction = content elif isinstance(content, list): - history.instruction = content[0].get("text") + history.system_instruction = content[0].get("text") + if history.system_instruction: + self._system_instruction = history.system_instruction # Process remaining messages to fill out conversation history. # Nova Sonic supports "user" and "assistant" messages in history. - print(f"[pk] standard messages: {messages}") + # print(f"[pk] standard messages: {messages}") for message in messages: history_message = self.from_standard_message(message) if history_message: @@ -96,6 +102,13 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): return history + def get_messages_for_persistent_storage(self): + messages = super().get_messages_for_persistent_storage() + # If we have a system instruction and messages doesn't already contain it, add it + if self._system_instruction and not (messages and messages[0].get("role") == "system"): + messages.insert(0, {"role": "system", "content": self._system_instruction}) + return messages + def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: role = message.get("role") if message.get("role") == "user" or message.get("role") == "assistant": From 2b02d08f4c7f03fba5f18706ec600476935a5c50 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 09:26:22 -0400 Subject: [PATCH 35/55] [WIP] AWS Nova Sonic service - add comments to examples pointing out the us-east-1 is the only supported region so far --- examples/foundational/20e-persistent-context-aws-nova-sonic.py | 2 +- examples/foundational/39-aws-nova-sonic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index 8a95f54b9..731c69c3a 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -185,7 +185,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): llm = AWSNovaSonicLLMService( secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), + region=os.getenv("AWS_REGION"), # as of 2025-05-06, us-east-1 is the only supported region voice_id="tiffany", # matthew, tiffany, amy # you could choose to pass instruction here rather than via context # system_instruction=system_instruction, diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index c80626962..a89796ea6 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -99,7 +99,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): llm = AWSNovaSonicLLMService( secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - region=os.getenv("AWS_REGION"), + region=os.getenv("AWS_REGION"), # as of 2025-05-06, us-east-1 is the only supported region voice_id="tiffany", # matthew, tiffany, amy # you could choose to pass instruction here rather than via context # system_instruction=system_instruction From 467233be046527da8be0bf40ebf5aaea147d8c36 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 09:48:50 -0400 Subject: [PATCH 36/55] [WIP] AWS Nova Sonic service - support multi-line system prompt --- .../foundational/20e-persistent-context-aws-nova-sonic.py | 4 ++++ examples/foundational/39-aws-nova-sonic.py | 1 - src/pipecat/services/aws_nova_sonic/aws.py | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index 731c69c3a..8ac1508f8 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -175,6 +175,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): ), ) + # Specify initial system instruction. + # HACK: note that, for now, we need to inject a special bit of text into this instruction to + # allow the first assistant response to be programmatically triggered (which happens in the + # on_client_connected handler, below) system_instruction = ( "You are a friendly assistant. The user and you will engage in a spoken dialog exchanging " "the transcripts of a natural real-time conversation. Keep your responses short, generally " diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index a89796ea6..fe0d07dca 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -87,7 +87,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): # HACK: note that, for now, we need to inject a special bit of text into this instruction to # allow the first assistant response to be programmatically triggered (which happens in the # on_client_connected handler, below) - # TODO: looks like Nova Sonic can't handle new lines? system_instruction = ( "You are a friendly assistant. The user and you will engage in a spoken dialog exchanging " "the transcripts of a natural real-time conversation. Keep your responses short, generally " diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 50b83d3e0..e5df20e66 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -498,7 +498,7 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(audio_content_start) async def _send_text_event(self, text: str, role: Role): - if not self._stream: + if not self._stream or not text: return content_name = str(uuid.uuid4()) @@ -521,13 +521,14 @@ class AWSNovaSonicLLMService(LLMService): ''' await self._send_client_event(text_content_start) + escaped_text = json.dumps(text) # includes quotes text_input = f''' {{ "event": {{ "textInput": {{ "promptName": "{self._prompt_name}", "contentName": "{content_name}", - "content": "{text}" + "content": {escaped_text} }} }} }} From c4d0f91a7fbed227b4c5ae80382a0cfeb9957389 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 09:53:12 -0400 Subject: [PATCH 37/55] [WIP] AWS Nova Sonic service - remove some old code that was accidentally still there, possibly sending a duplicate system instruction --- src/pipecat/services/aws_nova_sonic/aws.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index e5df20e66..aa44beb60 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -335,13 +335,6 @@ class AWSNovaSonicLLMService(LLMService): for message in history.messages: await self._send_text_event(text=message.text, role=message.role) - # Send initial context (system instruction and conversation history) - # TODO: finish implementing - # - pass additional message(s) - # - merge init-passed system instruction + context instruction (latter takes precedence) - # - merge init-passed tools + context tools (latter takes precedence) - await self._send_text_event(text=self._system_instruction, role=Role.SYSTEM) - # Start audio input await self._send_audio_input_start_event() From d388c057c039531a7f96a081eb69793e6e7f4df0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 11:12:47 -0400 Subject: [PATCH 38/55] [WIP] AWS Nova Sonic service - recover from unwanted disconnection due to an error --- src/pipecat/services/aws_nova_sonic/aws.py | 7 +++++++ src/pipecat/services/aws_nova_sonic/context.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index aa44beb60..aa264155c 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -153,6 +153,7 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_response_trigger_audio: bytes = None # Not cleared on _disconnect() self._disconnecting = False self._connected_time: float = None + self._wants_connection = False # # standard AIService frame handling @@ -160,6 +161,7 @@ class AWSNovaSonicLLMService(LLMService): async def start(self, frame: StartFrame): await super().start(frame) + self._wants_connection = True # TODO: maybe connect but don't send history until we get all of our settings? # how do we know how long to wait? # ah, i think we'll *always* get at least one OpenAILLMContextFrame which kicks things off @@ -171,10 +173,12 @@ class AWSNovaSonicLLMService(LLMService): async def stop(self, frame: EndFrame): await super().stop(frame) + self._wants_connection = False await self._disconnect() async def cancel(self, frame: CancelFrame): await super().cancel(frame) + self._wants_connection = False await self._disconnect() # @@ -183,6 +187,7 @@ class AWSNovaSonicLLMService(LLMService): async def reset_conversation(self): logger.debug("Resetting conversation") + await self._handle_bot_stopped_speaking() await self._disconnect() await self._start_connecting() # Use existing context @@ -694,6 +699,8 @@ class AWSNovaSonicLLMService(LLMService): except Exception as e: logger.error(f"{self} error processing responses: {e}") + if self._wants_connection: + await self.reset_conversation() async def _handle_completion_start_event(self, event_json): # print("[pk] completion start") diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index b12061e1e..d96c2d1ed 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -143,6 +143,8 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # print(f"[pk] assistant text buffered: {self._assistant_text}") def flush_aggregated_assistant_text(self): + if not self._assistant_text: + return message = { "role": "assistant", "content": [{"type": "text", "text": self._assistant_text}], From 73020be511c64e2026f1ed95730686198929b3df Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 12:25:25 -0400 Subject: [PATCH 39/55] [WIP] AWS Nova Sonic service - minor fix: only try to read received JSON if we have it --- src/pipecat/services/aws_nova_sonic/aws.py | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index aa264155c..c2b56ef74 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -519,7 +519,7 @@ class AWSNovaSonicLLMService(LLMService): ''' await self._send_client_event(text_content_start) - escaped_text = json.dumps(text) # includes quotes + escaped_text = json.dumps(text) # includes quotes text_input = f''' {{ "event": {{ @@ -673,29 +673,29 @@ class AWSNovaSonicLLMService(LLMService): response_data = result.value.bytes_.decode("utf-8") json_data = json.loads(response_data) - if "event" in json_data: - event_json = json_data["event"] - if "completionStart" in event_json: - # Handle the LLM completion starting - await self._handle_completion_start_event(event_json) - elif "contentStart" in event_json: - # Handle a piece of content starting - await self._handle_content_start_event(event_json) - elif "textOutput" in event_json: - # Handle text output content - await self._handle_text_output_event(event_json) - elif "audioOutput" in event_json: - # Handle audio output content - await self._handle_audio_output_event(event_json) - elif "toolUse" in event_json: - # Handle tool use - await self._handle_tool_use_event(event_json) - elif "contentEnd" in event_json: - # Handle a piece of content ending - await self._handle_content_end_event(event_json) - elif "completionEnd" in event_json: - # Handle the LLM completion ending - await self._handle_completion_end_event(event_json) + if "event" in json_data: + event_json = json_data["event"] + if "completionStart" in event_json: + # Handle the LLM completion starting + await self._handle_completion_start_event(event_json) + elif "contentStart" in event_json: + # Handle a piece of content starting + await self._handle_content_start_event(event_json) + elif "textOutput" in event_json: + # Handle text output content + await self._handle_text_output_event(event_json) + elif "audioOutput" in event_json: + # Handle audio output content + await self._handle_audio_output_event(event_json) + elif "toolUse" in event_json: + # Handle tool use + await self._handle_tool_use_event(event_json) + elif "contentEnd" in event_json: + # Handle a piece of content ending + await self._handle_content_end_event(event_json) + elif "completionEnd" in event_json: + # Handle the LLM completion ending + await self._handle_completion_end_event(event_json) except Exception as e: logger.error(f"{self} error processing responses: {e}") From 885b2d1d2f3dcfa6e04b1c347848ca331c4722b8 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 14:29:36 -0400 Subject: [PATCH 40/55] [WIP] AWS Nova Sonic service - make parameters configurable --- src/pipecat/services/aws_nova_sonic/aws.py | 73 ++++++++++++++-------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index c2b56ef74..c0367fa6b 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -28,6 +28,7 @@ from aws_sdk_bedrock_runtime.models import ( InvokeModelWithBidirectionalStreamOutput, ) from loguru import logger +from pydantic import BaseModel, Field from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver from smithy_aws_core.identity import AWSCredentialsIdentity from smithy_core.aio.eventstream import DuplexEventStream @@ -107,6 +108,23 @@ class CurrentContent: ) +class Params(BaseModel): + # Audio input + input_sample_rate: Optional[int] = Field(default=16000) + input_sample_size: Optional[int] = Field(default=16) + input_channel_count: Optional[int] = Field(default=1) + + # Audio output + output_sample_rate: Optional[int] = Field(default=24000) + output_sample_size: Optional[int] = Field(default=16) + output_channel_count: Optional[int] = Field(default=1) + + # Inference + max_tokens: Optional[int] = Field(default=1024) + top_p: Optional[float] = Field(default=0.9) + temperature: Optional[float] = Field(default=0.7) + + class AWSNovaSonicLLMService(LLMService): # Override the default adapter to use the AWSNovaSonicLLMAdapter one adapter_class = AWSNovaSonicLLMAdapter @@ -120,6 +138,7 @@ class AWSNovaSonicLLMService(LLMService): region: str, model: str = "amazon.nova-sonic-v1:0", voice_id: str = "matthew", # matthew, tiffany, amy + params: Params = Params(), system_instruction: Optional[str] = None, tools: Optional[ToolsSchema] = None, send_transcription_frames: bool = True, @@ -132,6 +151,7 @@ class AWSNovaSonicLLMService(LLMService): self._model = model self._client: BedrockRuntimeClient = None self._voice_id = voice_id + self._params = params self._system_instruction = system_instruction self._tools = tools self._send_transcription_frames = send_transcription_frames @@ -419,18 +439,18 @@ class AWSNovaSonicLLMService(LLMService): # TODO: make params configurable? async def _send_session_start_event(self): - session_start = """ - { - "event": { - "sessionStart": { - "inferenceConfiguration": { - "maxTokens": 1024, - "topP": 0.9, - "temperature": 0.7 - } - } - } - } + session_start = f""" + {{ + "event": {{ + "sessionStart": {{ + "inferenceConfiguration": {{ + "maxTokens": {self._params.max_tokens}, + "topP": {self._params.top_p}, + "temperature": {self._params.temperature} + }} + }} + }} + }} """ await self._send_client_event(session_start) @@ -458,9 +478,9 @@ class AWSNovaSonicLLMService(LLMService): }}, "audioOutputConfiguration": {{ "mediaType": "audio/lpcm", - "sampleRateHertz": 24000, - "sampleSizeBits": 16, - "channelCount": 1, + "sampleRateHertz": {self._params.output_sample_rate}, + "sampleSizeBits": {self._params.output_sample_size}, + "channelCount": {self._params.output_channel_count}, "voiceId": "{self._voice_id}", "encoding": "base64", "audioType": "SPEECH" @@ -483,9 +503,9 @@ class AWSNovaSonicLLMService(LLMService): "role": "USER", "audioInputConfiguration": {{ "mediaType": "audio/lpcm", - "sampleRateHertz": 16000, - "sampleSizeBits": 16, - "channelCount": 1, + "sampleRateHertz": {self._params.input_sample_rate}, + "sampleSizeBits": {self._params.input_sample_size}, + "channelCount": {self._params.input_channel_count}, "audioType": "SPEECH", "encoding": "base64" }} @@ -762,11 +782,10 @@ class AWSNovaSonicLLMService(LLMService): # Push audio frame audio = base64.b64decode(audio_content) - # TODO: make sample rate + channels (used in multiple places) consts frame = TTSAudioRawFrame( audio=audio, - sample_rate=24000, - num_channels=1, + sample_rate=self._params.output_sample_rate, + num_channels=self._params.output_channel_count, ) await self.push_frame(frame) @@ -941,11 +960,13 @@ class AWSNovaSonicLLMService(LLMService): self._triggering_assistant_response = False async def _send_assistant_response_trigger(self): - # TODO: if/when we make bitrate, etc configurable, avoid hard-coding this - chunk_size = 640 # equivalent to what we get from InputAudioRawFrame - chunk_duration = 640 / ( - 16000 * 2 - ) # 640 bytes of 16-bit (2-byte) PCM mono audio at 16kHz corresponds to 0.02 seconds + chunk_duration = 0.02 # what we might get from InputAudioRawFrame + chunk_size = int( + chunk_duration + * self._params.input_sample_rate + * self._params.input_channel_count + * (self._params.input_sample_size / 8) + ) # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes # Lead with a bit of blank audio, if needed. # It seems like the LLM can't quite "hear" the first little bit of audio sent on a From c7e223e85ae9d33d8194966c6de88a39311a7ef6 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 16:06:29 -0400 Subject: [PATCH 41/55] [WIP] AWS Nova Sonic service - remove print statements in favor of logger --- examples/foundational/39-aws-nova-sonic.py | 6 -- src/pipecat/services/aws_nova_sonic/aws.py | 72 +++++++------------ .../services/aws_nova_sonic/context.py | 7 +- 3 files changed, 29 insertions(+), 56 deletions(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index fe0d07dca..af44cf790 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -10,7 +10,6 @@ from datetime import datetime from dotenv import load_dotenv from loguru import logger -# import logging from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -27,11 +26,6 @@ from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection # Load environment variables load_dotenv(override=True) -# logging.basicConfig( -# level=logging.DEBUG, -# format='%(asctime)s - %(levelname)s - %(message)s' -# ) - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): temperature = 75 if args["format"] == "fahrenheit" else 24 diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index c0367fa6b..1e318f164 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -229,25 +229,10 @@ class AWSNovaSonicLLMService(LLMService): await self._handle_bot_stopped_speaking() elif isinstance(frame, AWSNovaSonicFunctionCallResultFrame): await self._handle_function_call_result(frame) - # TODO: do we need to do anything for the below four frame types? - elif isinstance(frame, StartInterruptionFrame): - # print("[pk] StartInterruptionFrame") - pass - elif isinstance(frame, UserStartedSpeakingFrame): - # print("[pk] UserStartedSpeakingFrame") - pass - elif isinstance(frame, StopInterruptionFrame): - # print("[pk] StopInterruptionFrame") - pass - elif isinstance(frame, UserStoppedSpeakingFrame): - # print("[pk] UserStoppedSpeakingFrame") - pass await self.push_frame(frame, direction) async def _handle_context(self, context: OpenAILLMContext): - # TODO: reset connection if needed (if entirely new context object provided, for instance) - print(f"[pk] received updated context: {context.get_messages_for_initializing_history()}") if not self._context: # We got our initial context - try to finish connecting self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( @@ -303,6 +288,8 @@ class AWSNovaSonicLLMService(LLMService): async def _start_connecting(self): try: + logger.info("Connecting...") + if self._client: # Here we assume that if we have a client we are connected or connecting return @@ -335,6 +322,8 @@ class AWSNovaSonicLLMService(LLMService): if not (self._context_available and self._ready_to_send_context): return + logger.info("Finishing connecting (setting up session)...") + # Read context history = self._context.get_messages_for_initializing_history() @@ -345,6 +334,7 @@ class AWSNovaSonicLLMService(LLMService): if self._context.tools else self.get_llm_adapter().from_standard_tools(self._tools) ) + logger.debug(f"Using tools: {tools}") await self._send_prompt_start_event(tools) # Send system instruction. @@ -352,7 +342,7 @@ class AWSNovaSonicLLMService(LLMService): # (NOTE: this prioritizing occurred automatically behind the scenes: the context was # initialized with self._system_instruction and then updated itself from its messages when # get_messages_for_initializing_history() was called). - # print(f"[pk] connecting, with system instruction: {history.system_instruction}") + logger.debug(f"Using system instruction: {history.system_instruction}") if history.system_instruction: await self._send_text_event(text=history.system_instruction, role=Role.SYSTEM) @@ -366,9 +356,11 @@ class AWSNovaSonicLLMService(LLMService): # Start receiving events self._receive_task = self.create_task(self._receive_task_handler()) - # Record finished connecting time + # Record finished connecting time (must be done before sending assistant response trigger) self._connected_time = time.time() + logger.info("Finished connecting") + # If we need to, send assistant response trigger (depends on self._connected_time) if self._triggering_assistant_response: await self._send_assistant_response_trigger() @@ -376,18 +368,18 @@ class AWSNovaSonicLLMService(LLMService): async def _disconnect(self): try: + logger.info("Disconnecting...") + # NOTE: see explanation of HACK, below self._disconnecting = True # Clean up client if self._client: - print("[pk] Cleaning up client") await self._send_session_end_events() self._client = None # Clean up stream if self._stream: - print("[pk] Cleaning up stream") await self._stream.input_stream.close() self._stream = None @@ -414,6 +406,8 @@ class AWSNovaSonicLLMService(LLMService): self._triggering_assistant_response = False self._disconnecting = False self._connected_time = None + + logger.info("Finished disconnecting") except Exception as e: logger.error(f"{self} error disconnecting: {e}") @@ -611,8 +605,6 @@ class AWSNovaSonicLLMService(LLMService): if not self._stream: return - # print(f"[pk] sending tool result. tool call ID: {tool_call_id}, result: {result}") - content_name = str(uuid.uuid4()) result_content_start = f''' @@ -723,7 +715,6 @@ class AWSNovaSonicLLMService(LLMService): await self.reset_conversation() async def _handle_completion_start_event(self, event_json): - # print("[pk] completion start") pass async def _handle_content_start_event(self, event_json): @@ -744,10 +735,6 @@ class AWSNovaSonicLLMService(LLMService): ) self._content_being_received = content - # print(f"[pk] content start: {content}") - # if content.role == Role.ASSISTANT: - # print(f"[pk] assistant content start: {content}") - if content.role == Role.ASSISTANT: if content.type == ContentType.AUDIO: # Note that an assistant response can comprise of multiple audio blocks @@ -763,9 +750,6 @@ class AWSNovaSonicLLMService(LLMService): content = self._content_being_received text_content = event_json["textOutput"]["content"] - # print(f"[pk] text output. content: {text_content}") - # if content.role == Role.ASSISTANT: - # print(f"[pk] assistant text output. content: {text_content}") # Bookkeeping: augment the current content being received with text # Assumption: only one text content per content block @@ -778,7 +762,6 @@ class AWSNovaSonicLLMService(LLMService): # Get audio audio_content = event_json["audioOutput"]["content"] - # print(f"[pk] audio output. content: {len(audio_content)}") # Push audio frame audio = base64.b64decode(audio_content) @@ -800,10 +783,6 @@ class AWSNovaSonicLLMService(LLMService): tool_call_id = tool_use["toolUseId"] arguments = json.loads(tool_use["content"]) - # print( - # f"[pk] tool use - function_name: {function_name}, tool_call_id: {tool_call_id}, arguments: {arguments}" - # ) - # Call tool function if self.has_function(function_name): if function_name in self._functions.keys(): @@ -833,9 +812,6 @@ class AWSNovaSonicLLMService(LLMService): content_end = event_json["contentEnd"] stop_reason = content_end["stopReason"] - # print(f"[pk] content end: {content}.\n stop_reason: {stop_reason}") - # if content.role == Role.ASSISTANT: - # print(f"[pk] assistant content end: {content}.\n stop_reason: {stop_reason}") # Bookkeeping: clear current content being received self._content_being_received = None @@ -856,25 +832,24 @@ class AWSNovaSonicLLMService(LLMService): self._content_being_received = False async def _handle_completion_end_event(self, event_json): - # print("[pk] completion end") pass async def _report_assistant_response_started(self): + logger.debug("Assistant response started") + # Report that the assistant has started their response. - print("[pk] LLM full response started") await self.push_frame(LLMFullResponseStartFrame()) # Report that equivalent of TTS (this is a speech-to-speech model) started - print("[pk] TTS started") await self.push_frame(TTSStartedFrame()) async def _report_assistant_response_text_added(self, text): + logger.debug(f"Assistant response text added: {text}") + # Report some text added to the ongoing assistant response - print(f"[pk] LLM text: {text}") await self.push_frame(LLMTextFrame(text)) # Report some text added to the *equivalent* of TTS (this is a speech-to-speech model) - print(f"[pk] TTS text: {text}") await self.push_frame(TTSTextFrame(text)) # TODO: this is a (hopefully temporary) HACK. Here we directly manipulate the context rather @@ -890,19 +865,20 @@ class AWSNovaSonicLLMService(LLMService): self._context.buffer_assistant_text(text) async def _report_assistant_response_ended(self): + logger.debug("Assistant response ended") + # Report that the assistant has finished their response. - print("[pk] LLM full response ended") await self.push_frame(LLMFullResponseEndFrame()) # Report that equivalent of TTS (this is a speech-to-speech model) stopped. - print("[pk] TTS stopped") await self.push_frame(TTSStoppedFrame()) # For an explanation of this hack, see _report_assistant_response_text_added. self._context.flush_aggregated_assistant_text() async def _report_user_transcription_text_added(self, text): - print(f"[pk] transcription: {text}") + logger.debug(f"User transcription text added: {text}") + # Manually add new user transcription text to context. # We can't rely on the user context aggregator to do this since it's upstream from the LLM. self._context.add_user_transcription_text(text) @@ -960,6 +936,8 @@ class AWSNovaSonicLLMService(LLMService): self._triggering_assistant_response = False async def _send_assistant_response_trigger(self): + logger.debug("Sending assistant response trigger...") + chunk_duration = 0.02 # what we might get from InputAudioRawFrame chunk_size = int( chunk_duration @@ -980,6 +958,9 @@ class AWSNovaSonicLLMService(LLMService): else None ) if blank_audio_duration: + logger.debug( + f"Leading assistant response trigger with {blank_audio_duration}s of blank audio" + ) blank_audio_chunk = b"\x00" * chunk_size num_chunks = int(blank_audio_duration / chunk_duration) for _ in range(num_chunks): @@ -991,7 +972,6 @@ class AWSNovaSonicLLMService(LLMService): # if we ever need to seed this service again with context it would make sense to include it # since the instruction (i.e. the "wait for the trigger" instruction) will be part of the # context as well. - # print(f"[pk] sending trigger audio! {len(self._assistant_response_trigger_audio)}") audio_chunks = [ self._assistant_response_trigger_audio[i : i + chunk_size] for i in range(0, len(self._assistant_response_trigger_audio), chunk_size) diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index d96c2d1ed..a8d9c4dba 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -94,7 +94,6 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # Process remaining messages to fill out conversation history. # Nova Sonic supports "user" and "assistant" messages in history. - # print(f"[pk] standard messages: {messages}") for message in messages: history_message = self.from_standard_message(message) if history_message: @@ -136,11 +135,11 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): "content": [{"type": "text", "text": text}], } self.add_message(message) - # print(f"[pk] context updated (user): {self.get_messages_for_logging()}") + # logger.debug(f"Context updated (user): {self.get_messages_for_logging()}") def buffer_assistant_text(self, text): - self._assistant_text += text # TODO: determine if we need to add space or something - # print(f"[pk] assistant text buffered: {self._assistant_text}") + self._assistant_text += text + # logger.debug(f"Assistant text buffered: {self._assistant_text}") def flush_aggregated_assistant_text(self): if not self._assistant_text: From 35848d10b36e71dd5761bf627a1b7794a6a11f86 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 16:12:23 -0400 Subject: [PATCH 42/55] [WIP] AWS Nova Sonic service - remove various TODO comments --- src/pipecat/services/aws_nova_sonic/aws.py | 14 +------------- src/pipecat/services/aws_nova_sonic/context.py | 7 +------ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 1e318f164..cac3cd53f 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -132,7 +132,6 @@ class AWSNovaSonicLLMService(LLMService): def __init__( self, *, - # TODO: if we have instruction here as an alternative to using context, we should do the same for tools...right? secret_access_key: str, access_key_id: str, region: str, @@ -182,13 +181,6 @@ class AWSNovaSonicLLMService(LLMService): async def start(self, frame: StartFrame): await super().start(frame) self._wants_connection = True - # TODO: maybe connect but don't send history until we get all of our settings? - # how do we know how long to wait? - # ah, i think we'll *always* get at least one OpenAILLMContextFrame which kicks things off - # so we need to send the initial history when: - # - we're connected - # - we've gotten the first context - # i *think* this is what's controlled by _api_session_ready/_run_llm_when_api_session_ready await self._start_connecting() async def stop(self, frame: EndFrame): @@ -247,7 +239,6 @@ class AWSNovaSonicLLMService(LLMService): if self._triggering_assistant_response: return - # TODO: check if _audio_input_paused? what causes that? await self._send_user_audio_event(frame.audio) async def _handle_bot_stopped_speaking(self): @@ -417,9 +408,7 @@ class AWSNovaSonicLLMService(LLMService): region=self._region, aws_credentials_identity_resolver=StaticCredentialsResolver( credentials=AWSCredentialsIdentity( - access_key_id=self._access_key_id, - secret_access_key=self._secret_access_key, - # TODO: add additional stuff like aws_session_token + access_key_id=self._access_key_id, secret_access_key=self._secret_access_key ) ), http_auth_scheme_resolver=HTTPAuthSchemeResolver(), @@ -431,7 +420,6 @@ class AWSNovaSonicLLMService(LLMService): # LLM communication: input events (pipecat -> LLM) # - # TODO: make params configurable? async def _send_session_start_event(self): session_start = f""" {{ diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index a8d9c4dba..561ae53db 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -150,7 +150,7 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): } self._assistant_text = "" self.add_message(message) - # print(f"[pk] context updated (assistant): {self.get_messages_for_logging()}") + # logger.debug(f"Context updated (assistant): {self.get_messages_for_logging()}") @dataclass @@ -168,11 +168,6 @@ class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): if isinstance(frame, LLMMessagesUpdateFrame): await self.push_frame(AWSNovaSonicMessagesUpdateFrame(context=self._context)) - # Parent also doesn't push the LLMSetToolsFrame - # TODO: this - # if isinstance(frame, LLMSetToolsFrame): - # await self.push_frame(frame, direction) - class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): async def process_frame(self, frame: Frame, direction: FrameDirection): From 5579145a0630a10fefbb1d9f1b71dfa599c84d07 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 21:19:16 -0400 Subject: [PATCH 43/55] [WIP] AWS Nova Sonic service - post-rebase, update examples to play nicely with recent pipecat changes --- .../20e-persistent-context-aws-nova-sonic.py | 42 +++++++++---------- examples/foundational/39-aws-nova-sonic.py | 14 +++---- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index 8ac1508f8..e092730fb 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import argparse import asyncio import glob import json @@ -22,6 +23,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.aws_nova_sonic.aws import AWSNovaSonicLLMService +from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import TransportParams from pipecat.transports.network.small_webrtc import SmallWebRTCTransport from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection @@ -31,21 +33,19 @@ load_dotenv(override=True) BASE_FILENAME = "/tmp/pipecat_conversation_" -async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): - temperature = 75 if args["format"] == "fahrenheit" else 24 - await result_callback( +async def fetch_weather_from_api(params: FunctionCallParams): + temperature = 75 if params.arguments["format"] == "fahrenheit" else 24 + await params.result_callback( { "conditions": "nice", "temperature": temperature, - "format": args["format"], + "format": params.arguments["format"], "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), } ) -async def get_saved_conversation_filenames( - function_name, tool_call_id, args, llm, context, result_callback -): +async def get_saved_conversation_filenames(params: FunctionCallParams): # Construct the full pattern including the BASE_FILENAME full_pattern = f"{BASE_FILENAME}*.json" @@ -53,7 +53,7 @@ async def get_saved_conversation_filenames( matching_files = glob.glob(full_pattern) logger.debug(f"matching files: {matching_files}") - await result_callback({"filenames": matching_files}) + await params.result_callback({"filenames": matching_files}) # async def get_saved_conversation_filenames( @@ -69,26 +69,26 @@ async def get_saved_conversation_filenames( # await result_callback({"filenames": matching_files}) -async def save_conversation(function_name, tool_call_id, args, llm, context, result_callback): +async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" logger.debug( - f"writing conversation to {filename}\n{json.dumps(context.get_messages_for_persistent_storage(), indent=4)}" + f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" ) try: with open(filename, "w") as file: - messages = context.get_messages_for_persistent_storage() + messages = params.context.get_messages_for_persistent_storage() # remove the last message, which is the instruction we just gave to save the conversation messages.pop() json.dump(messages, file, indent=2) - await result_callback({"success": True}) + await params.result_callback({"success": True}) except Exception as e: - await result_callback({"success": False, "error": str(e)}) + await params.result_callback({"success": False, "error": str(e)}) -async def load_conversation(function_name, tool_call_id, args, llm, context, result_callback): +async def load_conversation(params: FunctionCallParams): async def _reset(): - filename = args["filename"] + filename = params.arguments["filename"] logger.debug(f"loading conversation from {filename}") try: with open(filename, "r") as file: @@ -99,11 +99,11 @@ async def load_conversation(function_name, tool_call_id, args, llm, context, res "content": f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}", } ) - context.set_messages(messages) - await llm.reset_conversation() - await llm.trigger_assistant_response() + params.context.set_messages(messages) + await params.llm.reset_conversation() + await params.llm.trigger_assistant_response() except Exception as e: - await result_callback({"success": False, "error": str(e)}) + await params.result_callback({"success": False, "error": str(e)}) asyncio.create_task(_reset()) @@ -161,7 +161,7 @@ tools = ToolsSchema( ) -async def run_bot(webrtc_connection: SmallWebRTCConnection): +async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): logger.info(f"Starting bot") transport = SmallWebRTCTransport( @@ -169,9 +169,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): params=TransportParams( audio_in_enabled=True, audio_out_enabled=True, - vad_enabled=True, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - vad_audio_passthrough=True, ), ) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index af44cf790..9decc47ae 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import argparse import os from datetime import datetime @@ -19,6 +20,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.aws_nova_sonic import AWSNovaSonicLLMService +from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import TransportParams from pipecat.transports.network.small_webrtc import SmallWebRTCTransport from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection @@ -27,13 +29,13 @@ from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection load_dotenv(override=True) -async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): - temperature = 75 if args["format"] == "fahrenheit" else 24 - await result_callback( +async def fetch_weather_from_api(params: FunctionCallParams): + temperature = 75 if params.arguments["format"] == "fahrenheit" else 24 + await params.result_callback( { "conditions": "nice", "temperature": temperature, - "format": args["format"], + "format": params.arguments["format"], "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), } ) @@ -60,7 +62,7 @@ weather_function = FunctionSchema( tools = ToolsSchema(standard_tools=[weather_function]) -async def run_bot(webrtc_connection: SmallWebRTCConnection): +async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): logger.info(f"Starting bot") # Initialize the SmallWebRTCTransport with the connection @@ -71,8 +73,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): audio_in_sample_rate=16000, audio_out_enabled=True, camera_in_enabled=False, - vad_enabled=True, - vad_audio_passthrough=True, vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), ), ) From 84736472694592220b393830fb4f61c06ffd84d0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 21:57:41 -0400 Subject: [PATCH 44/55] [WIP] AWS Nova Sonic service - update persistent-context example to better avoid saving "transitional", as opposed to meaningful, context messages --- .../20e-persistent-context-aws-nova-sonic.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index e092730fb..1519f1c53 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -72,15 +72,24 @@ async def get_saved_conversation_filenames(params: FunctionCallParams): async def save_conversation(params: FunctionCallParams): timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") filename = f"{BASE_FILENAME}{timestamp}.json" - logger.debug( - f"writing conversation to {filename}\n{json.dumps(params.context.get_messages_for_persistent_storage(), indent=4)}" - ) try: with open(filename, "w") as file: messages = params.context.get_messages_for_persistent_storage() - # remove the last message, which is the instruction we just gave to save the conversation - messages.pop() - json.dump(messages, file, indent=2) + # remove the last few messages. in reverse order, they are: + # - the in progress save tool call + # - the invocation of the save tool call + # - the user ask to save (which may encompass one or more messages) + # the simplest thing to do is to pop messages until the last one is an assistant + # response + while messages and not ( + messages[-1].get("role") == "assistant" and "content" in messages[-1] + ): + messages.pop() + if messages: # we never expect this to be empty + logger.debug( + f"writing conversation to {filename}\n{json.dumps(messages, indent=4)}" + ) + json.dump(messages, file, indent=2) await params.result_callback({"success": True}) except Exception as e: await params.result_callback({"success": False, "error": str(e)}) From ed06cdd2c7ea2a27e95b207cc5b72cdc625cf2bd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 22:03:25 -0400 Subject: [PATCH 45/55] [WIP] AWS Nova Sonic service - add CHANGELOG entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2eec61bca..bf6fec1b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for the AWS Nova Sonic speech-to-speech model with the new + `AWSNovaSonicLLMService`. + (see https://docs.aws.amazon.com/nova/latest/userguide/speech.html) + - Added new AWS services `AWSBedrockLLMService` and `AWSTranscribeSTTService`. - Added `on_active_speaker_changed` event handler to the `DailyTransport` class. From 896f8d85f70a43f6c20c74e77fe67abc48925967 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 6 May 2025 22:08:55 -0400 Subject: [PATCH 46/55] [WIP] AWS Nova Sonic service - remove out-of-date TODO comment --- examples/foundational/39-aws-nova-sonic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/foundational/39-aws-nova-sonic.py b/examples/foundational/39-aws-nova-sonic.py index 9decc47ae..4ed533e18 100644 --- a/examples/foundational/39-aws-nova-sonic.py +++ b/examples/foundational/39-aws-nova-sonic.py @@ -108,7 +108,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # Set up context and context management. # AWSNovaSonicService will adapt OpenAI LLM context objects with standard message format to # what's expected by Nova Sonic. - # TODO: since we can't trigger a response upon joining, this isn't particularly useful context = OpenAILLMContext( messages=[ {"role": "system", "content": f"{system_instruction}"}, From 27bff7a75963298bd2325520afbc50c9f4dddc26 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 09:11:27 -0400 Subject: [PATCH 47/55] [WIP] AWS Nova Sonic service - fix comment --- src/pipecat/adapters/services/aws_nova_sonic_adapter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index b96980046..dc7eef92d 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # import json -from typing import Any, Dict, List, Union +from typing import Any, Dict, List from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.function_schema import FunctionSchema @@ -31,9 +31,9 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter): } def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]: - """Converts function schemas to Openai Realtime function-calling format. + """Converts function schemas to AWS Nova Sonic function-calling format. - :return: Openai Realtime formatted function call definition. + :return: AWS Nova Sonic formatted function call definition. """ functions_schema = tools_schema.standard_tools From 4ba9a428610e074124499849fb0a8e22ccadd5d5 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 09:52:05 -0400 Subject: [PATCH 48/55] [WIP] AWS Nova Sonic service - add more accurate typing --- src/pipecat/services/aws_nova_sonic/aws.py | 74 ++++++++++++++-------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index cac3cd53f..a2ef78f88 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -148,30 +148,34 @@ class AWSNovaSonicLLMService(LLMService): self._access_key_id = access_key_id self._region = region self._model = model - self._client: BedrockRuntimeClient = None + self._client: Optional[BedrockRuntimeClient] = None self._voice_id = voice_id self._params = params self._system_instruction = system_instruction self._tools = tools self._send_transcription_frames = send_transcription_frames - self._context: AWSNovaSonicLLMContext = None - self._stream: DuplexEventStream[ - InvokeModelWithBidirectionalStreamInput, - InvokeModelWithBidirectionalStreamOutput, - InvokeModelWithBidirectionalStreamOperationOutput, + self._context: Optional[AWSNovaSonicLLMContext] = None + self._stream: Optional[ + DuplexEventStream[ + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamOutput, + InvokeModelWithBidirectionalStreamOperationOutput, + ] ] = None - self._receive_task = None - self._prompt_name = None - self._input_audio_content_name = None - self._content_being_received = None + self._receive_task: Optional[asyncio.Task] = None + self._prompt_name: Optional[str] = None + self._input_audio_content_name: Optional[str] = None + self._content_being_received: Optional[CurrentContent] = None self._assistant_is_responding = False self._context_available = False self._ready_to_send_context = False self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False - self._assistant_response_trigger_audio: bytes = None # Not cleared on _disconnect() + self._assistant_response_trigger_audio: Optional[bytes] = ( + None # Not cleared on _disconnect() + ) self._disconnecting = False - self._connected_time: float = None + self._connected_time: Optional[float] = None self._wants_connection = False # @@ -437,6 +441,9 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(session_start) async def _send_prompt_start_event(self, tools: List[Any]): + if not self._prompt_name: + return + tools_config = ( f""", "toolUseOutputConfiguration": {{ @@ -474,6 +481,9 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(prompt_start) async def _send_audio_input_start_event(self): + if not self._prompt_name: + return + audio_content_start = f''' {{ "event": {{ @@ -498,7 +508,7 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(audio_content_start) async def _send_text_event(self, text: str, role: Role): - if not self._stream or not text: + if not self._stream or not self._prompt_name or not text: return content_name = str(uuid.uuid4()) @@ -566,7 +576,7 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(audio_event) async def _send_session_end_events(self): - if not self._stream: + if not self._stream or not self._prompt_name: return prompt_end = f''' @@ -590,7 +600,7 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(session_end) async def _send_tool_result(self, tool_call_id, result): - if not self._stream: + if not self._stream or not self._prompt_name: return content_name = str(uuid.uuid4()) @@ -643,6 +653,9 @@ class AWSNovaSonicLLMService(LLMService): await self._send_client_event(result_content_end) async def _send_client_event(self, event_json: str): + if not self._stream: # should never happen + return + event = InvokeModelWithBidirectionalStreamInputChunk( value=BidirectionalInputPayloadPart(bytes_=event_json.encode("utf-8")) ) @@ -732,8 +745,7 @@ class AWSNovaSonicLLMService(LLMService): await self._report_assistant_response_started() async def _handle_text_output_event(self, event_json): - # This should never happen - if not self._content_being_received: + if not self._content_being_received: # should never happen return content = self._content_being_received @@ -744,8 +756,7 @@ class AWSNovaSonicLLMService(LLMService): content.text_content = text_content async def _handle_audio_output_event(self, event_json): - # This should never happen - if not self._content_being_received: + if not self._content_being_received: # should never happen return # Get audio @@ -761,8 +772,7 @@ class AWSNovaSonicLLMService(LLMService): await self.push_frame(frame) async def _handle_tool_use_event(self, event_json): - # This should never happen - if not self._content_being_received: + if not self._content_being_received or not self._context: # should never happen return # Get tool use details @@ -793,8 +803,7 @@ class AWSNovaSonicLLMService(LLMService): ) async def _handle_content_end_event(self, event_json): - # This should never happen - if not self._content_being_received: + if not self._content_being_received: # should never happen return content = self._content_being_received @@ -817,8 +826,6 @@ class AWSNovaSonicLLMService(LLMService): # User transcription text added await self._report_user_transcription_text_added(content.text_content) - self._content_being_received = False - async def _handle_completion_end_event(self, event_json): pass @@ -832,6 +839,9 @@ class AWSNovaSonicLLMService(LLMService): await self.push_frame(TTSStartedFrame()) async def _report_assistant_response_text_added(self, text): + if not self._context: # should never happen + return + logger.debug(f"Assistant response text added: {text}") # Report some text added to the ongoing assistant response @@ -853,6 +863,9 @@ class AWSNovaSonicLLMService(LLMService): self._context.buffer_assistant_text(text) async def _report_assistant_response_ended(self): + if not self._context: # should never happen + return + logger.debug("Assistant response ended") # Report that the assistant has finished their response. @@ -865,6 +878,9 @@ class AWSNovaSonicLLMService(LLMService): self._context.flush_aggregated_assistant_text() async def _report_user_transcription_text_added(self, text): + if not self._context: # should never happen + return + logger.debug(f"User transcription text added: {text}") # Manually add new user transcription text to context. @@ -918,12 +934,16 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes()) # Send the trigger audio, if we're fully connected and set up - # NOTE: maybe there's a better way to determine whether we're done setting up? - if self._receive_task: + if self._connected_time is not None: await self._send_assistant_response_trigger() self._triggering_assistant_response = False async def _send_assistant_response_trigger(self): + if ( + not self._assistant_response_trigger_audio or self._connected_time is None + ): # should never happen + return + logger.debug("Sending assistant response trigger...") chunk_duration = 0.02 # what we might get from InputAudioRawFrame From 52036138c1ff2365f17c4b03df89e711b1aeb51a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 10:22:51 -0400 Subject: [PATCH 49/55] [WIP] AWS Nova Sonic service - remove unnecessary (no-op) code --- src/pipecat/services/aws_nova_sonic/aws.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index a2ef78f88..6838daad6 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -269,7 +269,6 @@ class AWSNovaSonicLLMService(LLMService): await asyncio.sleep(0.25) self._assistant_is_responding = False await self._report_assistant_response_ended() - self._handling_bot_stopped_speaking = False self._handling_bot_stopped_speaking = False From b013e375fb90f0810e1c1b367b5834e5f70d8fdc Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 10:38:23 -0400 Subject: [PATCH 50/55] [WIP] AWS Nova Sonic service - simplify a bit of logic (and do the same simplification in the OpenAI Realtime service) --- src/pipecat/services/aws_nova_sonic/aws.py | 9 +-------- src/pipecat/services/openai_realtime_beta/openai.py | 10 +--------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 6838daad6..3f41a0166 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -782,14 +782,7 @@ class AWSNovaSonicLLMService(LLMService): # Call tool function if self.has_function(function_name): - if function_name in self._functions.keys(): - await self.call_function( - context=self._context, - tool_call_id=tool_call_id, - function_name=function_name, - arguments=arguments, - ) - elif None in self._functions.keys(): + if function_name in self._functions.keys() or None in self._functions.keys(): await self.call_function( context=self._context, tool_call_id=tool_call_id, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 334ce98c8..0c37f73ce 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -577,15 +577,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): arguments = json.loads(item.arguments) if self.has_function(function_name): run_llm = index == total_items - 1 - if function_name in self._functions.keys(): - await self.call_function( - context=self._context, - tool_call_id=tool_id, - function_name=function_name, - arguments=arguments, - run_llm=run_llm, - ) - elif None in self._functions.keys(): + if function_name in self._functions.keys() or None in self._functions.keys(): await self.call_function( context=self._context, tool_call_id=tool_id, From c78f7798004a505522765c00b3e043ffbaa6784d Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 10:55:43 -0400 Subject: [PATCH 51/55] [WIP] AWS Nova Sonic service - log an error message if you try to use AWS Nova Sonic without the proper dependency (e.g. without having done `pip install pipecat-ai[aws]`) --- src/pipecat/services/aws_nova_sonic/aws.py | 40 ++++++++++++---------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 3f41a0166..b4989185a 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -15,23 +15,8 @@ from enum import Enum from importlib.resources import files from typing import Any, List, Optional -from aws_sdk_bedrock_runtime.client import ( - BedrockRuntimeClient, - InvokeModelWithBidirectionalStreamOperationInput, -) -from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme -from aws_sdk_bedrock_runtime.models import ( - BidirectionalInputPayloadPart, - InvokeModelWithBidirectionalStreamInput, - InvokeModelWithBidirectionalStreamInputChunk, - InvokeModelWithBidirectionalStreamOperationOutput, - InvokeModelWithBidirectionalStreamOutput, -) from loguru import logger from pydantic import BaseModel, Field -from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver -from smithy_aws_core.identity import AWSCredentialsIdentity -from smithy_core.aio.eventstream import DuplexEventStream from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter @@ -45,15 +30,11 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, StartFrame, - StartInterruptionFrame, - StopInterruptionFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, ) from pipecat.processors.aggregators.llm_response import ( LLMAssistantAggregatorParams, @@ -75,6 +56,27 @@ from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResul from pipecat.services.llm_service import LLMService from pipecat.utils.time import time_now_iso8601 +try: + from aws_sdk_bedrock_runtime.client import ( + BedrockRuntimeClient, + InvokeModelWithBidirectionalStreamOperationInput, + ) + from aws_sdk_bedrock_runtime.config import Config, HTTPAuthSchemeResolver, SigV4AuthScheme + from aws_sdk_bedrock_runtime.models import ( + BidirectionalInputPayloadPart, + InvokeModelWithBidirectionalStreamInput, + InvokeModelWithBidirectionalStreamInputChunk, + InvokeModelWithBidirectionalStreamOperationOutput, + InvokeModelWithBidirectionalStreamOutput, + ) + from smithy_aws_core.credentials_resolvers.static import StaticCredentialsResolver + from smithy_aws_core.identity import AWSCredentialsIdentity + from smithy_core.aio.eventstream import DuplexEventStream +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use AWS services, you need to `pip install pipecat-ai[aws]`.") + raise Exception(f"Missing module: {e}") + class AWSNovaSonicUnhandledFunctionException(Exception): pass From 1491462d157509eb336d5111a4f4a8f875e2cd90 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 11:10:54 -0400 Subject: [PATCH 52/55] [WIP] AWS Nova Sonic service - remove `_handling_bot_stopped_speaking`, which no longer seems to be necessary; I'm no longer observing back-to-back `BotStoppedSpeaking` frames --- src/pipecat/services/aws_nova_sonic/aws.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index b4989185a..9f2f1f72e 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -171,7 +171,6 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_is_responding = False self._context_available = False self._ready_to_send_context = False - self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False self._assistant_response_trigger_audio: Optional[bytes] = ( None # Not cleared on _disconnect() @@ -248,10 +247,6 @@ class AWSNovaSonicLLMService(LLMService): await self._send_user_audio_event(frame.audio) async def _handle_bot_stopped_speaking(self): - # Protect against back-to-back BotStoppedSpeaking calls, which I've observed - if self._handling_bot_stopped_speaking: - return - self._handling_bot_stopped_speaking = True if self._assistant_is_responding: # Consider the assistant finished with their response (after a short delay, to allow for @@ -272,8 +267,6 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_is_responding = False await self._report_assistant_response_ended() - self._handling_bot_stopped_speaking = False - async def _handle_function_call_result(self, frame: AWSNovaSonicFunctionCallResultFrame): result = frame.result_frame await self._send_tool_result(tool_call_id=result.tool_call_id, result=result.result) @@ -398,7 +391,6 @@ class AWSNovaSonicLLMService(LLMService): self._assistant_is_responding = False self._context_available = False self._ready_to_send_context = False - self._handling_bot_stopped_speaking = False self._triggering_assistant_response = False self._disconnecting = False self._connected_time = None From b53f9235e4bd9f9fefe6d0e0b6761e972a3d8bf0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 11:42:37 -0400 Subject: [PATCH 53/55] [WIP] AWS Nova Sonic service - remove unnecessary `_context_available` state, instead just relying on the presence of `_context` --- src/pipecat/services/aws_nova_sonic/aws.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 9f2f1f72e..4056d0ed0 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -169,7 +169,6 @@ class AWSNovaSonicLLMService(LLMService): self._input_audio_content_name: Optional[str] = None self._content_being_received: Optional[CurrentContent] = None self._assistant_is_responding = False - self._context_available = False self._ready_to_send_context = False self._triggering_assistant_response = False self._assistant_response_trigger_audio: Optional[bytes] = ( @@ -205,11 +204,13 @@ class AWSNovaSonicLLMService(LLMService): async def reset_conversation(self): logger.debug("Resetting conversation") await self._handle_bot_stopped_speaking() + + # Carry over previous context through disconnect + context = self._context await self._disconnect() + self._context = context + await self._start_connecting() - # Use existing context - self._context_available = True - await self._finish_connecting_if_context_available() # # frame processing @@ -235,7 +236,6 @@ class AWSNovaSonicLLMService(LLMService): self._context = AWSNovaSonicLLMContext.upgrade_to_nova_sonic( context, self._system_instruction ) - self._context_available = True await self._finish_connecting_if_context_available() async def _handle_input_audio_frame(self, frame: InputAudioRawFrame): @@ -247,7 +247,6 @@ class AWSNovaSonicLLMService(LLMService): await self._send_user_audio_event(frame.audio) async def _handle_bot_stopped_speaking(self): - if self._assistant_is_responding: # Consider the assistant finished with their response (after a short delay, to allow for # any FINAL text block to come in). @@ -308,7 +307,7 @@ class AWSNovaSonicLLMService(LLMService): async def _finish_connecting_if_context_available(self): # We can only finish connecting once we've gotten our initial context and we're ready to # send it - if not (self._context_available and self._ready_to_send_context): + if not (self._context and self._ready_to_send_context): return logger.info("Finishing connecting (setting up session)...") @@ -389,7 +388,6 @@ class AWSNovaSonicLLMService(LLMService): self._input_audio_content_name = None self._content_being_received = None self._assistant_is_responding = False - self._context_available = False self._ready_to_send_context = False self._triggering_assistant_response = False self._disconnecting = False From 93c9cc4a0e7dbd27f9d5adcbcb76215b35184bbf Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 12:03:23 -0400 Subject: [PATCH 54/55] [WIP] AWS Nova Sonic service - minor fix --- src/pipecat/services/aws_nova_sonic/aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 4056d0ed0..eab12272c 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -669,7 +669,7 @@ class AWSNovaSonicLLMService(LLMService): # The overall completion is wrapped by "completionStart" and "completionEnd" events. async def _receive_task_handler(self): try: - while self._client and not self._disconnecting: + while self._stream and not self._disconnecting: output = await self._stream.await_output() result = await output[1].receive() From 2920aa5af477720227666d93e057d7b25424b36b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 7 May 2025 14:32:32 -0400 Subject: [PATCH 55/55] [WIP] AWS Nova Sonic service - pull AWS Nova Sonic support out of the `aws` optional dependency in pyproject.toml and into its own `aws-nova-sonic` optional dependency. That's because it requires Python >= 3.12, a higher version than the base project's 3.10. This change allows anyone using any of the other AWS services (including our own unit tests) to continue using the lower Python version. --- CHANGELOG.md | 3 ++- pyproject.toml | 3 ++- src/pipecat/services/aws_nova_sonic/aws.py | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6fec1b2..319dce632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for the AWS Nova Sonic speech-to-speech model with the new `AWSNovaSonicLLMService`. - (see https://docs.aws.amazon.com/nova/latest/userguide/speech.html) + See https://docs.aws.amazon.com/nova/latest/userguide/speech.html. + Note that it requires Python >= 3.12 and `pip install pipecat-ai[aws-nova-sonic]`. - Added new AWS services `AWSBedrockLLMService` and `AWSTranscribeSTTService`. diff --git a/pyproject.toml b/pyproject.toml index 7ce167d77..06d7fb0a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,8 @@ Website = "https://pipecat.ai" [project.optional-dependencies] anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "assemblyai~=0.37.0" ] -aws = [ "boto3~=1.37.16", "websockets~=13.1", "aws_sdk_bedrock_runtime~=0.0.2" ] +aws = [ "boto3~=1.37.16", "websockets~=13.1" ] +aws-nova-sonic = [ "aws_sdk_bedrock_runtime~=0.0.2" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] cerebras = [] diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index eab12272c..b53578f5a 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -74,7 +74,9 @@ try: from smithy_core.aio.eventstream import DuplexEventStream except ModuleNotFoundError as e: logger.error(f"Exception: {e}") - logger.error("In order to use AWS services, you need to `pip install pipecat-ai[aws]`.") + logger.error( + "In order to use AWS services, you need to `pip install pipecat-ai[aws-nova-sonic]`." + ) raise Exception(f"Missing module: {e}")