From 0f9e69d3c7520cdb024111d636c76ed8f13e5deb Mon Sep 17 00:00:00 2001 From: antonyesk601 Date: Thu, 17 Jul 2025 11:50:39 +0000 Subject: [PATCH 1/5] feat: Add Simli Trinity models support to pipecat --- examples/foundational/27-simli-layer.py | 2 +- src/pipecat/services/simli/video.py | 81 +++++++++++++++++++------ 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 40f6b0c3d..a8fdf7c3b 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -61,7 +61,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si ) simli_ai = SimliVideoService( - SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")) + SimliConfig(os.getenv("SIMLI_API_KEY"), os.getenv("SIMLI_FACE_ID")), ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index 6ba7da4b8..41d1c47b9 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -18,6 +18,8 @@ from pipecat.frames.frames import ( OutputImageRawFrame, StartInterruptionFrame, TTSAudioRawFrame, + TTSStoppedFrame, + UserStartedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator @@ -45,24 +47,39 @@ class SimliVideoService(FrameProcessor): simli_config: SimliConfig, use_turn_server: bool = False, latency_interval: int = 0, + simli_url: str = "https://api.simli.ai", + isTrinityAvatar: bool = False, ): """Initialize the Simli video service. Args: simli_config: Configuration object for Simli client settings. use_turn_server: Whether to use TURN server for connection. Defaults to False. - latency_interval: Latency interval setting for video processing. Defaults to 0. + latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. + simli_url: URL of the simli servers. Can be changed for custom deployments of enterprise users. + """ super().__init__() - self._simli_client = SimliClient(simli_config, use_turn_server, latency_interval) + self._initialized = False + simli_config.maxIdleTime += 5 + simli_config.maxSessionLength += 5 + self._simli_client = SimliClient( + simli_config, + use_turn_server, + latency_interval, + simliURL=simli_url, + ) - self._pipecat_resampler_event = asyncio.Event() self._pipecat_resampler: AudioResampler = None + self._pipecat_resampler_event = asyncio.Event() self._simli_resampler = AudioResampler("s16", "mono", 16000) - self._initialized = False self._audio_task: asyncio.Task = None self._video_task: asyncio.Task = None + self._started = False + self._is_gs_avatar = isTrinityAvatar + self._previously_interrupted = True + self._audio_buffer = bytearray() async def _start_connection(self): """Start the connection to Simli service and begin processing tasks.""" @@ -71,11 +88,9 @@ class SimliVideoService(FrameProcessor): self._initialized = True # Create task to consume and process audio and video - if not self._audio_task: - self._audio_task = self.create_task(self._consume_and_process_audio()) - - if not self._video_task: - self._video_task = self.create_task(self._consume_and_process_video()) + await self._simli_client.sendSilence() + self._audio_task = self.create_task(self._consume_and_process_audio()) + self._video_task = self.create_task(self._consume_and_process_video()) async def _consume_and_process_audio(self): """Consume audio frames from Simli and push them downstream.""" @@ -118,8 +133,8 @@ class SimliVideoService(FrameProcessor): """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): - await self.push_frame(frame, direction) await self._start_connection() + await self.push_frame(frame, direction) elif isinstance(frame, TTSAudioRawFrame): # Send audio frame to Simli try: @@ -137,19 +152,47 @@ class SimliVideoService(FrameProcessor): resampled_frames = self._simli_resampler.resample(old_frame) for resampled_frame in resampled_frames: - await self._simli_client.send( - resampled_frame.to_ndarray().astype(np.int16).tobytes() - ) + audioBytes = resampled_frame.to_ndarray().astype(np.int16).tobytes() + if self._previously_interrupted and self._is_gs_avatar: + self._audio_buffer.extend(audioBytes) + if len(self._audio_buffer) >= 128000: + try: + for flushFrame in self._simli_resampler.resample(None): + self._audio_buffer.extend( + flushFrame.to_ndarray().astype(np.int16).tobytes() + ) + finally: + await self._simli_client.playImmediate(self._audio_buffer) + self._previously_interrupted = False + self._audio_buffer = bytearray() + else: + await self._simli_client.send(audioBytes) + return except Exception as e: logger.exception(f"{self} exception: {e}") + elif isinstance(frame, TTSStoppedFrame): + try: + if self._previously_interrupted and len(self._audio_buffer) > 0: + await self._simli_client.playImmediate(self._audio_buffer) + self._previously_interrupted = False + self._audio_buffer = bytearray() + + except Exception as e: + logger.exception(f"{self} exception: {e}") + return + if isinstance(frame, StartFrame): + if not self._preinitialize: + await self._start_connection() + self._started = True + await self._simli_client.send((0).to_bytes(1, "big") * 6000) elif isinstance(frame, (EndFrame, CancelFrame)): await self._stop() - await self.push_frame(frame, direction) - elif isinstance(frame, StartInterruptionFrame): - await self._simli_client.clearBuffer() - await self.push_frame(frame, direction) - else: - await self.push_frame(frame, direction) + elif isinstance(frame, (StartInterruptionFrame, UserStartedSpeakingFrame)): + if not self._previously_interrupted: + await self._simli_client.clearBuffer() + self._previously_interrupted = True and self._is_gs_avatar + + await self.push_frame(frame, direction) async def _stop(self): """Stop the Simli client and cancel processing tasks.""" From 688031efd655057979661e68a833241ee16b6604 Mon Sep 17 00:00:00 2001 From: antonyesk601 Date: Fri, 18 Jul 2025 08:23:04 +0000 Subject: [PATCH 2/5] fix: use undeclared variable _preinitialized. fix: double send of start frame --- src/pipecat/services/simli/video.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index 41d1c47b9..a02612e34 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -48,7 +48,7 @@ class SimliVideoService(FrameProcessor): use_turn_server: bool = False, latency_interval: int = 0, simli_url: str = "https://api.simli.ai", - isTrinityAvatar: bool = False, + is_trinity_avatar: bool = False, ): """Initialize the Simli video service. @@ -57,6 +57,7 @@ class SimliVideoService(FrameProcessor): use_turn_server: Whether to use TURN server for connection. Defaults to False. latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. simli_url: URL of the simli servers. Can be changed for custom deployments of enterprise users. + is_trinity_avatar: boolean to tell simli client that this is a Trinity avatar which reduces latency when using Trinity. """ super().__init__() @@ -77,7 +78,7 @@ class SimliVideoService(FrameProcessor): self._audio_task: asyncio.Task = None self._video_task: asyncio.Task = None self._started = False - self._is_gs_avatar = isTrinityAvatar + self._is_trinity_avatar = is_trinity_avatar self._previously_interrupted = True self._audio_buffer = bytearray() @@ -134,7 +135,6 @@ class SimliVideoService(FrameProcessor): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): await self._start_connection() - await self.push_frame(frame, direction) elif isinstance(frame, TTSAudioRawFrame): # Send audio frame to Simli try: @@ -153,7 +153,7 @@ class SimliVideoService(FrameProcessor): resampled_frames = self._simli_resampler.resample(old_frame) for resampled_frame in resampled_frames: audioBytes = resampled_frame.to_ndarray().astype(np.int16).tobytes() - if self._previously_interrupted and self._is_gs_avatar: + if self._previously_interrupted and self._is_trinity_avatar: self._audio_buffer.extend(audioBytes) if len(self._audio_buffer) >= 128000: try: @@ -180,9 +180,6 @@ class SimliVideoService(FrameProcessor): except Exception as e: logger.exception(f"{self} exception: {e}") return - if isinstance(frame, StartFrame): - if not self._preinitialize: - await self._start_connection() self._started = True await self._simli_client.send((0).to_bytes(1, "big") * 6000) elif isinstance(frame, (EndFrame, CancelFrame)): @@ -190,7 +187,7 @@ class SimliVideoService(FrameProcessor): elif isinstance(frame, (StartInterruptionFrame, UserStartedSpeakingFrame)): if not self._previously_interrupted: await self._simli_client.clearBuffer() - self._previously_interrupted = True and self._is_gs_avatar + self._previously_interrupted = self._is_trinity_avatar await self.push_frame(frame, direction) From 1cbf7ae48054ecc974f91d5f9f50598e39cdb74d Mon Sep 17 00:00:00 2001 From: antonyesk601 Date: Wed, 23 Jul 2025 08:26:44 +0000 Subject: [PATCH 3/5] fix: remove unused variable; fix: remove redundant logic --- src/pipecat/services/simli/video.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index a02612e34..d398a87f3 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -77,9 +77,8 @@ class SimliVideoService(FrameProcessor): self._audio_task: asyncio.Task = None self._video_task: asyncio.Task = None - self._started = False self._is_trinity_avatar = is_trinity_avatar - self._previously_interrupted = True + self._previously_interrupted = is_trinity_avatar self._audio_buffer = bytearray() async def _start_connection(self): @@ -153,7 +152,7 @@ class SimliVideoService(FrameProcessor): resampled_frames = self._simli_resampler.resample(old_frame) for resampled_frame in resampled_frames: audioBytes = resampled_frame.to_ndarray().astype(np.int16).tobytes() - if self._previously_interrupted and self._is_trinity_avatar: + if self._previously_interrupted: self._audio_buffer.extend(audioBytes) if len(self._audio_buffer) >= 128000: try: @@ -176,12 +175,9 @@ class SimliVideoService(FrameProcessor): await self._simli_client.playImmediate(self._audio_buffer) self._previously_interrupted = False self._audio_buffer = bytearray() - except Exception as e: logger.exception(f"{self} exception: {e}") return - self._started = True - await self._simli_client.send((0).to_bytes(1, "big") * 6000) elif isinstance(frame, (EndFrame, CancelFrame)): await self._stop() elif isinstance(frame, (StartInterruptionFrame, UserStartedSpeakingFrame)): From 72168070f1b6048c2a84a72e8806d5873c7ff37e Mon Sep 17 00:00:00 2001 From: antonyesk601 Date: Tue, 5 Aug 2025 14:18:41 +0000 Subject: [PATCH 4/5] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c15476b8..98ca322d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 pushed by the `BaseInputTransport` at Start and any time a `VADParamsUpdateFrame` is received. +- Add support for Simli Trinity Avatars, a new `is_trinity_avatar` parameter is added to indicate whether the Simli `faceId` is a trinity faceId or not which is essential for getting a consistent experience with trinity faces. + ### Changed - Two package dependencies have been updated: From fb82dc8308f7b5c198f5ac7956e9d0ac076d8429 Mon Sep 17 00:00:00 2001 From: Antonyesk601 <40663824+Antonyesk601@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:46:01 +0200 Subject: [PATCH 5/5] Update CHANGELOG.md Co-authored-by: Mark Backman --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98ca322d2..2c339c9ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 pushed by the `BaseInputTransport` at Start and any time a `VADParamsUpdateFrame` is received. -- Add support for Simli Trinity Avatars, a new `is_trinity_avatar` parameter is added to indicate whether the Simli `faceId` is a trinity faceId or not which is essential for getting a consistent experience with trinity faces. +- Added support for Simli Trinity Avatars. A new `is_trinity_avatar` parameter + has been introduced to specify whether the provided `faceId` corresponds to a + Trinity avatar, which is required for optimal Trinity avatar performance. ### Changed