From ceaf53fdb0cd9f9f1fe7c58c2ae62e454b8915fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 12:08:09 -0800 Subject: [PATCH 1/4] LLMContext: async create_image_message/create_audio_message fixes --- CHANGELOG.md | 7 ++++--- .../22d-natural-conversation-gemini-audio.py | 2 +- src/pipecat/processors/aggregators/llm_context.py | 14 ++++++++------ .../aggregators/llm_response_universal.py | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47be7d2bf..5ace3aa14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,9 +26,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- ⚠️ Breaking change: `LLMContext.create_image_message()` and - `LLMContext.create_audio_message()` are now async methods. This fixes and - issue where the asyncio event loop would be blocked while encoding audio or +- ⚠️ Breaking change: `LLMContext.create_image_message()`, + `LLMContext.create_audio_message()`, `LLMContext.add_image_frame_message()` + and `LLMContext.add_audio_frames_message()` are now async methods. This fixes + an issue where the asyncio event loop would be blocked while encoding audio or images. - `ConsumerProcessor` now queues frames from the producer internally instead of diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 7a7155297..a7837ce60 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -391,7 +391,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = LLMContext() - context.add_audio_frames_message(audio_frames=self._audio_frames) + await context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(LLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index b9216103a..99b9aeaa9 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -180,7 +180,7 @@ class LLMContext: text: Optional text to include with the audio. """ - def encode_audio(): + async def encode_audio(): sample_rate = audio_frames[0].sample_rate num_channels = audio_frames[0].num_channels @@ -198,7 +198,7 @@ class LLMContext: encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") return encoded_audio - encoded_audio = asyncio.to_thread(encode_audio) + encoded_audio = await asyncio.to_thread(encode_audio) content.append( { @@ -333,7 +333,7 @@ class LLMContext: """ self._tool_choice = tool_choice - def add_image_frame_message( + async def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None ): """Add a message containing an image frame. @@ -344,10 +344,12 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - message = LLMContext.create_image_message(format=format, size=size, image=image, text=text) + message = await LLMContext.create_image_message( + format=format, size=size, image=image, text=text + ) self.add_message(message) - def add_audio_frames_message( + async def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" ): """Add a message containing audio frames. @@ -356,7 +358,7 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text) + message = await LLMContext.create_audio_message(audio_frames=audio_frames, text=text) self.add_message(message) @staticmethod diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index d0ac67257..87974bed2 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -793,7 +793,7 @@ class LLMAssistantAggregator(LLMContextAggregator): logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})") - self._context.add_image_frame_message( + await self._context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, From 39b4e6183765b8d59d34d0d93900d4b7057e9b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 13:50:47 -0800 Subject: [PATCH 2/4] SimliVideoService: fix connection issue --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- src/pipecat/services/simli/video.py | 16 +++++++++++++--- uv.lock | 9 +++++---- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ace3aa14..ff3934b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `SimliVideoService` connection issue. + - Fixed an issue in the `Runner` where, when using `SmallWebRTCTransport`, the `request_data` was not being passed to the `SmallWebRTCRunnerArguments` body. diff --git a/pyproject.toml b/pyproject.toml index e313c789d..9354c2066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ local-smart-turn = [ "coremltools>=8.0", "transformers", "torch>=2.5.0,<3", "tor local-smart-turn-v3 = [ "transformers", "onnxruntime>=1.20.1,<2" ] remote-smart-turn = [] silero = [ "onnxruntime>=1.20.1,<2" ] -simli = [ "simli-ai~=0.1.25"] +simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] speechmatics = [ "speechmatics-rt>=0.5.0" ] diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index bac54f35b..e6514ca7b 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -84,6 +84,10 @@ class SimliVideoService(FrameProcessor): Please use 'api_key' and 'face_id' parameters instead. use_turn_server: Whether to use TURN server for connection. Defaults to False. + + .. deprecated:: 0.0.95 + The 'use_turn_server' parameter is deprecated and will be removed in a future version. + latency_interval: Latency interval setting for sending health checks to check the latency to Simli Servers. Defaults to 0. simli_url: URL of the simli servers. Can be changed for custom deployments @@ -135,14 +139,20 @@ class SimliVideoService(FrameProcessor): config = SimliConfig(**config_kwargs) + if use_turn_server: + warnings.warn( + "The 'use_turn_server' parameter is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + self._initialized = False # Add buffer time to session limits config.maxIdleTime += 5 config.maxSessionLength += 5 self._simli_client = SimliClient( - config, - use_turn_server, - latency_interval, + config=config, + latencyInterval=latency_interval, simliURL=simli_url, ) diff --git a/uv.lock b/uv.lock index eabe03714..6dec2d8dc 100644 --- a/uv.lock +++ b/uv.lock @@ -4727,7 +4727,7 @@ requires-dist = [ { name = "resampy", specifier = "~=0.4.3" }, { name = "sarvamai", marker = "extra == 'sarvam'", specifier = "==0.1.21" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.28.0,<3" }, - { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=0.1.25" }, + { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, { name = "speechmatics-rt", marker = "extra == 'speechmatics'", specifier = ">=0.5.0" }, @@ -6496,18 +6496,19 @@ wheels = [ [[package]] name = "simli-ai" -version = "0.1.25" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiortc" }, { name = "av" }, { name = "httpx" }, + { name = "livekit" }, { name = "numpy" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/6a/b28f90baf76f6a60865985f6233ff44abc72d45b66b76658bff3961e20a7/simli_ai-0.1.25.tar.gz", hash = "sha256:7a00b3426dc26a6a421641072c3e49014b7950c621cf4544152f35c58d13fcff", size = 13182, upload-time = "2025-11-06T16:27:08.862Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/03/b0b3e12c68fd3f9c57f6afeee67841349e4866b88760f413357af3043ae4/simli_ai-1.0.3.tar.gz", hash = "sha256:e96b0621a1dbd9582b2ae3d51eefd4995983b49c1f1061eb9239707b15a1ee27", size = 13350, upload-time = "2025-11-13T12:22:32.514Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/57/ae1032fd88214ea4ee6d3028c817c12a999eb90a67766bbab31e9819385a/simli_ai-0.1.25-py3-none-any.whl", hash = "sha256:7d01f65321dc9052f25e15d0463af6a20a86c6d37d9a7b3a2c4b01cbec0a54ed", size = 13651, upload-time = "2025-11-06T16:27:07.765Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d1/dc382ba529de0d2d51f35e9bfd20b41d8f5c96404a3aa24bae97a5a5e51f/simli_ai-1.0.3-py3-none-any.whl", hash = "sha256:ffafa7540aa28833e207be8f3b199367c7f500dac1a8ba0108395bfb7d8362bc", size = 13863, upload-time = "2025-11-13T12:22:31.218Z" }, ] [[package]] From 51ba245e1030af0f7b7366f5d804fff3d6165c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 14:13:34 -0800 Subject: [PATCH 3/4] scripts(evals): fix EVAL_CONVERSATION/EVAL_WEATHER eval --- scripts/evals/run-release-evals.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index da6df053d..4f8268d78 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -30,8 +30,8 @@ EVAL_SIMPLE_MATH = EvalConfig( ) EVAL_WEATHER = EvalConfig( - prompt="What's the weather in San Francisco?", - eval="The user says something specific about the current weather in San Francisco, including the degrees.", + prompt="What's the weather in San Francisco (in farhenheit or celsius)?", + eval="The user says something specific about the current weather in San Francisco, including the degrees (in farhenheit or celsius).", ) EVAL_ONLINE_SEARCH = EvalConfig( @@ -70,7 +70,7 @@ EVAL_VOICEMAIL = EvalConfig( EVAL_CONVERSATION = EvalConfig( prompt="Hello, this is Mark.", - eval="The user replies with a greeting.", + eval="The user acknowledges the greeting.", eval_speaks_first=True, ) From cf1a9c1548d46b616a071bedcf3991ee8bd04ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Nov 2025 10:54:44 -0800 Subject: [PATCH 4/4] update CHANGELOG for 0.0.95 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff3934b2e..d69eb897a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.95] - 2025-11-18 ### Added