From 9acc36c58ea66c12b44ce9558d65b89fccbe7c8b Mon Sep 17 00:00:00 2001 From: Matej Marinko Date: Thu, 16 Oct 2025 08:51:40 +0200 Subject: [PATCH 01/20] Update model params for Soniox STT - remove deprecated parameters and add new ones - add support for v3 context --- src/pipecat/services/soniox/stt.py | 49 ++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index 1cf2d5194..1447774e1 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -49,6 +49,33 @@ END_TOKEN = "" FINALIZED_TOKEN = "" +class SonioxContextGeneralItem(BaseModel): + """Represents a key-value pair for structured general context information.""" + + key: str + value: str + + +class SonioxContextTranslationTerm(BaseModel): + """Represents a custom translation mapping for ambiguous or domain-specific terms.""" + + source: str + target: str + + +class SonioxContextObject(BaseModel): + """Context object for models with context_version 2, for Soniox stt-rt-v3-preview and higher. + + Learn more about context in the documentation: + https://soniox.com/docs/stt/concepts/context + """ + + general: Optional[List[SonioxContextGeneralItem]] = None + text: Optional[str] = None + terms: Optional[List[str]] = None + translation_terms: Optional[List[SonioxContextTranslationTerm]] = None + + class SonioxInputParams(BaseModel): """Real-time transcription settings. @@ -60,9 +87,9 @@ class SonioxInputParams(BaseModel): audio_format: Audio format to use for transcription. num_channels: Number of channels to use for transcription. language_hints: List of language hints to use for transcription. - context: Customization for transcription. - enable_non_final_tokens: Whether to enable non-final tokens. If false, only final tokens will be returned. - max_non_final_tokens_duration_ms: Maximum duration of non-final tokens. + context: Customization for transcription. String for models with context_version 1 and ContextObject for models with context_version 2. + enable_speaker_diarization: Whether to enable speaker diarization. Tokens are annotated with speaker IDs. + enable_language_identification: Whether to enable language identification. Tokens are annotated with language IDs. client_reference_id: Client reference ID to use for transcription. """ @@ -72,10 +99,10 @@ class SonioxInputParams(BaseModel): num_channels: Optional[int] = 1 language_hints: Optional[List[Language]] = None - context: Optional[str] = None + context: Optional[SonioxContextObject | str] = None - enable_non_final_tokens: Optional[bool] = True - max_non_final_tokens_duration_ms: Optional[int] = None + enable_speaker_diarization: Optional[bool] = False + enable_language_identification: Optional[bool] = False client_reference_id: Optional[str] = None @@ -173,6 +200,10 @@ class SonioxSTTService(STTService): # Either one or the other is required. enable_endpoint_detection = not self._vad_force_turn_endpoint + context = self._params.context + if isinstance(context, SonioxContextObject): + context = context.model_dump() + # Send the initial configuration message. config = { "api_key": self._api_key, @@ -182,9 +213,9 @@ class SonioxSTTService(STTService): "enable_endpoint_detection": enable_endpoint_detection, "sample_rate": self.sample_rate, "language_hints": _prepare_language_hints(self._params.language_hints), - "context": self._params.context, - "enable_non_final_tokens": self._params.enable_non_final_tokens, - "max_non_final_tokens_duration_ms": self._params.max_non_final_tokens_duration_ms, + "context": context, + "enable_speaker_diarization": self._params.enable_speaker_diarization, + "enable_language_identification": self._params.enable_language_identification, "client_reference_id": self._params.client_reference_id, } From 4050e8b7dc65f39f03d1a7e13c211157f923d4ba Mon Sep 17 00:00:00 2001 From: Aaron Ng Date: Wed, 29 Oct 2025 14:53:20 +0000 Subject: [PATCH 02/20] add speechmatics tts --- .../07a-interruptible-speechmatics-vad.py | 35 ++-- .../07a-interruptible-speechmatics.py | 38 ++-- src/pipecat/services/speechmatics/tts.py | 174 ++++++++++++++++++ 3 files changed, 217 insertions(+), 30 deletions(-) create mode 100644 src/pipecat/services/speechmatics/tts.py diff --git a/examples/foundational/07a-interruptible-speechmatics-vad.py b/examples/foundational/07a-interruptible-speechmatics-vad.py index 55514017f..666f580f8 100644 --- a/examples/foundational/07a-interruptible-speechmatics-vad.py +++ b/examples/foundational/07a-interruptible-speechmatics-vad.py @@ -20,10 +20,10 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.openai.base_llm import BaseOpenAILLMService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.speechmatics.stt import SpeechmaticsSTTService +from pipecat.services.speechmatics.tts import SpeechmaticsTTSService from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -51,35 +51,41 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - """Speechmatics STT Service Example + """Speechmatics STT and TTS Service Example - This example demonstrates using Speechmatics Speech-to-Text service with speaker diarization and intelligent speaker management. Key features: + This example demonstrates using Speechmatics Speech-to-Text and Text-to-Speech services + with speaker diarization and intelligent speaker management. Key features: - 1. Speaker Diarization + 1. Speaker Diarization (STT) - Automatically identifies and distinguishes between different speakers - First speaker is identified as 'S1', others get subsequent IDs - Uses `enable_diarization` parameter to manage speaker detection - 2. Smart Speaker Control + 2. Smart Speaker Control (STT) - `focus_speakers` parameter lets you target specific speakers (e.g. ["S1"]) - Other speakers will be wrapped in PASSIVE tags - Only processes speech from focused speakers - Words from all speakers are wrapped with XML tags for clear speaker identification - Other speakers' speech only sent when focused speaker is active - 3. Voice Activity Detection + 3. Voice Activity Detection (STT) - Built-in VAD using `enable_vad` parameter - Remove `vad_analyzer` from `transport` config to use module's VAD - Emits speaker started/stopped events - 4. Configuration Options + 4. Text-to-Speech (TTS) + - Low latency streaming audio synthesis + - Multiple voice options available including `sarah`, `theo`, and `megan` + + 5. Configuration Options - `operating_point` parameter defaults to `ENHANCED` for optimal accuracy - Configurable `end_of_utterance_silence_trigger` (default 0.5s) - Customizable speaker formatting - Additional diarization settings available - For detailed information about operating points and configuration: - https://docs.speechmatics.com/rt-api-ref + For detailed information: + - STT: https://docs.speechmatics.com/rt-api-ref + - TTS: https://docs.speechmatics.com/text-to-speech/quickstart """ logger.info(f"Starting bot") @@ -97,10 +103,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ), ) - tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), - model="eleven_turbo_v2_5", + tts = SpeechmaticsTTSService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + params=SpeechmaticsTTSService.InputParams( + voice="sarah", + ), ) llm = OpenAILLMService( @@ -112,7 +119,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): { "role": "system", "content": ( - "You are a helpful British assistant called Alfred. " + "You are a helpful British assistant called Sarah. " "Your goal is to demonstrate your capabilities in a succinct way. " "Your output will be converted to audio so don't include special characters in your answers. " "Always include punctuation in your responses. " diff --git a/examples/foundational/07a-interruptible-speechmatics.py b/examples/foundational/07a-interruptible-speechmatics.py index 3d1e639b9..196b6bf68 100644 --- a/examples/foundational/07a-interruptible-speechmatics.py +++ b/examples/foundational/07a-interruptible-speechmatics.py @@ -24,10 +24,10 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.openai.base_llm import BaseOpenAILLMService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.speechmatics.stt import SpeechmaticsSTTService +from pipecat.services.speechmatics.tts import SpeechmaticsTTSService from pipecat.transcriptions.language import Language from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams @@ -61,19 +61,24 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - """Run example using Speechmatics STT. + """Run example using Speechmatics STT and TTS. - This example will use diarization within our STT service and output the words spoken by - each individual speaker and wrap them with XML tags for the LLM to process. Note the - instructions in the system context for the LLM. This greatly improves the conversation - experience by allowing the LLM to understand who is speaking in a multi-party call. + This example demonstrates a complete Speechmatics integration with both Speech-to-Text + and Text-to-Speech services: - By default, this example will use our ENHANCED operating point, which is optimized for - high accuracy. You can change this by setting the `operating_point` parameter to a different - value. + STT Features: + - Diarization to identify and distinguish between different speakers + - Words spoken by each speaker are wrapped with XML tags for LLM processing + - System context instructions help the LLM understand multi-party conversations + - ENHANCED operating point by default for optimal accuracy - For more information on operating points, see the Speechmatics documentation: - https://docs.speechmatics.com/rt-api-ref + TTS Features: + - Low latency streaming audio synthesis + - Multiple voice options available including `sarah`, `theo`, and `megan` + + For more information: + - STT: https://docs.speechmatics.com/rt-api-ref + - TTS: https://docs.speechmatics.com/text-to-speech/quickstart """ logger.info(f"Starting bot") @@ -87,10 +92,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ), ) - tts = ElevenLabsTTSService( - api_key=os.getenv("ELEVENLABS_API_KEY"), - voice_id=os.getenv("ELEVENLABS_VOICE_ID"), - model="eleven_turbo_v2_5", + tts = SpeechmaticsTTSService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + params=SpeechmaticsTTSService.InputParams( + voice="sarah", + ), ) llm = OpenAILLMService( @@ -102,7 +108,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): { "role": "system", "content": ( - "You are a helpful British assistant called Alfred. " + "You are a helpful British assistant called Sarah. " "Your goal is to demonstrate your capabilities in a succinct way. " "Your output will be converted to audio so don't include special characters in your answers. " "Always include punctuation in your responses. " diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py new file mode 100644 index 000000000..46421360f --- /dev/null +++ b/src/pipecat/services/speechmatics/tts.py @@ -0,0 +1,174 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Speechmatics TTS service integration.""" + +import os +from typing import AsyncGenerator, Optional + +import aiohttp +import numpy as np +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TTSAudioRawFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.services.tts_service import TTSService +from pipecat.utils.tracing.service_decorators import traced_tts + + +class SpeechmaticsTTSService(TTSService): + """Speechmatics TTS service implementation. + + This service provides text-to-speech synthesis using the Speechmatics HTTP API. + It converts text to speech and returns raw PCM audio data for real-time playback. + """ + + class InputParams(BaseModel): + """Configuration parameters for Speechmatics TTS service. + + Parameters: + voice: Voice model to use for synthesis. Defaults to "sarah". + """ + + voice: str = "sarah" + + def __init__( + self, + *, + api_key: str | None = None, + base_url: str | None = None, + aiohttp_session: aiohttp.ClientSession | None = None, + sample_rate: Optional[int] = 16000, + params: InputParams | None = None, + **kwargs, + ): + """Initialize the Speechmatics TTS service. + + Args: + api_key: Speechmatics API key for authentication. Uses environment variable + `SPEECHMATICS_API_KEY` if not provided. + base_url: Base URL for Speechmatics TTS API. Defaults to + `https://preview.tts.speechmatics.com`. + aiohttp_session: Shared aiohttp session for HTTP requests. + sample_rate: Audio sample rate in Hz. Defaults to 16000. + params: Optional[InputParams]: Input parameters for the service. + **kwargs: Additional arguments passed to TTSService. + """ + super().__init__(sample_rate=sample_rate, **kwargs) + + # Service parameters + self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY") + self._base_url: str = base_url or "https://preview.tts.speechmatics.com" + self._session = aiohttp_session or aiohttp.ClientSession() + + # Check we have required attributes + if not self._api_key: + raise ValueError("Missing Speechmatics API key") + if not self._base_url: + raise ValueError("Missing Speechmatics base URL") + + # Default parameters + self._params = params or SpeechmaticsTTSService.InputParams() + + # Set voice from parameters + self.set_voice(self._params.voice) + + def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Speechmatics service supports metrics generation. + """ + return True + + @traced_tts + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Speechmatics' HTTP API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + logger.debug(f"{self}: Generating TTS [{text}]") + + headers = { + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + } + + payload = { + "text": text, + } + + url = f"{self._base_url}/generate/{self._voice_id}?output_format=pcm_16000" + + try: + await self.start_ttfb_metrics() + + async with self._session.post(url, json=payload, headers=headers) as response: + if response.status != 200: + error_message = f"Speechmatics TTS error: HTTP {response.status}" + logger.error(error_message) + yield ErrorFrame(error=error_message) + return + + await self.start_tts_usage_metrics(text) + + yield TTSStartedFrame() + + # Process the response in streaming chunks + first_chunk = True + buffer = b"" + + # Helper to move all complete 2-byte int16 samples from buffer into a frame + def _emit_complete_samples(): + nonlocal buffer + if len(buffer) < 2: + return None + complete_samples = len(buffer) // 2 + complete_bytes = complete_samples * 2 + + audio_data = buffer[:complete_bytes] + buffer = buffer[complete_bytes:] # Keep remaining bytes for next iteration + + return TTSAudioRawFrame( + audio=audio_data, + sample_rate=self.sample_rate, + num_channels=1, + ) + + async for chunk in response.content.iter_any(): + if not chunk: + continue + if first_chunk: + await self.stop_ttfb_metrics() + first_chunk = False + + buffer += chunk + + # Emit a frame for all complete samples currently in buffer + frame = _emit_complete_samples() + if frame: + yield frame + + # Process any remaining bytes in buffer after streaming ends + frame = _emit_complete_samples() + if frame: + yield frame + + except Exception as e: + logger.exception(f"Error generating TTS: {e}") + yield ErrorFrame(error=f"Speechmatics TTS error: {str(e)}") + finally: + yield TTSStoppedFrame() From b0acbeffb9e0591fbcf41703df9a723835cb1299 Mon Sep 17 00:00:00 2001 From: Aaron Ng Date: Wed, 29 Oct 2025 16:33:18 +0000 Subject: [PATCH 03/20] add sm-app param --- src/pipecat/services/speechmatics/tts.py | 32 ++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 46421360f..207f898aa 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -8,9 +8,9 @@ import os from typing import AsyncGenerator, Optional +from urllib.parse import urlencode import aiohttp -import numpy as np from loguru import logger from pydantic import BaseModel @@ -24,6 +24,15 @@ from pipecat.frames.frames import ( from pipecat.services.tts_service import TTSService from pipecat.utils.tracing.service_decorators import traced_tts +try: + from speechmatics.rt import __version__ +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Speechmatics, you need to `pip install pipecat-ai[speechmatics]`." + ) + raise Exception(f"Missing module: {e}") + class SpeechmaticsTTSService(TTSService): """Speechmatics TTS service implementation. @@ -111,7 +120,7 @@ class SpeechmaticsTTSService(TTSService): "text": text, } - url = f"{self._base_url}/generate/{self._voice_id}?output_format=pcm_16000" + url = _get_endpoint_url(self._base_url, self._voice_id, self.sample_rate) try: await self.start_ttfb_metrics() @@ -172,3 +181,22 @@ class SpeechmaticsTTSService(TTSService): yield ErrorFrame(error=f"Speechmatics TTS error: {str(e)}") finally: yield TTSStoppedFrame() + + +def _get_endpoint_url(base_url: str, voice: str, sample_rate: int) -> str: + """Format the TTS endpoint URL with voice, output format, and version params. + + Args: + base_url: The base URL for the TTS endpoint. + voice: The voice model to use. + sample_rate: The audio sample rate. + + Returns: + str: The formatted TTS endpoint URL. + """ + query_params = {} + query_params["output_format"] = f"pcm_{sample_rate}" + query_params["sm-app"] = f"pipecat/{__version__}" + query = urlencode(query_params) + + return f"{base_url}/generate/{voice}?{query}" From 9d509bb40956c7ebcbf8a3d478a7a19038c047fc Mon Sep 17 00:00:00 2001 From: Aaron Ng Date: Thu, 30 Oct 2025 16:25:10 +0000 Subject: [PATCH 04/20] address changes --- .../07a-interruptible-speechmatics-vad.py | 150 +++++++++--------- .../07a-interruptible-speechmatics.py | 141 ++++++++-------- src/pipecat/services/speechmatics/tts.py | 85 +++++----- 3 files changed, 182 insertions(+), 194 deletions(-) diff --git a/examples/foundational/07a-interruptible-speechmatics-vad.py b/examples/foundational/07a-interruptible-speechmatics-vad.py index 666f580f8..6e78a5147 100644 --- a/examples/foundational/07a-interruptible-speechmatics-vad.py +++ b/examples/foundational/07a-interruptible-speechmatics-vad.py @@ -6,6 +6,7 @@ import os +import aiohttp from dotenv import load_dotenv from loguru import logger @@ -89,90 +90,89 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ logger.info(f"Starting bot") - - stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), - params=SpeechmaticsSTTService.InputParams( - language=Language.EN, - enable_vad=True, - enable_diarization=True, - focus_speakers=["S1"], - end_of_utterance_silence_trigger=0.5, - speaker_active_format="<{speaker_id}>{text}", - speaker_passive_format="<{speaker_id}>{text}", - ), - ) - - tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), - params=SpeechmaticsTTSService.InputParams( - voice="sarah", - ), - ) - - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - params=BaseOpenAILLMService.InputParams(temperature=0.75), - ) - - messages = [ - { - "role": "system", - "content": ( - "You are a helpful British assistant called Sarah. " - "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers. " - "Always include punctuation in your responses. " - "Give very short replies - do not give longer replies unless strictly necessary. " - "Respond to what the user said in a concise, funny, creative and helpful way. " - "Use `` tags to identify different speakers - do not use tags in your replies. " - "Do not respond to speakers within `` tags unless explicitly asked to. " + async with aiohttp.ClientSession() as session: + stt = SpeechmaticsSTTService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + params=SpeechmaticsSTTService.InputParams( + language=Language.EN, + enable_vad=True, + enable_diarization=True, + focus_speakers=["S1"], + end_of_utterance_silence_trigger=0.5, + speaker_active_format="<{speaker_id}>{text}", + speaker_passive_format="<{speaker_id}>{text}", ), - }, - ] + ) - context = LLMContext(messages) - context_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), - ) + tts = SpeechmaticsTTSService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + voice_id="sarah", + aiohttp_session=session, + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + params=BaseOpenAILLMService.InputParams(temperature=0.75), + ) + + messages = [ + { + "role": "system", + "content": ( + "You are a helpful British assistant called Sarah. " + "Your goal is to demonstrate your capabilities in a succinct way. " + "Your output will be converted to audio so don't include special characters in your answers. " + "Always include punctuation in your responses. " + "Give very short replies - do not give longer replies unless strictly necessary. " + "Respond to what the user said in a concise, funny, creative and helpful way. " + "Use `` tags to identify different speakers - do not use tags in your replies. " + "Do not respond to speakers within `` tags unless explicitly asked to. " + ), + }, ] - ) - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), + ) - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - messages.append({"role": "system", "content": "Say a short hello to the user."}) - await task.queue_frames([LLMRunFrame()]) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Say a short hello to the user."}) + await task.queue_frames([LLMRunFrame()]) - await runner.run(task) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) async def bot(runner_args: RunnerArguments): diff --git a/examples/foundational/07a-interruptible-speechmatics.py b/examples/foundational/07a-interruptible-speechmatics.py index 196b6bf68..36ac39b82 100644 --- a/examples/foundational/07a-interruptible-speechmatics.py +++ b/examples/foundational/07a-interruptible-speechmatics.py @@ -6,6 +6,7 @@ import os +import aiohttp from dotenv import load_dotenv from loguru import logger @@ -82,85 +83,85 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): """ logger.info(f"Starting bot") - stt = SpeechmaticsSTTService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), - params=SpeechmaticsSTTService.InputParams( - language=Language.EN, - enable_diarization=True, - end_of_utterance_silence_trigger=0.5, - speaker_active_format="<{speaker_id}>{text}", - ), - ) - - tts = SpeechmaticsTTSService( - api_key=os.getenv("SPEECHMATICS_API_KEY"), - params=SpeechmaticsTTSService.InputParams( - voice="sarah", - ), - ) - - llm = OpenAILLMService( - api_key=os.getenv("OPENAI_API_KEY"), - params=BaseOpenAILLMService.InputParams(temperature=0.75), - ) - - messages = [ - { - "role": "system", - "content": ( - "You are a helpful British assistant called Sarah. " - "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers. " - "Always include punctuation in your responses. " - "Give very short replies - do not give longer replies unless strictly necessary. " - "Respond to what the user said in a concise, funny, creative and helpful way. " - "Use `` tags to identify different speakers - do not use tags in your replies." + async with aiohttp.ClientSession() as session: + stt = SpeechmaticsSTTService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + params=SpeechmaticsSTTService.InputParams( + language=Language.EN, + enable_diarization=True, + end_of_utterance_silence_trigger=0.5, + speaker_active_format="<{speaker_id}>{text}", ), - }, - ] + ) - context = LLMContext(messages) - context_aggregator = LLMContextAggregatorPair( - context, - user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), - ) + tts = SpeechmaticsTTSService( + api_key=os.getenv("SPEECHMATICS_API_KEY"), + voice_id="sarah", + aiohttp_session=session, + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + params=BaseOpenAILLMService.InputParams(temperature=0.75), + ) + + messages = [ + { + "role": "system", + "content": ( + "You are a helpful British assistant called Sarah. " + "Your goal is to demonstrate your capabilities in a succinct way. " + "Your output will be converted to audio so don't include special characters in your answers. " + "Always include punctuation in your responses. " + "Give very short replies - do not give longer replies unless strictly necessary. " + "Respond to what the user said in a concise, funny, creative and helpful way. " + "Use `` tags to identify different speakers - do not use tags in your replies." + ), + }, ] - ) - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams(aggregation_timeout=0.005), + ) - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - messages.append({"role": "system", "content": "Say a short hello to the user."}) - await task.queue_frames([LLMRunFrame()]) + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Say a short hello to the user."}) + await task.queue_frames([LLMRunFrame()]) - await runner.run(task) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) async def bot(runner_args: RunnerArguments): diff --git a/src/pipecat/services/speechmatics/tts.py b/src/pipecat/services/speechmatics/tts.py index 207f898aa..23d10c5e1 100644 --- a/src/pipecat/services/speechmatics/tts.py +++ b/src/pipecat/services/speechmatics/tts.py @@ -6,7 +6,6 @@ """Speechmatics TTS service integration.""" -import os from typing import AsyncGenerator, Optional from urllib.parse import urlencode @@ -41,55 +40,56 @@ class SpeechmaticsTTSService(TTSService): It converts text to speech and returns raw PCM audio data for real-time playback. """ + SPEECHMATICS_SAMPLE_RATE = 16000 + class InputParams(BaseModel): - """Configuration parameters for Speechmatics TTS service. + """Optional input parameters for Speechmatics TTS configuration.""" - Parameters: - voice: Voice model to use for synthesis. Defaults to "sarah". - """ - - voice: str = "sarah" + pass def __init__( self, *, - api_key: str | None = None, - base_url: str | None = None, - aiohttp_session: aiohttp.ClientSession | None = None, - sample_rate: Optional[int] = 16000, - params: InputParams | None = None, + api_key: str, + base_url: str = "https://preview.tts.speechmatics.com", + voice_id: str = "sarah", + aiohttp_session: aiohttp.ClientSession, + sample_rate: Optional[int] = SPEECHMATICS_SAMPLE_RATE, + params: Optional[InputParams] = None, **kwargs, ): """Initialize the Speechmatics TTS service. Args: - api_key: Speechmatics API key for authentication. Uses environment variable - `SPEECHMATICS_API_KEY` if not provided. - base_url: Base URL for Speechmatics TTS API. Defaults to - `https://preview.tts.speechmatics.com`. + api_key: Speechmatics API key for authentication. + base_url: Base URL for Speechmatics TTS API. + voice_id: Voice model to use for synthesis. aiohttp_session: Shared aiohttp session for HTTP requests. - sample_rate: Audio sample rate in Hz. Defaults to 16000. + sample_rate: Audio sample rate in Hz. params: Optional[InputParams]: Input parameters for the service. **kwargs: Additional arguments passed to TTSService. """ + if sample_rate and sample_rate != self.SPEECHMATICS_SAMPLE_RATE: + logger.warning( + f"Speechmatics TTS only supports {self.SPEECHMATICS_SAMPLE_RATE}Hz sample rate. " + f"Current rate of {sample_rate}Hz may cause issues." + ) super().__init__(sample_rate=sample_rate, **kwargs) # Service parameters - self._api_key: str = api_key or os.getenv("SPEECHMATICS_API_KEY") - self._base_url: str = base_url or "https://preview.tts.speechmatics.com" - self._session = aiohttp_session or aiohttp.ClientSession() + self._api_key: str = api_key + self._base_url: str = base_url + self._session = aiohttp_session # Check we have required attributes if not self._api_key: raise ValueError("Missing Speechmatics API key") - if not self._base_url: - raise ValueError("Missing Speechmatics base URL") # Default parameters self._params = params or SpeechmaticsTTSService.InputParams() - # Set voice from parameters - self.set_voice(self._params.voice) + # Set voice from constructor parameter + self.set_voice(voice_id) def can_generate_metrics(self) -> bool: """Check if this service can generate processing metrics. @@ -140,23 +140,6 @@ class SpeechmaticsTTSService(TTSService): first_chunk = True buffer = b"" - # Helper to move all complete 2-byte int16 samples from buffer into a frame - def _emit_complete_samples(): - nonlocal buffer - if len(buffer) < 2: - return None - complete_samples = len(buffer) // 2 - complete_bytes = complete_samples * 2 - - audio_data = buffer[:complete_bytes] - buffer = buffer[complete_bytes:] # Keep remaining bytes for next iteration - - return TTSAudioRawFrame( - audio=audio_data, - sample_rate=self.sample_rate, - num_channels=1, - ) - async for chunk in response.content.iter_any(): if not chunk: continue @@ -166,15 +149,19 @@ class SpeechmaticsTTSService(TTSService): buffer += chunk - # Emit a frame for all complete samples currently in buffer - frame = _emit_complete_samples() - if frame: - yield frame + # Emit all complete 2-byte int16 samples from buffer + if len(buffer) >= 2: + complete_samples = len(buffer) // 2 + complete_bytes = complete_samples * 2 - # Process any remaining bytes in buffer after streaming ends - frame = _emit_complete_samples() - if frame: - yield frame + audio_data = buffer[:complete_bytes] + buffer = buffer[complete_bytes:] # Keep remaining bytes for next iteration + + yield TTSAudioRawFrame( + audio=audio_data, + sample_rate=self.sample_rate, + num_channels=1, + ) except Exception as e: logger.exception(f"Error generating TTS: {e}") From 52b33e5106055d63612ed981a7b546dcbd788850 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 30 Oct 2025 16:09:07 -0300 Subject: [PATCH 05/20] New event handlers for the DeepgramFluxSTTService. --- CHANGELOG.md | 3 +++ .../foundational/07c-interruptible-deepgram-flux.py | 4 ++++ src/pipecat/services/deepgram/flux/stt.py | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24484687f..c0ba85a6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- New event handlers for the `DeepgramFluxSTTService`: `on_start_of_turn`, + `on_turn_resumed`, `on_end_of_turn`, `on_eager_end_of_turn`, `on_update`. + - Added `generation_config` parameter support to `CartesiaTTSService` and `CartesiaHttpTTSService` for Cartesia Sonic-3 models. Includes a new `GenerationConfig` class with `volume` (0.5-2.0), `speed` (0.6-1.5), diff --git a/examples/foundational/07c-interruptible-deepgram-flux.py b/examples/foundational/07c-interruptible-deepgram-flux.py index 1331ab2a3..75a022a5c 100644 --- a/examples/foundational/07c-interruptible-deepgram-flux.py +++ b/examples/foundational/07c-interruptible-deepgram-flux.py @@ -101,6 +101,10 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Client disconnected") await task.cancel() + @stt.event_handler("on_update") + async def on_deepgram_flux_update(stt, transcript): + logger.debug(f"On deeggram flux update: {transcript}") + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) await runner.run(task) diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index f0b1a5baa..78f615a4d 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -156,6 +156,12 @@ class DeepgramFluxSTTService(WebsocketSTTService): self._language = Language.EN self._websocket_url = None self._receive_task = None + # Flux event handlers + self._register_event_handler("on_start_of_turn") + self._register_event_handler("on_turn_resumed") + self._register_event_handler("on_end_of_turn") + self._register_event_handler("on_eager_end_of_turn") + self._register_event_handler("on_update") async def _connect(self): """Connect to WebSocket and start background tasks. @@ -523,6 +529,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): await self.push_frame(UserStartedSpeakingFrame(), FrameDirection.DOWNSTREAM) await self.push_frame(UserStartedSpeakingFrame(), FrameDirection.UPSTREAM) await self.start_metrics() + await self._call_event_handler("on_start_of_turn", transcript) if transcript: logger.trace(f"Start of turn transcript: {transcript}") @@ -537,6 +544,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): event: The event type string for logging purposes. """ logger.trace(f"Received event TurnResumed: {event}") + await self._call_event_handler("on_turn_resumed") async def _handle_end_of_turn(self, transcript: str, data: Dict[str, Any]): """Handle EndOfTurn events from Deepgram Flux. @@ -571,6 +579,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): await self.stop_processing_metrics() await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM) await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.UPSTREAM) + await self._call_event_handler("on_end_of_turn", transcript) async def _handle_eager_end_of_turn(self, transcript: str, data: Dict[str, Any]): """Handle EagerEndOfTurn events from Deepgram Flux. @@ -615,6 +624,7 @@ class DeepgramFluxSTTService(WebsocketSTTService): result=data, ) ) + await self._call_event_handler("on_eager_end_of_turn", transcript) async def _handle_update(self, transcript: str): """Handle Update events from Deepgram Flux. @@ -638,3 +648,4 @@ class DeepgramFluxSTTService(WebsocketSTTService): # both the "user started speaking" event and the first transcript simultaneously, # making this timing measurement meaningless in this context. # await self.stop_ttfb_metrics() + await self._call_event_handler("on_update", transcript) From b094418d1e65185e6d00a357aa7e783275d5a61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:00:13 -0700 Subject: [PATCH 06/20] LLMContext: add create_image_message and create_audio_message --- CHANGELOG.md | 9 ++ .../processors/aggregators/llm_context.py | 149 +++++++++--------- 2 files changed, 86 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0ba85a6e..0aab3986b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for including images or audio to LLM context messages using + `LLMContext.create_image_message()` and `LLMContext.create_audio_message()`. + For example, when creating `LLMMessagesAppendFrame`: + + ```python + message = LLMContext.create_image_message(image=..., size= ...) + await self.push_frame(LLMMessagesAppendFrame(messages=[message], run_llm=True)) + ``` + - New event handlers for the `DeepgramFluxSTTService`: `on_start_of_turn`, `on_turn_resumed`, `on_end_of_turn`, `on_eager_end_of_turn`, `on_update`. diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 8dc79fb50..768df0e5e 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -16,6 +16,7 @@ service-specific adapter. import base64 import io +import wave from dataclasses import dataclass from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union @@ -113,6 +114,77 @@ class LLMContext: self._tools: ToolsSchema | NotGiven = LLMContext._normalize_and_validate_tools(tools) self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice + @staticmethod + def create_image_message( + *, + role: str = "user", + format: str, + size: tuple[int, int], + image: bytes, + text: Optional[str] = None, + ) -> LLMContextMessage: + """Create a context message containing an image. + + Args: + role: The role of this message (defaults to "user"). + format: Image format (e.g., 'RGB', 'RGBA'). + size: Image dimensions as (width, height) tuple. + image: Raw image bytes. + text: Optional text to include with the image. + """ + buffer = io.BytesIO() + Image.frombytes(format, size, image).save(buffer, format="JPEG") + encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + + content = [] + if text: + content.append({"type": "text", "text": text}) + + content.append( + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}, + }, + ) + + return {"role": role, "content": content} + + @staticmethod + def create_audio_message( + *, role: str = "user", audio_frames: list[AudioRawFrame], text: str = "Audio follows" + ) -> LLMContextMessage: + """Create a context message containing audio. + + Args: + role: The role of this message (defaults to "user"). + audio_frames: List of audio frame objects to include. + text: Optional text to include with the audio. + """ + sample_rate = audio_frames[0].sample_rate + num_channels = audio_frames[0].num_channels + + content = [] + content.append({"type": "text", "text": text}) + data = b"".join(frame.audio for frame in audio_frames) + + with io.BytesIO() as buffer: + with wave.open(buffer, "wb") as wf: + wf.setsampwidth(2) + wf.setnchannels(num_channels) + wf.setframerate(sample_rate) + wf.writeframes(data) + + encoded_audio = base64.b64encode(buffer.getvalue()).decode("utf-8") + + content.append( + { + "type": "input_audio", + "input_audio": {"data": encoded_audio, "format": "wav"}, + } + ) + + return {"role": role, "content": content} + @property def messages(self) -> List[LLMContextMessage]: """Get the current messages list. @@ -238,7 +310,7 @@ class LLMContext: self._tool_choice = tool_choice def add_image_frame_message( - self, *, format: str, size: tuple[int, int], image: bytes, text: str = None + self, *, format: str, size: tuple[int, int], image: bytes, text: Optional[str] = None ): """Add a message containing an image frame. @@ -248,17 +320,8 @@ class LLMContext: image: Raw image bytes. text: Optional text to include with the image. """ - buffer = io.BytesIO() - Image.frombytes(format, size, image).save(buffer, format="JPEG") - encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") - - content = [] - if text: - content.append({"type": "text", "text": text}) - content.append( - {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}}, - ) - self.add_message({"role": "user", "content": content}) + message = LLMContext.create_image_message(format=format, size=size, image=image, text=text) + self.add_message(message) def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" @@ -269,66 +332,8 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - if not audio_frames: - return - - sample_rate = audio_frames[0].sample_rate - num_channels = audio_frames[0].num_channels - - content = [] - content.append({"type": "text", "text": text}) - data = b"".join(frame.audio for frame in audio_frames) - data = bytes( - self._create_wav_header( - sample_rate, - num_channels, - 16, - len(data), - ) - + data - ) - encoded_audio = base64.b64encode(data).decode("utf-8") - content.append( - { - "type": "input_audio", - "input_audio": {"data": encoded_audio, "format": "wav"}, - } - ) - self.add_message({"role": "user", "content": content}) - - def _create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size): - """Create a WAV file header for audio data. - - Args: - sample_rate: Audio sample rate in Hz. - num_channels: Number of audio channels. - bits_per_sample: Bits per audio sample. - data_size: Size of audio data in bytes. - - Returns: - WAV header as a bytearray. - """ - # RIFF chunk descriptor - header = bytearray() - header.extend(b"RIFF") # ChunkID - header.extend((data_size + 36).to_bytes(4, "little")) # ChunkSize: total size - 8 - header.extend(b"WAVE") # Format - # "fmt " sub-chunk - header.extend(b"fmt ") # Subchunk1ID - header.extend((16).to_bytes(4, "little")) # Subchunk1Size (16 for PCM) - header.extend((1).to_bytes(2, "little")) # AudioFormat (1 for PCM) - header.extend(num_channels.to_bytes(2, "little")) # NumChannels - header.extend(sample_rate.to_bytes(4, "little")) # SampleRate - # Calculate byte rate and block align - byte_rate = sample_rate * num_channels * (bits_per_sample // 8) - block_align = num_channels * (bits_per_sample // 8) - header.extend(byte_rate.to_bytes(4, "little")) # ByteRate - header.extend(block_align.to_bytes(2, "little")) # BlockAlign - header.extend(bits_per_sample.to_bytes(2, "little")) # BitsPerSample - # "data" sub-chunk - header.extend(b"data") # Subchunk2ID - header.extend(data_size.to_bytes(4, "little")) # Subchunk2Size - return header + message = LLMContext.create_audio_message(audio_frames=audio_frames, text=text) + self.add_message(message) @staticmethod def _normalize_and_validate_tools(tools: ToolsSchema | NotGiven) -> ToolsSchema | NotGiven: From 817a485d9435e74eb83a7f037192c2e0aff30c38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:04:16 -0700 Subject: [PATCH 07/20] frames: added VisionImageRawFrame --- CHANGELOG.md | 4 ++++ src/pipecat/frames/frames.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aab3986b..f6e5143b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `VisionImageRawFrame`. This is an input image frame with an associated + text. It is usually processed by vision services (e.g. Moondream). The text + guides the vision service on how to analyze the image. + - Added support for including images or audio to LLM context messages using `LLMContext.create_image_message()` and `LLMContext.create_audio_message()`. For example, when creating `LLMMessagesAppendFrame`: diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 2a9651b83..94f8702b3 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1305,6 +1305,24 @@ class UserImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, request: {self.request})" +@dataclass +class VisionImageRawFrame(InputImageRawFrame): + """Raw image input frame to be analyzed by vision services. + + This is just an image with an associated text describing how the vision + service should analyze the image. + + Parameters: + text: Description of how the vision service should analyze the image. + """ + + text: str + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text})" + + @dataclass class InputDTMFFrame(DTMFFrame, SystemFrame): """DTMF keypress input frame from transport.""" From ce13155d26ee0dff6c36c95071ccf2efa36d343a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:06:03 -0700 Subject: [PATCH 08/20] vision(moondream): process VisionImageRawFrame --- CHANGELOG.md | 4 ++ src/pipecat/services/moondream/vision.py | 50 ++++-------------------- src/pipecat/services/vision_service.py | 13 +++--- 3 files changed, 18 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6e5143b3..f4bd487d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -167,6 +167,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `MoondreamService` to process `VisionImageRawFrame`. + +- `VisionService` expects `VisionImageRawFrame` in order to analyze images. + - `DailyTransport` triggers `on_error` event if transcription can't be started or stopped. diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index bd01daf34..6e470071e 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -11,15 +11,12 @@ for image analysis and description generation. """ import asyncio -import base64 -from io import BytesIO from typing import AsyncGenerator, Optional from loguru import logger from PIL import Image -from pipecat.frames.frames import ErrorFrame, Frame, TextFrame -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.frames.frames import ErrorFrame, Frame, TextFrame, VisionImageRawFrame from pipecat.services.vision_service import VisionService try: @@ -92,16 +89,16 @@ class MoondreamService(VisionService): trust_remote_code=True, revision=revision, device_map={"": device}, - torch_dtype=dtype, + dtype=dtype, ).eval() logger.debug("Loaded Moondream model") - async def run_vision(self, context: LLMContext) -> AsyncGenerator[Frame, None]: + async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: """Analyze an image and generate a description. Args: - context: The context to process, containing image data. + frame: The vision image frame to process. Yields: Frame: TextFrame containing the generated image description, or ErrorFrame @@ -112,45 +109,14 @@ class MoondreamService(VisionService): yield ErrorFrame("Moondream model not available") return - image_bytes = None - text = None - try: - messages = context.get_messages() - last_message = messages[-1] - last_message_content = last_message.get("content") + logger.debug(f"Analyzing image (bytes length: {len(frame.image)})") - for item in last_message_content: - if isinstance(item, dict): - if ( - "image_url" in item - and isinstance(item["image_url"], dict) - and item["image_url"].get("url") - ): - image_bytes = base64.b64decode(item["image_url"]["url"].split(",")[1]) - elif "text" in item and isinstance(item["text"], str): - text = item["text"] - - except Exception as e: - logger.error(f"Exception during image extraction: {e}") - yield ErrorFrame("Failed to extract image from context") - return - - if not image_bytes: - logger.error("No image found in context") - yield ErrorFrame("No image found in context") - return - - logger.debug( - f"Analyzing image (bytes length: {len(image_bytes) if image_bytes else 'None'})" - ) - - def get_image_description(bytes: bytes, text: Optional[str]) -> str: - image_buffer = BytesIO(bytes) - image = Image.open(image_buffer) + def get_image_description(image_bytes: bytes, text: Optional[str]) -> str: + image = Image.frombytes(frame.format, frame.size, image_bytes) image_embeds = self._model.encode_image(image) description = self._model.query(image_embeds, text)["answer"] return description - description = await asyncio.to_thread(get_image_description, image_bytes, text) + description = await asyncio.to_thread(get_image_description, frame.image, frame.text) yield TextFrame(text=description) diff --git a/src/pipecat/services/vision_service.py b/src/pipecat/services/vision_service.py index 0eeee98cd..16b25277c 100644 --- a/src/pipecat/services/vision_service.py +++ b/src/pipecat/services/vision_service.py @@ -14,8 +14,7 @@ visual content. from abc import abstractmethod from typing import AsyncGenerator -from pipecat.frames.frames import Frame, LLMContextFrame -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.frames.frames import Frame, VisionImageRawFrame from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService @@ -38,15 +37,15 @@ class VisionService(AIService): self._describe_text = None @abstractmethod - async def run_vision(self, context: LLMContext) -> AsyncGenerator[Frame, None]: - """Process the latest image in the context and generate results. + async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: + """Process the given vision image and generate results. This method must be implemented by subclasses to provide actual computer vision functionality such as image description, object detection, or visual question answering. Args: - context: The context to process, containing image data. + frame: The vision image frame to process. Yields: Frame: Frames containing the vision analysis results, typically TextFrame @@ -66,9 +65,9 @@ class VisionService(AIService): """ await super().process_frame(frame, direction) - if isinstance(frame, LLMContextFrame): + if isinstance(frame, VisionImageRawFrame): await self.start_processing_metrics() - await self.process_generator(self.run_vision(frame.context)) + await self.process_generator(self.run_vision(frame)) await self.stop_processing_metrics() else: await self.push_frame(frame, direction) From e0933e20d2b235546c5b889f61af69ff232257b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:18:40 -0700 Subject: [PATCH 09/20] deprecated UserResponseAggregator --- CHANGELOG.md | 2 ++ src/pipecat/processors/aggregators/user_response.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4bd487d1..056c548ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -195,6 +195,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- `UserResponseAggregator` is deprecated and will be removed in a future version. + - The `send_transcription_frames` argument to `OpenAIRealtimeLLMService` is deprecated. Transcription frames are now always sent. They go upstream, to be handled by the user context aggregator. See "Added" section for details. diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 274a31d52..a7872c35e 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -27,11 +27,24 @@ class UserResponseAggregator(LLMUserAggregator): def __init__(self, **kwargs): """Initialize the user response aggregator. + .. deprecated:: 0.0.92 + `UserResponseAggregator` is deprecated and will be removed in a future version. + Args: **kwargs: Additional arguments passed to parent LLMUserAggregator. """ super().__init__(context=LLMContext(), **kwargs) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`UserResponseAggregator` is deprecated and will be removed in a future version.", + DeprecationWarning, + stacklevel=2, + ) + async def push_aggregation(self): """Push the aggregated user response as a TextFrame. From 9c5690d670e71503564e697e1399265321c51288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:54:28 -0700 Subject: [PATCH 10/20] LLMContext: added support for image messages with URLs --- CHANGELOG.md | 5 +-- .../adapters/services/anthropic_adapter.py | 26 ++++++++++---- .../adapters/services/bedrock_adapter.py | 23 +++++++----- .../adapters/services/gemini_adapter.py | 5 ++- .../processors/aggregators/llm_context.py | 36 ++++++++++++------- 5 files changed, 65 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 056c548ca..07596b3b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 guides the vision service on how to analyze the image. - Added support for including images or audio to LLM context messages using - `LLMContext.create_image_message()` and `LLMContext.create_audio_message()`. - For example, when creating `LLMMessagesAppendFrame`: + `LLMContext.create_image_message()` or `LLMContext.create_image_url_message()` + (not all LLMs support URLs) and `LLMContext.create_audio_message()`. For + example, when creating `LLMMessagesAppendFrame`: ```python message = LLMContext.create_image_message(image=..., size= ...) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index a106b4de4..75fa5899d 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -245,13 +245,25 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): item["text"] = "(empty)" # handle image_url -> image conversion if item["type"] == "image_url": - item["type"] = "image" - item["source"] = { - "type": "base64", - "media_type": "image/jpeg", - "data": item["image_url"]["url"].split(",")[1], - } - del item["image_url"] + if item["image_url"]["url"].startswith("data:"): + item["type"] = "image" + item["source"] = { + "type": "base64", + "media_type": "image/jpeg", + "data": item["image_url"]["url"].split(",")[1], + } + del item["image_url"] + elif item["image_url"]["url"].startswith("http"): + item["type"] = "image" + item["source"] = { + "type": "url", + "url": item["image_url"]["url"], + } + del item["image_url"] + else: + url = item["image_url"]["url"] + logger.warning(f"Unsupported 'image_url': {url}") + # In the case where there's a single image in the list (like what # would result from a UserImageRawFrame), ensure that the image # comes before text, as recommended by Anthropic docs diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index 852ea17a4..213e3b01c 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -256,15 +256,22 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): new_content.append({"text": text_content}) # handle image_url -> image conversion if item["type"] == "image_url": - new_item = { - "image": { - "format": "jpeg", - "source": { - "bytes": base64.b64decode(item["image_url"]["url"].split(",")[1]) - }, + if item["image_url"]["url"].startswith("data:"): + new_item = { + "image": { + "format": "jpeg", + "source": { + "bytes": base64.b64decode( + item["image_url"]["url"].split(",")[1] + ) + }, + } } - } - new_content.append(new_item) + new_content.append(new_item) + else: + url = item["image_url"]["url"] + logger.warning(f"Unsupported 'image_url': {url}") + # In the case where there's a single image in the list (like what # would result from a UserImageRawFrame), ensure that the image # comes before text diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index 1fa8d9e6f..dc5bae559 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -343,7 +343,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): for c in content: if c["type"] == "text": parts.append(Part(text=c["text"])) - elif c["type"] == "image_url": + elif c["type"] == "image_url" and c["image_url"]["url"].startswith("data:"): parts.append( Part( inline_data=Blob( @@ -352,6 +352,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): ) ) ) + elif c["type"] == "image_url": + url = c["image_url"]["url"] + logger.warning(f"Unsupported 'image_url': {url}") elif c["type"] == "input_audio": input_audio = c["input_audio"] audio_bytes = base64.b64decode(input_audio["data"]) diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index 768df0e5e..d9280f9c0 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -114,6 +114,28 @@ class LLMContext: self._tools: ToolsSchema | NotGiven = LLMContext._normalize_and_validate_tools(tools) self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice + @staticmethod + def create_image_url_message( + *, + role: str = "user", + url: str, + text: Optional[str] = None, + ) -> LLMContextMessage: + """Create a context message containing an image URL. + + Args: + role: The role of this message (defaults to "user"). + url: The URL of the image. + text: Optional text to include with the image. + """ + content = [] + if text: + content.append({"type": "text", "text": text}) + + content.append({"type": "image_url", "image_url": {"url": url}}) + + return {"role": role, "content": content} + @staticmethod def create_image_message( *, @@ -135,19 +157,9 @@ class LLMContext: buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") + url = f"data:image/jpeg;base64,{encoded_image}" - content = [] - if text: - content.append({"type": "text", "text": text}) - - content.append( - { - "type": "image_url", - "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}, - }, - ) - - return {"role": role, "content": content} + return LLMContext.create_image_url_message(role=role, url=url, text=text) @staticmethod def create_audio_message( From 5174b181760027434270b2f83ec7a70a8d5d38eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 29 Oct 2025 11:02:45 -0700 Subject: [PATCH 11/20] LLMAssistantAggregator: don't mark function calls as completed when receiving user image Before, when requesting a user image from a function call we had to wait for a random time before we could indicate the function call was done. This was to given time to the aggregator to process the image before marking the function call as completed. To avoid this, we now wait for the requested image to be received by the LLM assistant agrgegator (using an asyncio event). Then, we can successfully mark the function call as completed. --- src/pipecat/frames/frames.py | 2 ++ .../aggregators/llm_response_universal.py | 10 ++++------ src/pipecat/services/llm_service.py | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 94f8702b3..afa647b1c 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -11,6 +11,7 @@ including data frames, system frames, and control frames for audio, video, text, and LLM processing. """ +import asyncio from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, @@ -1218,6 +1219,7 @@ class UserImageRequestFrame(SystemFrame): function_name: Optional[str] = None tool_call_id: Optional[str] = None video_source: Optional[str] = None + request_event: Optional[asyncio.Event] = None def __str__(self): return f"{self.name}(user: {self.user_id}, video_source: {self.video_source}, function: {self.function_name}, request: {self.tool_call_id})" diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 9f1e04fe0..920775853 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -777,12 +777,6 @@ class LLMAssistantAggregator(LLMContextAggregator): ) return - del self._function_calls_in_progress[frame.request.tool_call_id] - - # Update context with the image frame - self._update_function_call_result( - frame.request.function_name, frame.request.tool_call_id, "COMPLETED" - ) self._context.add_image_frame_message( format=frame.format, size=frame.size, @@ -793,6 +787,10 @@ class LLMAssistantAggregator(LLMContextAggregator): await self.push_aggregation() await self.push_context_frame(FrameDirection.UPSTREAM) + # Notify who ever requested the image that we have added it to the context. + if frame.request and frame.request.request_event: + frame.request.request_event.set() + async def _handle_llm_start(self, _: LLMFullResponseStartFrame): self._started += 1 diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 6f87b95a5..c75893ea3 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -492,11 +492,16 @@ class LLMService(AIService): tool_call_id: Optional[str] = None, text_content: Optional[str] = None, video_source: Optional[str] = None, + timeout: Optional[float] = 10.0, ): """Request an image from a user. Pushes a UserImageRequestFrame upstream to request an image from the - specified user. + specified user. The user image can then be processed by the LLM. + + Use this function from a function call if you want the LLM to process + the image. If you expect the image to be processed by a vision service, + you might want to push a UserImageRequestFrame upstream directly. Args: user_id: The ID of the user to request an image from. @@ -504,7 +509,11 @@ class LLMService(AIService): tool_call_id: Optional tool call ID associated with the request. text_content: Optional text content/context for the image request. video_source: Optional video source identifier. + timeout: Optional timeout for the requested image to be added to the LLM context. + """ + request_event = asyncio.Event() if timeout else None + await self.push_frame( UserImageRequestFrame( user_id=user_id, @@ -512,10 +521,15 @@ class LLMService(AIService): tool_call_id=tool_call_id, context=text_content, video_source=video_source, + request_event=request_event, ), FrameDirection.UPSTREAM, ) + # Wait for the requested image to be added to the context. + if request_event: + await asyncio.wait_for(request_event.wait(), timeout=timeout) + async def _create_sequential_runner_task(self): if not self._sequential_runner_task: self._sequential_runner_queue = asyncio.Queue() From d7d409df606ee4637a470e7d6055abec0f95b893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 28 Oct 2025 15:09:46 -0700 Subject: [PATCH 12/20] examples(foundational): move 12-* to 14-*-video --- CHANGELOG.md | 11 +- .../foundational/12b-describe-video-gpt-4o.py | 184 ------------------ .../12c-describe-video-anthropic.py | 184 ------------------ ...> 14d-function-calling-anthropic-video.py} | 105 +++------- ...s.py => 14d-function-calling-aws-video.py} | 138 +++++++------ ...4d-function-calling-gemini-flash-video.py} | 135 ++++++------- ...> 14d-function-calling-moondream-video.py} | 120 +++++++----- ...y => 14d-function-calling-openai-video.py} | 105 ++++------ 8 files changed, 275 insertions(+), 707 deletions(-) delete mode 100644 examples/foundational/12b-describe-video-gpt-4o.py delete mode 100644 examples/foundational/12c-describe-video-anthropic.py rename examples/foundational/{14b-function-calling-anthropic-video.py => 14d-function-calling-anthropic-video.py} (63%) rename examples/foundational/{12d-describe-video-aws.py => 14d-function-calling-aws-video.py} (58%) rename examples/foundational/{12a-describe-video-gemini-flash.py => 14d-function-calling-gemini-flash-video.py} (55%) rename examples/foundational/{12-describe-video.py => 14d-function-calling-moondream-video.py} (58%) rename examples/foundational/{14d-function-calling-video.py => 14d-function-calling-openai-video.py} (63%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07596b3b9..b4de8325c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -216,6 +216,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue in `HumeTTSService` that was only using Octave 2, which does + not support the `description` field. Now, if a description is provided, it + switches to Octave 1. + - Fixed an issue where `DailyTransport` would timeout prematurely on join and on leave. @@ -225,7 +229,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `ServiceSwitcher` where the `STTService`s would result in all STT services producing `TranscriptionFrame`s. -- Fixed an issue in `HumeTTSService` that was only using Octave 2, which does not support the `description` field. Now, if a description is provided, it switches to Octave 1. +### Other + +- Updated all vision 12-series foundational examples to use function calling to + request for a camera image and also to push `LLMMessagesAppendFrame` with the + retrieved image. For the specific `Moondream` example (`12-describe-video.py`) + we now use a regular LLM and a parallel pipeline with the `MoondreamService`. ## [0.0.91] - 2025-10-21 diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py deleted file mode 100644 index 894d70d7b..000000000 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ /dev/null @@ -1,184 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os -from typing import Optional - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import ( - Frame, - LLMContextFrame, - TextFrame, - TTSSpeakFrame, - UserImageRawFrame, - UserImageRequestFrame, -) -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.user_response import UserResponseAggregator -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams - -load_dotenv(override=True) - - -class UserImageRequester(FrameProcessor): - """Converts incoming text into requests for user images.""" - - def __init__(self, participant_id: Optional[str] = None): - super().__init__() - self._participant_id = participant_id - - def set_participant_id(self, participant_id: str): - self._participant_id = participant_id - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if self._participant_id and isinstance(frame, TextFrame): - await self.push_frame( - UserImageRequestFrame(self._participant_id, context=frame.text), - FrameDirection.UPSTREAM, - ) - else: - await self.push_frame(frame, direction) - - -class UserImageProcessor(FrameProcessor): - """Converts incoming user images into context frames.""" - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, UserImageRawFrame): - if frame.request and frame.request.context: - context = LLMContext() - context.add_image_frame_message( - image=frame.image, - text=frame.request.context, - size=frame.size, - format=frame.format, - ) - frame = LLMContextFrame(context) - await self.push_frame(frame) - else: - await self.push_frame(frame, direction) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - user_response = UserResponseAggregator() - - # Initialize the image requester without setting the participant ID yet - image_requester = UserImageRequester() - - image_processor = UserImageProcessor() - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - # OpenAI GPT-4o for vision analysis - openai = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - pipeline = Pipeline( - [ - transport.input(), - stt, - user_response, - image_requester, - image_processor, - openai, - tts, - transport.output(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - # Set the participant ID in the image requester - client_id = get_transport_client_id(transport, client) - image_requester.set_participant_id(client_id) - - # Welcome message - await task.queue_frame(TTSSpeakFrame("Hi there! Feel free to ask me about what I see.")) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py deleted file mode 100644 index a8134d535..000000000 --- a/examples/foundational/12c-describe-video-anthropic.py +++ /dev/null @@ -1,184 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os -from typing import Optional - -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import ( - Frame, - LLMContextFrame, - TextFrame, - TTSSpeakFrame, - UserImageRawFrame, - UserImageRequestFrame, -) -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.user_response import UserResponseAggregator -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import ( - create_transport, - get_transport_client_id, - maybe_capture_participant_camera, -) -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams - -load_dotenv(override=True) - - -class UserImageRequester(FrameProcessor): - """Converts incoming text into requests for user images.""" - - def __init__(self, participant_id: Optional[str] = None): - super().__init__() - self._participant_id = participant_id - - def set_participant_id(self, participant_id: str): - self._participant_id = participant_id - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if self._participant_id and isinstance(frame, TextFrame): - await self.push_frame( - UserImageRequestFrame(self._participant_id, context=frame.text), - FrameDirection.UPSTREAM, - ) - else: - await self.push_frame(frame, direction) - - -class UserImageProcessor(FrameProcessor): - """Converts incoming user images into context frames.""" - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, UserImageRawFrame): - if frame.request and frame.request.context: - context = LLMContext() - context.add_image_frame_message( - image=frame.image, - text=frame.request.context, - size=frame.size, - format=frame.format, - ) - frame = LLMContextFrame(context) - await self.push_frame(frame) - else: - await self.push_frame(frame, direction) - - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - user_response = UserResponseAggregator() - - # Initialize the image requester without setting the participant ID yet - image_requester = UserImageRequester() - - image_processor = UserImageProcessor() - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - # Anthropic for vision analysis - anthropic = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - pipeline = Pipeline( - [ - transport.input(), - stt, - user_response, - image_requester, - image_processor, - anthropic, - tts, - transport.output(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - - await maybe_capture_participant_camera(transport, client) - - # Set the participant ID in the image requester - client_id = get_transport_client_id(transport, client) - image_requester.set_participant_id(client_id) - - # Welcome message - await task.queue_frame(TTSSpeakFrame("Hi there! Feel free to ask me about what I see.")) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py similarity index 63% rename from examples/foundational/14b-function-calling-anthropic-video.py rename to examples/foundational/14d-function-calling-anthropic-video.py index 009f59500..a4daed481 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -4,8 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # - -import asyncio import os from dotenv import load_dotenv @@ -39,34 +37,21 @@ from pipecat.transports.daily.transport import DailyParams load_dotenv(override=True) -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def get_image(params: FunctionCallParams): +async def fetch_user_image(params: FunctionCallParams): + user_id = params.arguments["user_id"] question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the image frame + # Request the user image frame. Note that this image is associated to a + # function call and will be handled by the LLM assistant aggregators. await params.llm.request_image_frame( - user_id=client_id, + user_id=user_id, function_name=params.function_name, tool_call_id=params.tool_call_id, text_content=question, ) - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) + await params.result_callback(None) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -100,70 +85,32 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), - model="claude-3-7-sonnet-latest", - params=AnthropicLLMService.InputParams(enable_prompt_caching=True), - ) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) + # Anthropic for vision analysis + llm = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) + llm.register_function("fetch_user_image", fetch_user_image) - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", + fetch_image_function = FunctionSchema( + name="fetch_user_image", + description="Called when the user requests a description of their camera feed", properties={ - "location": { + "user_id": { "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + "description": "The ID of the user to grab the image from", }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ "question": { "type": "string", - "description": "The question that the user is asking about the image.", - } + "description": "The question that the user is asking about the image", + }, }, - required=["question"], + required=["user_id", "question"], ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) - - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to two tools: get_weather and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? - -If you need to use a tool, simply use the tool. Do not tell the user the tool you are using. Be brief and concise. - """ + tools = ToolsSchema(standard_tools=[fetch_image_function]) messages = [ { "role": "system", - "content": [ - { - "type": "text", - "text": system_prompt, - } - ], + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", }, - {"role": "user", "content": "Start the conversation by introducing yourself."}, ] context = LLMContext(messages, tools) @@ -173,11 +120,11 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo [ transport.input(), # Transport user input stt, # STT - context_aggregator.user(), # User speech to text + context_aggregator.user(), # User responses llm, # LLM tts, # TTS transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -196,10 +143,16 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo await maybe_capture_participant_camera(transport, client) - global client_id + # Set the participant ID in the image requester client_id = get_transport_client_id(transport, client) # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") diff --git a/examples/foundational/12d-describe-video-aws.py b/examples/foundational/14d-function-calling-aws-video.py similarity index 58% rename from examples/foundational/12d-describe-video-aws.py rename to examples/foundational/14d-function-calling-aws-video.py index 5436b81ba..78bacbedf 100644 --- a/examples/foundational/12d-describe-video-aws.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -5,29 +5,22 @@ # import os -from typing import Optional from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import ( - Frame, - LLMContextFrame, - TextFrame, - TTSSpeakFrame, - UserImageRawFrame, - UserImageRequestFrame, -) +from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.user_response import UserResponseAggregator -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -37,54 +30,28 @@ from pipecat.runner.utils import ( from pipecat.services.aws.llm import AWSBedrockLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams load_dotenv(override=True) -class UserImageRequester(FrameProcessor): - """Converts incoming text into requests for user images.""" +async def fetch_user_image(params: FunctionCallParams): + user_id = params.arguments["user_id"] + question = params.arguments["question"] + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - def __init__(self, participant_id: Optional[str] = None): - super().__init__() - self._participant_id = participant_id + # Request the user image frame. Note that this image is associated to a + # function call and will be handled by the LLM assistant aggregators. + await params.llm.request_image_frame( + user_id=user_id, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + text_content=question, + ) - def set_participant_id(self, participant_id: str): - self._participant_id = participant_id - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if self._participant_id and isinstance(frame, TextFrame): - await self.push_frame( - UserImageRequestFrame(self._participant_id, context=frame.text), - FrameDirection.UPSTREAM, - ) - else: - await self.push_frame(frame, direction) - - -class UserImageProcessor(FrameProcessor): - """Converts incoming user images into context frames.""" - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, UserImageRawFrame): - if frame.request and frame.request.context: - # Note: AWS Bedrock does not yet support the universal LLMContext - context = LLMContext() - context.add_image_frame_message( - image=frame.image, - text=frame.request.context, - size=frame.size, - format=frame.format, - ) - frame = LLMContextFrame(context) - await self.push_frame(frame) - else: - await self.push_frame(frame, direction) + await params.result_callback(None) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -111,17 +78,15 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - user_response = UserResponseAggregator() - - # Initialize the image requester without setting the participant ID yet - image_requester = UserImageRequester() - - image_processor = UserImageProcessor() - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + # AWS for vision analysis - aws = AWSBedrockLLMService( + llm = AWSBedrockLLMService( aws_region="us-west-2", model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", # Note: usually, prefer providing latency="optimized" param. @@ -129,22 +94,44 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # which we need for image input. params=AWSBedrockLLMService.InputParams(temperature=0.8), ) + llm.register_function("fetch_user_image", fetch_user_image) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + fetch_image_function = FunctionSchema( + name="fetch_user_image", + description="Called when the user requests a description of their camera feed", + properties={ + "user_id": { + "type": "string", + "description": "The ID of the user to grab the image from", + }, + "question": { + "type": "string", + "description": "The question that the user is asking about the image", + }, + }, + required=["user_id", "question"], ) + tools = ToolsSchema(standard_tools=[fetch_image_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) pipeline = Pipeline( [ - transport.input(), - stt, - user_response, - image_requester, - image_processor, - aws, - tts, - transport.output(), + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -165,10 +152,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set the participant ID in the image requester client_id = get_transport_client_id(transport, client) - image_requester.set_participant_id(client_id) - # Welcome message - await task.queue_frame(TTSSpeakFrame("Hi there! Feel free to ask me about what I see.")) + # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/14d-function-calling-gemini-flash-video.py similarity index 55% rename from examples/foundational/12a-describe-video-gemini-flash.py rename to examples/foundational/14d-function-calling-gemini-flash-video.py index 63c1fd677..a669e1e46 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -5,29 +5,22 @@ # import os -from typing import Optional from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import ( - Frame, - LLMContextFrame, - TextFrame, - TTSSpeakFrame, - UserImageRawFrame, - UserImageRequestFrame, -) +from pipecat.frames.frames import LLMRunFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.user_response import UserResponseAggregator -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -37,53 +30,28 @@ from pipecat.runner.utils import ( from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService +from pipecat.services.llm_service import FunctionCallParams from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams load_dotenv(override=True) -class UserImageRequester(FrameProcessor): - """Converts incoming text into requests for user images.""" +async def fetch_user_image(params: FunctionCallParams): + user_id = params.arguments["user_id"] + question = params.arguments["question"] + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - def __init__(self, participant_id: Optional[str] = None): - super().__init__() - self._participant_id = participant_id + # Request the user image frame. Note that this image is associated to a + # function call and will be handled by the LLM assistant aggregators. + await params.llm.request_image_frame( + user_id=user_id, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + text_content=question, + ) - def set_participant_id(self, participant_id: str): - self._participant_id = participant_id - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if self._participant_id and isinstance(frame, TextFrame): - await self.push_frame( - UserImageRequestFrame(self._participant_id, context=frame.text), - FrameDirection.UPSTREAM, - ) - else: - await self.push_frame(frame, direction) - - -class UserImageProcessor(FrameProcessor): - """Converts incoming user images into context frames.""" - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, UserImageRawFrame): - if frame.request and frame.request.context: - context = LLMContext() - context.add_image_frame_message( - image=frame.image, - text=frame.request.context, - size=frame.size, - format=frame.format, - ) - frame = LLMContextFrame(context) - await self.push_frame(frame) - else: - await self.push_frame(frame, direction) + await params.result_callback(None) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -110,33 +78,53 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - user_response = UserResponseAggregator() - - # Initialize the image requester without setting the participant ID yet - image_requester = UserImageRequester() - - image_processor = UserImageProcessor() - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - # Google Gemini model for vision analysis - google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) - tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) + # Google Gemini model for vision analysis + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + llm.register_function("fetch_user_image", fetch_user_image) + + fetch_image_function = FunctionSchema( + name="fetch_user_image", + description="Called when the user requests a description of their camera feed", + properties={ + "user_id": { + "type": "string", + "description": "The ID of the user to grab the image from", + }, + "question": { + "type": "string", + "description": "The question that the user is asking about the image", + }, + }, + required=["user_id", "question"], + ) + tools = ToolsSchema(standard_tools=[fetch_image_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + pipeline = Pipeline( [ - transport.input(), - stt, - user_response, - image_requester, - image_processor, - google, - tts, - transport.output(), + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -157,10 +145,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set the participant ID in the image requester client_id = get_transport_client_id(transport, client) - image_requester.set_participant_id(client_id) - # Welcome message - await task.queue_frame(TTSSpeakFrame("Hi there! Feel free to ask me about what I see.")) + # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/14d-function-calling-moondream-video.py similarity index 58% rename from examples/foundational/12-describe-video.py rename to examples/foundational/14d-function-calling-moondream-video.py index eb783ad75..0bad950ed 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -5,28 +5,29 @@ # import os -from typing import Optional from dotenv import load_dotenv from loguru import logger +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.frames.frames import ( Frame, - LLMContextFrame, - TextFrame, - TTSSpeakFrame, + LLMRunFrame, UserImageRawFrame, UserImageRequestFrame, + VisionImageRawFrame, ) +from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.user_response import UserResponseAggregator +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( @@ -36,33 +37,27 @@ from pipecat.runner.utils import ( ) from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.llm_service import FunctionCallParams from pipecat.services.moondream.vision import MoondreamService +from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams load_dotenv(override=True) -class UserImageRequester(FrameProcessor): - """Converts incoming text into requests for user images.""" +async def fetch_user_image(params: FunctionCallParams): + user_id = params.arguments["user_id"] + question = params.arguments["question"] + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - def __init__(self, participant_id: Optional[str] = None): - super().__init__() - self._participant_id = participant_id + # Request the user image frame frame. In this case we don't use + # `llm.request_image_frame()` because we don't want the LLM to analyze it. + await params.llm.push_frame( + UserImageRequestFrame(user_id=user_id, context=question), FrameDirection.UPSTREAM + ) - def set_participant_id(self, participant_id: str): - self._participant_id = participant_id - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if self._participant_id and isinstance(frame, TextFrame): - await self.push_frame( - UserImageRequestFrame(self._participant_id, context=frame.text), - FrameDirection.UPSTREAM, - ) - else: - await self.push_frame(frame, direction) + await params.result_callback(None) class UserImageProcessor(FrameProcessor): @@ -73,14 +68,12 @@ class UserImageProcessor(FrameProcessor): if isinstance(frame, UserImageRawFrame): if frame.request and frame.request.context: - context = LLMContext() - context.add_image_frame_message( + frame = VisionImageRawFrame( image=frame.image, text=frame.request.context, size=frame.size, format=frame.format, ) - frame = LLMContextFrame(context) await self.push_frame(frame) else: await self.push_frame(frame, direction) @@ -110,16 +103,6 @@ transport_params = { async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") - user_response = UserResponseAggregator() - - # Initialize the image requester without setting the participant ID yet - image_requester = UserImageRequester() - - image_processor = UserImageProcessor() - - # If you run into weird description, try with use_cpu=True - moondream = MoondreamService() - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -127,16 +110,54 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm.register_function("fetch_user_image", fetch_user_image) + + fetch_image_function = FunctionSchema( + name="fetch_user_image", + description="Called when the user requests a description of their camera feed", + properties={ + "user_id": { + "type": "string", + "description": "The ID of the user to grab the image from", + }, + "question": { + "type": "string", + "description": "The question that the user is asking about the image", + }, + }, + required=["user_id", "question"], + ) + tools = ToolsSchema(standard_tools=[fetch_image_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + }, + ] + + context = LLMContext(messages, tools) + context_aggregator = LLMContextAggregatorPair(context) + + # This will get the get the user image frame and push it to the LLM. + image_processor = UserImageProcessor() + + # If you run into weird description, try with use_cpu=True + moondream = MoondreamService() + pipeline = Pipeline( [ - transport.input(), - stt, - user_response, - image_requester, - image_processor, - moondream, - tts, - transport.output(), + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + ParallelPipeline( + [llm], # LLM + [image_processor, moondream], + ), + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -153,10 +174,15 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): # Set the participant ID in the image requester client_id = get_transport_client_id(transport, client) - image_requester.set_participant_id(client_id) - # Welcome message - await task.queue_frame(TTSSpeakFrame("Hi there! Feel free to ask me about what I see.")) + # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) + await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-openai-video.py similarity index 63% rename from examples/foundational/14d-function-calling-video.py rename to examples/foundational/14d-function-calling-openai-video.py index 48cf95ee9..c6320b1ec 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -5,7 +5,6 @@ # -import asyncio import os from dotenv import load_dotenv @@ -39,34 +38,21 @@ from pipecat.transports.daily.transport import DailyParams load_dotenv(override=True) -# Global variable to store the client ID -client_id = "" - - -async def get_weather(params: FunctionCallParams): - location = params.arguments["location"] - await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") - - -async def get_image(params: FunctionCallParams): +async def fetch_user_image(params: FunctionCallParams): + user_id = params.arguments["user_id"] question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={client_id}, question={question}") + logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the image frame + # Request the user image frame. Note that this image is associated to a + # function call and will be handled by the LLM assistant aggregators. await params.llm.request_image_frame( - user_id=client_id, + user_id=user_id, function_name=params.function_name, tool_call_id=params.tool_call_id, text_content=question, ) - # Wait a short time for the frame to be processed - await asyncio.sleep(0.5) - - # Return a result to complete the function call - await params.result_callback( - f"I've captured an image from your camera and I'm analyzing what you asked about: {question}" - ) + await params.result_callback(None) # We store functions so objects (e.g. SileroVADAnalyzer) don't get @@ -101,58 +87,30 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - llm.register_function("get_weather", get_weather) - llm.register_function("get_image", get_image) + llm.register_function("fetch_user_image", fetch_user_image) - weather_function = FunctionSchema( - name="get_weather", - description="Get the current weather", + fetch_image_function = FunctionSchema( + name="fetch_user_image", + description="Called when the user requests a description of their camera feed", properties={ - "location": { + "user_id": { "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + "description": "The ID of the user to grab the image from", }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location"], - ) - get_image_function = FunctionSchema( - name="get_image", - description="Get an image from the video stream.", - properties={ "question": { "type": "string", - "description": "The question that the user is asking about the image.", - } + "description": "The question that the user is asking about the image", + }, }, - required=["question"], + required=["user_id", "question"], ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) + tools = ToolsSchema(standard_tools=[fetch_image_function]) - system_prompt = """\ -You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. - -Your response will be turned into speech so use only simple words and punctuation. - -You have access to two tools: get_weather and get_image. - -You can respond to questions about the weather using the get_weather tool. - -You can answer questions about the user's video stream using the get_image tool. Some examples of phrases that \ -indicate you should use the get_image tool are: -- What do you see? -- What's in the video? -- Can you describe the video? -- Tell me about what you see. -- Tell me something interesting about what you see. -- What's happening in the video? -""" messages = [ - {"role": "system", "content": system_prompt}, + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are able to describe images from the user camera.", + }, ] context = LLMContext(messages, tools) @@ -160,13 +118,13 @@ indicate you should use the get_image tool are: pipeline = Pipeline( [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] ) @@ -185,10 +143,15 @@ indicate you should use the get_image tool are: await maybe_capture_participant_camera(transport, client) - global client_id client_id = get_transport_client_id(transport, client) # Kick off the conversation. + messages.append( + { + "role": "system", + "content": f"Please introduce yourself to the user. Use '{client_id}' as the user ID during function calls.", + } + ) await task.queue_frames([LLMRunFrame()]) @transport.event_handler("on_client_disconnected") From e458d3edfe96657348df62f028a472ce34b45389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 29 Oct 2025 11:13:07 -0700 Subject: [PATCH 13/20] scripts(evals): update 12-* for 14-*-video --- scripts/evals/run-release-evals.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 14f9dee52..ce380158b 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -47,7 +47,7 @@ PROMPT_SWITCH_LANGUAGE = "Say something in Spanish." EVAL_SWITCH_LANGUAGE = "The user is now talking in Spanish." # Vision -PROMPT_VISION = ("What do you see?", Image.open(ASSETS_DIR / "cat.jpg")) +PROMPT_VISION = ("Briefly describe what you see.", Image.open(ASSETS_DIR / "cat.jpg")) EVAL_VISION = "A cat description." # Voicemail @@ -110,18 +110,9 @@ TESTS_07 = [ # ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), ] -TESTS_12 = [ - ("12-describe-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("12a-describe-video-gemini-flash.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("12b-describe-video-gpt-4o.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("12c-describe-video-anthropic.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), -] - TESTS_14 = [ ("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14b-function-calling-anthropic-video.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14d-function-calling-video.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14e-function-calling-google.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -137,6 +128,12 @@ TESTS_14 = [ ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), ("14x-function-calling-openpipe.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # Video + ("14d-function-calling-anthropic-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("14d-function-calling-aws-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("14d-function-calling-gemini-flash-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("14d-function-calling-moondream-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("14d-function-calling-openai-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), # Currently not working. # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), @@ -211,7 +208,6 @@ TESTS_44 = [ TESTS = [ *TESTS_07, - *TESTS_12, *TESTS_14, *TESTS_15, *TESTS_19, From 3b3a215155369ece0e38cbaf7f1b7acc2fdf7339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 29 Oct 2025 15:14:47 -0700 Subject: [PATCH 14/20] examples(foundational): re-add 12-* but load image from file --- CHANGELOG.md | 9 +- .../foundational/12-describe-image-openai.py | 141 +++++++++++++++++ .../12a-describe-image-anthropic.py | 141 +++++++++++++++++ .../foundational/12b-describe-image-aws.py | 148 ++++++++++++++++++ .../12c-describe-image-gemini-flash.py | 141 +++++++++++++++++ examples/foundational/assets/cat.jpg | Bin 0 -> 64732 bytes 6 files changed, 576 insertions(+), 4 deletions(-) create mode 100644 examples/foundational/12-describe-image-openai.py create mode 100644 examples/foundational/12a-describe-image-anthropic.py create mode 100644 examples/foundational/12b-describe-image-aws.py create mode 100644 examples/foundational/12c-describe-image-gemini-flash.py create mode 100644 examples/foundational/assets/cat.jpg diff --git a/CHANGELOG.md b/CHANGELOG.md index b4de8325c..78620da43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -231,10 +231,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other -- Updated all vision 12-series foundational examples to use function calling to - request for a camera image and also to push `LLMMessagesAppendFrame` with the - retrieved image. For the specific `Moondream` example (`12-describe-video.py`) - we now use a regular LLM and a parallel pipeline with the `MoondreamService`. +- Updated all vision 12-series foundational examples to load images from a file + and push `LLMMessagesAppendFrame` with the loaded image. + +- Added 14-series video examples for different services. These new examples + request an image from the user camera through a function call. ## [0.0.91] - 2025-10-21 diff --git a/examples/foundational/12-describe-image-openai.py b/examples/foundational/12-describe-image-openai.py new file mode 100644 index 000000000..97cb82054 --- /dev/null +++ b/examples/foundational/12-describe-image-openai.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger +from PIL import Image + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + + if not runner_args.body: + script_dir = os.path.dirname(__file__) + runner_args.body = { + "image_path": os.path.join(script_dir, "assets", "cat.jpg"), + "question": "Describe this image", + } + + image_path = runner_args.body["image_path"] + question = runner_args.body["question"] + + # Kick off the conversation. + image = Image.open(image_path) + message = LLMContext.create_image_message( + image=image.tobytes(), + format="RGB", + size=image.size, + text=question, + ) + messages.append(message) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/12a-describe-image-anthropic.py b/examples/foundational/12a-describe-image-anthropic.py new file mode 100644 index 000000000..1690a06bf --- /dev/null +++ b/examples/foundational/12a-describe-image-anthropic.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger +from PIL import Image + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.anthropic.llm import AnthropicLLMService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + + if not runner_args.body: + script_dir = os.path.dirname(__file__) + runner_args.body = { + "image_path": os.path.join(script_dir, "assets", "cat.jpg"), + "question": "Describe this image", + } + + image_path = runner_args.body["image_path"] + question = runner_args.body["question"] + + # Kick off the conversation. + image = Image.open(image_path) + message = LLMContext.create_image_message( + image=image.tobytes(), + format="RGB", + size=image.size, + text=question, + ) + messages.append(message) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/12b-describe-image-aws.py b/examples/foundational/12b-describe-image-aws.py new file mode 100644 index 000000000..1827c8906 --- /dev/null +++ b/examples/foundational/12b-describe-image-aws.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger +from PIL import Image + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.aws.llm import AWSBedrockLLMService +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = AWSBedrockLLMService( + aws_region="us-west-2", + model="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + # Note: usually, prefer providing latency="optimized" param. + # Here we can't because AWS Bedrock doesn't support it for Claude 3.7, + # which we need for image input. + params=AWSBedrockLLMService.InputParams(temperature=0.8), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + + if not runner_args.body: + script_dir = os.path.dirname(__file__) + runner_args.body = { + "image_path": os.path.join(script_dir, "assets", "cat.jpg"), + "question": "Describe this image", + } + + image_path = runner_args.body["image_path"] + question = runner_args.body["question"] + + # Kick off the conversation. + image = Image.open(image_path) + message = LLMContext.create_image_message( + image=image.tobytes(), + format="RGB", + size=image.size, + text=question, + ) + messages.append(message) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/12c-describe-image-gemini-flash.py b/examples/foundational/12c-describe-image-gemini-flash.py new file mode 100644 index 000000000..9a36785e8 --- /dev/null +++ b/examples/foundational/12c-describe-image-gemini-flash.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger +from PIL import Image + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.google.llm import GoogleLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. You are also able to describe images.", + }, + ] + + context = LLMContext(messages) + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + + if not runner_args.body: + script_dir = os.path.dirname(__file__) + runner_args.body = { + "image_path": os.path.join(script_dir, "assets", "cat.jpg"), + "question": "Describe this image", + } + + image_path = runner_args.body["image_path"] + question = runner_args.body["question"] + + # Kick off the conversation. + image = Image.open(image_path) + message = LLMContext.create_image_message( + image=image.tobytes(), + format="RGB", + size=image.size, + text=question, + ) + messages.append(message) + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/assets/cat.jpg b/examples/foundational/assets/cat.jpg new file mode 100644 index 0000000000000000000000000000000000000000..700b5fc922d82e5be23873c36d0853a20d5d6eac GIT binary patch literal 64732 zcmbSy1zc52*Z1ZSl1i79(yerdNE{H5Zjch`?z{*nEr*hpZjf$JkT|4p=#*}xJHE}m zPu%zU-Y*xwJ^c5~?3vkn*34RK);f2ScfSB!c^Rk-fP{nu$bo;r-4eDoR7%R|wW_iV z^p!MN007u>uPp8CA-DiwYv_}_Fo4uCBI0D@iaH4V+b^Z&09o~eVA3jiR!25WPhnLC?;I0(cx?k*1Z`THOy zG_k(N5R`lD1P&0yr1#k3FU^I|Bd;_IF)KLatgle(%Dh(!PZ9oyn>*yL~6 z#oPlNCjdy|7k+=l>V|Zwvob>wgdK=k~7}M=F2L84_{-zw`e6?7#Eu zvj9Ng1pGFi{+(x%1ORox06;qZ?>xFM0PrXX0BVQ+_CCb-{PNDl#X*RZ)7{;j!_wT8 zj|77C- z?+5=jtiO$eP1XFZxs$ma*p(*8WtMgp;B?!WS-M!-+tFCs{jX;D|MRiGjo}{t#n&Jp z$hidwjW__@AwmFow;wv?f6dz~3{BvE2;0J^O2|yZ<2b2MIKnE}c zOo4ZRE#L&W13thXKnM^Hd;;QuWFP~`0SbXKpc1G9nt|^?7tja%1jc|F;1{q8Yy?BO+d{w?hv^ zPe3n1Z$%$LUq`>fz{Q})5W-N#FvIY~h{njlXv7%8SjD))#KUC76vb4>w8H!Y^D|}{ zW+&z><`EVa)>AA&EEOzEtN^UfSYNSvuokf{u<@~3u%)o|v0bsFu=BCMVNYQn;o#sf z;7H);;<(^M;S}O@;LPKkKO}g_@eumZ?4kd|l!vtsM;`7y!gxggNb-^4Bd@2h988Vjo*R4 zOn^*4OCUvHO7NZ_hoF;Sl@N`Pkq}B~MHo(4N;pWkPlQjzL!?RML6kz&LiCFmnV5kX zN^ChD@5wnkYo~D3Muu!N{cvECk^i!NrQc=oMI#4E2c2Mq6 z5mQM}Sy9DNwNh%J)O3n;K6J%&)AShh0`%|b6Y0C@&lngPv=}}x)G=%@ zk}^UWy%>ubXPI!A#F-qKvY1Ah(U=98t(nu9f3iSWcv;@Dq_7OJLRk4&Em_l8huKis z1ljD^ve+irvDqcr-PnuS7dePIUU39+)N$-_(sAl?#&C9X-ag}fX7eoP+01kN=km`3 zpEo`~uVPeqMI zGev)i(TKef`y%#B{E4`+c$WB*1g(UbM4rUDB(tQIWQpY7i)SyKUevrelM<5hmHPG) z`K9d3u$P0<_|h8E$g?d1bw2zsaG>DagghO+YE3rqDv@zC6FYulx@M zYy~xiWQC!lV<>z6jCwy$=d4!Mq{PQC5} zT}|CQ-4i`&y?DJhv4-W0vLHdZ#yGCnbZ znxvZSn@X7`nr@p(n8lfGnv0sl%-7$FzJ#; z2tj44-<8QV*mcoO%q_+3++EYX$^*~C&ZFOx#WU1%-Al$R*Biy#)Vtm1iO(NCi@uV+ znSMxqCVuVywEjW4 zf5`ZV^3n3+U?_KJQs`}%Sy*rQv+#uQ>j=|`-pJ>XpCj+0-bM{Z^F^nBLi=R*X*@QpT@!zOe)YLNi7x3=(slB?8I&2}8JAxyzf5MzX4YmgWF=&yWxHjs zKW~4{0PjG}Am3o^5dTp9Pr;v!!y?12BjO|95ib#4qtMa5F{QELagFiG3H^y*lV+0} zQ#Mlv({9sOGXb-xvte_%a|!bl^H~eb3+2D~ezh#VT? z1N{LyCI$v34)H@AY-}8I0zzD3YKkW`)D%=yw2WLVwDg?xR8*|OY@FP@0s;b0SVSd7 z_$0ab1$gfpLBhbq#KFQLd-#xymyU{#_y3vhz5#gXke{dkGSX83f`^2RhjjM?pa4}G z2=cul^e;j}fgq!zJwOLbWpM!r1PQ!F#zcjnK_I{b0KCIPd4m635|!Yoim~G#F=!76 zx%l5_R#bhbeW5zMZ?Z^4$L$n&AO&Mo=zC%Y8#AsAymrGalGG ztrH9Kr0PRF7P#e=*y?CaWQDn z0pL3+9Uy^^1nOf@lpG?uG6@(GO2G4Pj3{rugnoPZ2~_-O&|d~1;lEgu1K;4mvwIus z-;|03z`(nZH~mRL=>%+qSF&1^49RNtJ~lpyt43CLz)-z=G`>#kV9zNvtGA|dRAWxU z7S>e;>9||u*;vV_PLh}y=@2!ApDab$b{T=1RjrlXxX}V{YHfCF&m4weJ`b!|xb7tH z!glp0?0A1hzATq{fwN>8Zjr?%Uned9%yVy??vnAUim$N;Ax1xtlQNY3)@7jK;Kd_R zva+5T6V>E?&yovbW=%cACl8`|NBb1RxR%_Y^6)l;#F|ev80(MwdIW^2(e$S`8vQzN z_(E_)Y^+#k>NSf@g=?)p47Bw$C?#nn)*5L_rQ-j164zihGB++x(7OSf#9|HJooC#^ z4>CJTv4%g%M@*in8pYI1kd`RD9?GUwd2w!1TkP#Ry1$D*Rmws*2M6iOQkntPoSm8JH4KNLk3S6NkZ3WkY`x|uqT$?!>Q zFHKwms>s z_9_4d0e~l;YZacwvXR!xNYNC)R6?0B=mJn+k~H%4xS^O_7_>AJ01W_GPbL{i$GR^0X}Cqkj2PLN#bu^4oGtmZajz$b+r?S_ z?E97(B_gtB@0>kDzU`{n!V{PLJ1sV`=_W=2s&3rjGny;tk-53$=pNG)rb{Yhxewhr z81QeRRhFu?&vl$v-Dqpbg(_VPc6Y2r-t3e*1V0;a)LA?^GAoOzV05i7cB%7G5MTdV zL8`;E$Sm}$*1G&V(Q>hRA8weBm>ydK3u4O5;{+>Epf|+bj5s#yvq_1FZ9-e;0nb^l zn+HK^hUVeR;@@;}*U$80r#Kxg6tbN?g5r$R?toJR>?hI_3KZV1)-Vo@>@p0;xAtQ@ zHf2`N9TB{Oc&18*!bni^Z~ACd^{>ygedeI68BI{P1uY zvB%q-ZNjo)r?tsiAyOz!95c5Xiu{G<)1PyC&$_^~e?UNz20{}D1VB;2`In#rnUs{a zkQR`_h`=U%CdnK?6DWs{D+d90D0%?+C$mzQ>+Gs5-Fo`LuT1yE4J&-v{N4Z*GyiPz zzy9~jxJbTY+=*_z@qT3GQJz#@>DPUA2TXiD&08^}`NTpKD2c{4L2LP5kI>qTjK}0P zJ+3_ZCtOUJ7A*mN&r|422=ga&CICW*0tw2(fT%D-?vQOBxd zQu}(@zWbzND43%a@qjfg{EziehG}ok8~mc*)nm|_PMZ|(y{uPj*~yt4%*Kuik2&c~ znqD%stNnWKYUJ0(x(_F zbUdu+k)PUJ8q=6xIL=T1OCdu!{3#|;8LL}j9I>9?NT=^9)a~Ly0 zBC@Vrd7U&WbK4gft|;5H`{U66slD4KeSCXW<*}%~vW!@UXv`Co*Wl!%g4bUJ^73l{ zmlE9AXg;CfW72@RC=hT*26yWRk^v1|!vG9IgAWAAK_DmrB3eV7Q{pzh-P_~dYMnlZ zH?eNrvwkfmm)h6g0KwY7TKv!SM*N8PRC)3a@V`90-^#g-%xz)e(@Lv7Id-2- z#I`N*YmH{Qq71$@Rwy1K(QVbfGMFnQ|K<+u^57oHUU!CfQr zIIh8>e}Y0Qm~af_KAZL1AFuOjZ3Pv|U)q&18zl8nQh8Rg+PdlpG9)|u8IbI_@LWP2 zsuu_|_!>A(Zu(h1V5{zrk(Sh+wVxcBhwa2I&6Af4D3nHzezh7EXs=OKti8?hPvMo+ z>An^+xiM#}Vx=dY+@D>$t&+jdz+cqq)=DKZ73T1i@2fN5uF##C@z=%i%#EcY%{e!% zOIfk5oi2XgZ<2Gt6H#JS*;AKjXY{q)XXgE4$0^2^M{Biui-T6J{Uhvk>m2LSzGzP2 zBU$IFrPWB-p+I$Cn1_9aT3xnk5P6+zFMBDyp_JlBe%3Hvy_1;1;bZObyu5WX^WX2caOcd9 zjW>S{TC)t`e|(t@o}d?kvkejoc{|A_be*X{^=X$*Vd19^D^2xSnmTbhu+7!HVgL5BgnL8AF{OmqPOaEKrQ z41HZFh;1J^`2*IPCm675X1nRdO5=NASuy*IYg z7R~0L)0bBty_1Dz$2l*-Jj9p}vo)9*&DDOS6Z`7TcDeiV2i-K+X6tT`YcG=4@g;|N z(zo9J{B_lOd75h*9hq0>CpVUt<+g3JtQ^LfsOFNl>~kPa%|?IX%Vj>!jZhKh<+2m3 zRA4s6WF1JU!FSz9XpQNw+yO}!$;)K^hXWH-cC4u_B{lr7m~C@%Mbix(+Dvh(rO!Q< z4v6Xqws(ZQC`rgQ1sW7moe^X4td42%?a9lo(eGA0v??g&zR#yA%_1mwQVj_+V8_y* zZ1R;wH6n#+F`pg@T5!c49rJB*{PrOfBb7D{-L1>ibZ>{M<@M$Xi&3wgCXcdLYDYSC zE7tO2Jk~8^K2L24NwNLm^Q=px4|f=vvy=L!c}8Y(EW1#QmGEGly%jyawBb{w=IX#M zwQ^5!;d1J1F3J9iYw4HKaQ{s=VE*JMAhSgV&;p)(TYFE3?b_<1#!RZ<5!1O%glxjgrNL4|SDv zFW7jh;(R|S-T^PDPqSO8(y4SxQlL-BXzkwJFO5G5T~GA&AuBoHXW#}pukPTP5Xs=ifmKnnTRf3rqy6!B83N@^pjn~KNOA~jXA-N$JS{O+zSWs&yjnwx1>-XMLpmkOF4EfVk+cJ6?<*DWUOt1jM;lWrPj*}Pu%$1)?z%2um7 z*kmkA@@x}E`(w)32Dq?e=$^jD2dyxG4jtv!n=H_Rp$Q0l8325ufk=Ro9n@Rz_Xz+S z35W^!a*qHcl$f7S$;#jb(se%vs^VvoJ9CE{1Wkn>Np<;r-F>yCPcUWSipV@{N{MQO z>xfb9S*i9-ne*R7P1w41CeJCgNb`R;i7_UyG3!4M?_Xu?=@RlIEQvEG<*)QvsE#kF$SUjO zX2rKo?wd;>PS7%*GKnBzx=6O}&xw1E3U8b8OiSZS7s8}1ZgA8FbvYUR#LJA`+aHl$ z<{G#^N4FgfjuQ{4kIOefhJ*OUQmrz;C5M?7!Nc$Pc}p7#|+ky zmj?QT)r_vXVTD@6RLSNKT$Sa~&FQ&RCFheLMOY*#8uXUwzna5J=So=j zSMr-GWp#6$9#J@8(~dw)i%B2qs(g4?a3+th*GDmRGEJYHbv-;2jp*pRNf~D96pe@(9-yJQR|k9EmKx7`~;F6l`CD7Hb}G>|NmEdc3| z9`_r<#KXfPX?H+txV~BNNH)JFC*4iNqBozZT{So+TJbVz2%&xDQ8vA+Gd0gNS5Rod zZ2ab^a$?I|j8x~bL&N?d$7%R)+qg^@%LEbYp;-r7@1Rsh!F7?5?*miAi;=TxE$xrU z=3Kc|mOBCsXXA@<4G3rR6K@QByCyeB$rs}dZ{AxI=`M=wJ6YTFUwKC+M@nmHq*y0@ z5I1sbxs|~!l+jTu(phUr8OiiDLv-^}kQp$&nDJOUAaFaynyKBINbup9i%SI-L`(tYg_+9K>qOKuv=pY{h}-v0 z6*JZBTZE|CGD{q3@_V_?juqYkd@CC2mOmD$4AlAzNBM@rH(HVo8-@#XZOu~k=DB?G znh`hA248kNBUh4rT2v0KnqHK$onLF;zGGkBg{Iju+78w`TG}}kmX6ZKwanotl)uua zwnbPMH^r{$sfwvh`m@G|@k}vxgVNpbw1H}&>dH|^*9is_dUZQn#wSW112Y%pDV?*KYGVXo!(7RnBUq_pj*Ac_kza?7;j9 zQc7&n67&kx<>hh9(@8kb`1J(lgjQ{8{WYmanimQSRpz(#VfuIn(7K?hT5?aTlb=~( za|v^7jtb^`2KYw|$%zU#@#>tTdIhN-F2}!_*;5X~PW%&enpC={$rJ6eyv2+4iPg1U zMwE`W<3=S)w;zx4hxWz8w#9chtS)U!_Y2{S)Z4c^OCy_@hMJXLM)Ur z;m0a(KbC5o3O2FNz?S(Zlya-nP7B4Eov!6uat&;#G}IJ5JCpo>*xLt>SKJn)td(Rn z`!Z~b^D}5A)TuTIGL<=3>S0z^G-)Q%qul|!+A=j>u>Lf0oZic_5$FBf$3`at<=@K6 z8uiXFIK?PPOS`o;aUDD2bnk#VQ)yVSV02L7V?NtOFM{swisV_{B21^5nBn2^Sh&=y zly+Iw`g*N8?mW`%%5kklk!Vs{I$$G4U!s=k*b*Q?x!BpF9#7$4IKg)O#<%5<{t{Y;BTGPa1Z#hQn9DZbvUgm{adA;LZ@oB19I(>}s&ye?$Mh zI==wgtcqO{_whiqnRH~Nl^8TxQUH$g92NCysp>B~X45j~E2!pFE`PP+epoivWAlwze?`U&UnF3`kUE9)o_RG`wUS$dvild zHoA;(XR~LDKE+gp}D!S*y{T7g43>y$$H5@EoB%-psFZC6+_RDJVW|)yb!_;g(^BsaS=IX+AzWoxbiH4v`MV z7Uj6&=5N^Ln25!619$P43(2A#xJL>20W2Uu*g|aEXbl{m*`T4a@ z=_B6FQW#g^wLF*praoWeYdhuhPKP`n#P5nuShx*%edJ)~w7l4cPLqd$KHz4SeTM%n7YT=^ukKvt&K9%!Cd5F0*RJ>e2kTKg0gb% zSceLB=T#J}clB)og6_Kc)5&hvqmVN{iSMxY1-7drZBuU??D%`{+ys>8Vt3*|5P{x2C6eEV zryMglHit34`p|Z_Vr17qOxVxi!&tjvd)vE>Uo}Kh$`O5UMzxBy*q3o~#<^E_gk2m? z#Vw>QZru$iem9lnU-ccHrtWq9l-(qrD^}i^7C4wq6cqb(3g6x78MU&Rux4*l5s;Z` zWtdQw-hp%`{f14I+50R{8^0}6c&^9P`f=gacK)l9)Lgy`SpntIpEZ2&R%#S4x`red zbHm!{{T*A$JAh?|e!51r;%y5LVO!a&Vyxe`i(1qt0{Q#gK^Ek9K%^72<)FztaNU<8 zZ<$Q{Qby3Gea5Q6pRxc*I7n;O(Qe+wvIWR5>0AS#CuakdA37|zG!$pBeyu3e=gGhkt77bVi zfYuRs630Kq`LC0>e;QzBTB|>i5(56L^q&cJ{y(g&Hp7k?D#pL;uPe~}s;KV)SU?dB zNQR<+LIOgOG1#MI?KU;DtJWHS?TAU+D*CRfMsX#R&r+lB+Om4}nKDW4!xvkfzJF^h zG_ZK6PURLs++BBJ@g*U)nmtZLP+uo(_8>|%j>O%kKIed8{F$Sdg*ExR7r5OdgbQ+6 zC(PsHj?_QaH`hBYP`F9Ew-t|Keikyo7E?Xxo#{ z43jV;y0E(9QZ^bR8RcWrU{Pw;6o<7k4fV^GzpGnK6`)_NYkm)y)HN@C_+TdCw}asXNMWo?Mj ziK(;VWtw!21DvmFP(apEi^%j~QC!e$xDYXB=8SG$R`~1POpG+Y68o6*hrH)7^Sp|( zy(vf2K% z3=!&A)f}$V`zm46Y*q}+g7{h` zdy7_)D3ado)Q!E-242bxk+cDk^QCq8ft_dB4D>WY|5~NIo9@!?Eo<-0fN81tHQS0aEW{nF zXA-AlK3HsKPyMu0E%IH!h0P!s)fzXU4RxtUv4gwG z%%f0-W%MrAggKEZ^2o1+jARRo5uS8*<(j352iyh>6Ig_^!;bU`c|E^K_+EGVIz)uDf&uPBj zJFHnzx>Zb1Ig#&WUZ3c&W8#gwV})3h%S|%%PR9+W4HZSLg%P9e~q(3^i-Ok&pTvBl4`~pwm>; z!FjcJna>fPHj0&|V6w{`sg>Cfkf_n#tt~Id>(QYx6WS*{9 z?jF%WDLdi)a8PV0>N*N0NXB*DADk2F02=zp_hcR_7( z=pWi-j8UQl8W`{W-;rPXUv})hL(k^FI`o!lfCBIk7K-)~w4hPsB+z*W9%Iwd>}PN` zv2GHvS{HF82)2Ict|3x@y3DS!gGQs=&7+w{>7t!JlW~ViA3eRP#)56Ons&YyWY9tV zC{IN#JE0}ZbefzmhZhZ$w!Enr$`|SI)?lyEFSBJ4=Jv1bvh0e<&IOH9~2J&SmotXX{p=j>@b zCnn6@Q*f#oGRsE$dQm%rqFz;H5PJS}dYhy{ZNu)Nb|T7zG;KwPZTxSv%&{AFlg)C@ z(EP0xMJd;gG%ZJgK@tB@osBoE>BRY~u`L1{hfAw$mn{+2BbxbSOg0xHt3J@&nLpSm z7I|moF%48r9O8_>yM~I`x$PjNrOO{`iSDa5Riz#=NtUZ$TuZ-?7DY>f_bS0x{TXQ$ zTN2e%^Iw)IQ|iv{_>w#H_E+@5n5=o6I~Id-CoCG@AY?M2LOZGr1@(5h1Dswf28b;K zH|h2<0SgZ1#XDd&si%@;^t?o~JG=dQz;R+cr+#}e%zN?8L@k>y>sLRWDXjSvmBm_t z#lE#WV2)a1XMUkvO*bN^D)M?L#651hs7X=mvd4Akd@$FDiR0&KUZUEo1!t!p%jwER zH+}38`li3t7ap@Ja=VzvdHBa>Fk4T^^6GR(SsuMQY4laC#mFDso}yiY-O z3!mc<`J!eiiqDV7;}Sf%0;?=CEKBDzNM!$5u^jOlpZKC%7n#%@ReT;L-nV9(Rh02=-bDQhWtr^(oT)M3I2Yql83054< zO_aLQ$!)!eU3E9kP9=F8hs33lSoL|1oDAtMxY^k>ymY;(+?T$0j9VCBnKFJb59Om_H#n^)onsc;m*m8YH z)=v_`?8>N>uQIBYlbJsbCuOn8?r2{Yc;vV||T82o&Yl81_;k9TbMp;HR#z zu!v_>1GO&92$rp-RCTWUdfJITw%($hsC3P8vMrY? z+*LSFQ;jBA8z*U?#4#tg^6{D)EGE9`6i_=mj&AdRk$fSsZsAHM#O&>GQA{0 zB_h0b<`_bZj=TyEI9Ey?6>h?l4eZiOHwrnK72FR5E+eID)*FH`u`=8a?*PhyWW%3v zPZCKLZZ~2P&UKo5;|m=cc1C0DQ$hx2JFq34p-h>!94lw~>Sz<(b^`BqRl!@8urGw%!+(|WiG75Q zNb#_YyvB_gOh`22yP1H%sZaEi{N3>R;bQw^O*dhFBeQaOhZ@iyn+w}!E=pUT^}-Tn z9$2U7^THuOh?r81+mFmFXeA`KuMBLJG(GxwT`LD*)XYS@Cs} zJEUgw79*~oVsb3!Na9Z2wl#J&Vu@s*NIrF%8}!AdkH^-cyPl@Zd}%cf_s4@b(`5~) zOKOlg?()OMKCqlDE#$o}v(gP+^yIKf5q>*0)QFI33iqaO@*7uaOk`eWuN$q>%dWCs zZaIM7=w_c}+zjP^?Jy9!MAs2;oDyT4@QoS%%}AQ7Z!WICpWWYGTN2y9Qmmh?PW*hA zt)qIKH;R|~x^k*PyMB7tK@l!o!}93TdM;Mdlo%m7jBr!4WPUYc4E1=iPYWYDsS+x!?P+a$dF# zN{=IYm6cq5Grm)?W+vWny4~b$3>P>lVSM+j3*Hxc7>=)x*S^irW~!j%^dpzA`D<6K zw?-?u*;t}TU##oJl^{92x%V0KY;)|KcU#5ShquLUk*tq02%g^oe{@bop$@@3$Q8TM zoVEH`o1y+}T59UYm!U%sC)6EJhz~6a+mtn|$wQ^gd&TvZZ+c@9j307Kp*42^oRi?p z^^8C^1#Wdbh-=GSH`bRln8w>gLTuW3U8#2!ty6%|BZ*LZkXjevhpJGJGSj4KTCBqm zzdoPb7-xg%<-|8wagRu>pRujcqo3Ch)I9tBWJs)_qIg7uK6HGM?TCe`uWD_b{M6gg z{P!Nm6ryUuM0eR_!2@0CZ>&+#%Hep&Sl!+Z$I;yS~I(UwRwiKl*s8q;J-V&)IsLi&iLiBjwE zu|!9n-9mFSX0u$GLQ$4j?@Yr&-qiT`ZZWF24>$7rvkS-W0H~5~68}&PKZ!!!)<{Lk zj!NC@_0(kcrXMrDf24_hNvL!0^CB&SxQmtgq%+kxcaL5>^408#m6Eph%uK9d-aKYnOxNU0&xBhR2fmWNV%~ns zL%%xx>KYD)1G-#l5!Y+<;o?-v;Z%n6Ob#{Pey8Z`)Z!YB zC%qp%YgsB{?!#L&^u=c;IXRVy`W6;lvEAdUtRO}{EVPbc&28P8fl-^(&1$@UG3S|{wuSc^fKZ)3bCV( z+Vc&U<3{;hXzWVDPLAxMLQy~zErxYdecgG>0p16 zr?qIcA_bvF_Ci20w}WV<`6iD@$xVlqk+{t-*S$0~e@;4QX6}SxLa|PLZ?r@CVKF=T z$w?QVOhIX}{__R%ExnO7{K_%no#Y={T6aK|)lXRk(>guUDSzG<-m zzJ@e^Yicw5*l@7;!=|+cQ{KvQ6w#HRxtF(;a(PN+(UWWOTPO#c_v6*Q@2-s1hjoZF z3%DZXg#4t2O?JdN<3}*~(oCv~xws>5bhCQb`3EJ;=t49qa^kF7>GkNoeX!u_o^!_A zt#aqM6W%!r4=sk4Z2y;ur^6O+H!Z}3e~)eF-iEUIt!%END+IsG*3s3Ayoa^+m zp0#Opbo6H8@M{nvgzNCm^}_w<4^5g(DOpa`A`EHuRSyFN4Kk;b3-~(jFmPQ`pnp? zI(eq+%a)xJ;TWY%H(}jSHveJeqzVSBROZaK$jLP=K2O)HV5!*h6x$Q7TN{F!AC!8f z`7h4|{brXnsz|BRpgaokdg-|k`>B8S=Fy9&RR6K{)!P%;iu@CN7w5mC-LM1Dk!b=$ zd*DEz6PE-;5){OL5e?`8zei{QdH@s*4UzzHKmZhgF8O)h!oBZfH>;-8S=TfE;Wog$ zZUne~-aqj_5%MuOr(n>r=?%CZz^LQSKf{s57jD6DWFh5O0pJPpYZ_#lzYNPKFezmH z82IiD|A?@V$2w8m@Z{T+ZA}fPR>e2o$<$;M?`mRvtaiU1#kW30;rMOikigZ@OtrpB zVqtjV6znFCHxw#uHxaW|Hr%c3tda#+G8&-oSvfKGQuneT_sCowdYl?*!1aD>{pWGZ zY`jMt-9mL{5u(}Y{By;)R*GktwtXVEfX*17?&#(!#cqT+t%j0H`gJ1gd0t{Q=2UfI zmg+*EoBQ!4E4k{}mZ_9aCP@+nskJMyY*U%Rs8{uwg_Oc90&S;O$w{w`&Lt+Ic6Xw1 zc=OAmYCLRcBBkB@qZOl>N3I~#p*&u@AdwYM&2KWci~3p{-N`wNN51?knb=i!ejH`2 z5%s&HdUbMrW;%!kvfN{@)H`5|G5$bcxK0s+UW~b>sc%p;H~PHJBd4&ugQ)J)vA%)M zmQ{#Gsgm`@n>J@EIEHFb*0)ML9bEe$)K>YR+~vB&0xiZTzo#xGVrrhQFy zDPrLy?&K_uoM_JVB@z&Podz?4b6C$!%<44V8nO#NU0tR6AO^RxX^#yeIWc7mr`T19 zKY`Y_k{T2?vYUA!(lIj7t8Zad=ZI^TS>hQVi*GZ>sgK~wm;{z zhUpCsRNuoqq7U16L3NJnM~m3|L;vcHqIG??j`S_;{m!Z?4; z!Yt|zAPF|!8-Cc8H7>7ft3~8^`fFzXz#g_UGCf*9(UBNslBSI9S0@%sbz{l}TeH&o z9d|I=q$qUQ;BN0Mow&b~e$L#X3RB>OJ%!f^Kt)p)Ci#->+?kb}d};T2d;LTdQlR9# z+_yx{oR1!%)i}jlmU(EyiG~qdMDd=xfh~s`B8N+ZID}RdZ(`TmEvzo36~#qvpOC2~ z=G*(1Wgqa(Fb&8I7Lu}R=(-Z`H`HF*Y1M9&$~t4%ntCAH!hT!FIfvXb^fEF_oqt|Z z$Vk#vU345`X0sT5F+n_YAdBOT$<=Bw>oIa!%3!>}w0hX%>{PfL&m`t&&TK?FMNwNH zmoQZH6jsCH-E(B!xhqZiDy?R*4JT$Q#o%*LyyNA-s#ly{n~>Ew71icSST3c4VwaI2 zySTlK=J}pPab8_$0Z*cDI&nW+)yBx){xs|k*iwlf>k>`j zP5eBEp&$k)bEC>~-1Vl6UuzMe_dfX*9qi*^K_=|to|A0<2mEM({dvBT#PDfIR_7g{ zGL|15t9a12FIO_X#;%^Xwu&~Rl5Akk(ZD*p+`N=6xL|rNk=W7M-aP;6M2odN?0s((MeX48TR%oF5~v{m;4?ce zz2z+b(DW!LS1W<6b(^pCzKaHfgJZpLMX30aP6TwXx&Qbk!@MP%19L7mZaUD=&>sFY z!`SlonEo=hql|Kt?b36F)Rc{wk+IVO9+^3@;JOFFtQAWwx5Gp92Tx0_YEDl5n0~^Q zHp(e)4L{vpUw<6gtTFgbUHSik1g3!)RzVo_f{ zEt`A;2dh?650k24S`B%8*y3fgr`{3YVBAN2Hj*_G=DT`0RKIPt8)Q0f#5O?{ev$FgG=D_t`1hzu zeZl%;G#Je%; zWW6>u{@n?xu^K-5MQIILs&lxl<*HfL5W@zO7Bzo*$n&|qMsu^|u$W=l&1d%89I|-` zJ+gucHffH?y&-;K+l0)%trx6fBNK1+SlPWoMKdem2wjKTiN>GM2~?fnmO0?+`^j>e@!LJ7E!O#e#?Bi*0kf)E|| zpE4XQ0iFH-l-l<}hW`}jU(5TxAhZAV+<&Ix%GdWlA^J;Z|Me51e@gS5;~oGVR3?y6 zKsN}eBR~NYQ#y-4a;C!tmL~q@LOs-7YtuEaO)EdxRp=BwNfZU2Z@EFfZY0~bwV-Uj zu!5ppM)gkF$0;(#wphVcUW59*)ak{7c&w;0>?4`tV<(N(lJ&y0aV3()hAZw^8}?|3 z625hRUsJr>A9EDBA+>_zr#6VicPA=Y+@9sX*oF_J&)rW+DEP?_5&ar@8d9ze?|VIk zbt4}u?x?gWoqIZQBp4KHW?vtA97=yS__*IeBvmF@_0r0bJ0xv}&5=%A%7>TSg98;q znUfv9<5FECY!&xgKFo1Z*{0mMUo{)bv+27Vp1Mkn?fbFwG}1X`IXDsCww5UF$T#3t zysJo+z(H8Xd8|dTK0UA?H*xx2zJ@ z*wnY`<3|pRzugfpXWa)2ucln=QUWz-)^cIgC2>L%%NT0D7onbDXaSn7kl1^dI zSId9Biy_A5(5)6xvy#`A87*3COE4m0(1-+4`_9;lqC~{5L=b!Id!KZE|Nnje zpXZ$WKF{YI^7T1~b8=36KiBttz2C2kA&)`M*O&Ti``Hj0VEOJ4*?b7prC7-%u1>s` zU%Gdb|b6aEN`Cc+6z)m8Ktog1%SE_ApuF7g!H%f*21zv=%Pw3iUD**yj#|vLt&bEZ4 zFSydI#N6TS;pa!H*@4;)Tg^BvxRo)E9>uTT^W}2qptNjzaY5Adio#H}0lhu)9232I z(&vkv5(J1_&e)MQWtV~1aR5R)Oqh5hnGEo-$Pr2p9+M8pG5s-#xw^@7cj#=>(ZMY|(6t$=u+0zof;3 zidM<-alQ1Ku8wpCid0fTiA?q8OiSxGE}cs?adRuNef-w)nn;xS@KT@m#o$BSDkSe^ z<68q+23f6+9;~x5i%Dzx#-(rLvQ8r2W$Vi#y_Vl8O#)8;dbzTd`hv3XvkX|fka={8 zcWQ?C)C<)zSJr5;-<8(|f(mvcOz_Xgra2FFf2%!wnll-Q_Dc@43HQ!Gt_grVB&)5* zi=|J4lJl!-;kU=zlH3n;s`jF1F~X=AD{#2U*6|xc#s(p!BdI@+DQ16t#;bFf4TzPg zdpqg?U6#pfuntZE^lsIP5`;QHY|i_cz3diTcSBS97a2=z^3HtL*B3*hSN45juh4(a z958YLw!{>#*+d>jXLQ$G!9V07^?atWJojcmUpuc)w=ntl>~dC>tk+1UEkL!sSyx12 zc%c~|H)Pa>fm332Zg-O%2bY~Qu+?bPY0j%iF{bR|tuzX-@ToT{PT zsce6>e|2FwZW8;Z`$`)%rc+yQ#X8mPOMwFpj$mXdFqOjQ;ot|>f61_|SIm{1)S4?{ zu$8U{@~v%o!-4gUlEMk_k85?Z8=d^)X6ji@=w|-lD~ z*d4F3^Q9C>msc198&vBf>o<=%6{yqV4ML7>dbaC{L2he)twg^Ru0`q4VS%>BLnddC z6WUh-`ntz^E2ZAL3NI(T@0VJ43bNjwO!RFsEWKVEow1YHZkN%`2QV$8C$Z*j3lll=I)YC^qB5wohdv3UGTZeA|T7iL;&f@8GSG1Hi*-kOyz zvqwM+Lfqd|H~5chd$w+Q=l!*ekcd>`Es;>*XZ2!c7i0~43Y&J}R93DpZKL(hD_BIx z^m`}PNClN-B=ay3xWHV!Syly8G*?gahX)$V=gfIfW#n!Mwi{g9o*$Hl*;MHZaB|{hf%A6HDRL)S%wymD3k*vd`(per_nlty6%Ey4QqA@uI*op(O{x z5YQTW0npjh`eoFDzu1&p?F}F-TXd~&#n9fxirfgBfnJN5_f;Gu>75^^fNDeKy}Yn( z4viO!U@6e(v~-2>JT=JkhcmX2aW~#r2*jmi(4h_^@P^wl)t+pShTBdhEA@ zI4_*VF)KXgV8S?7_U|!M|S{PKTu z(wzQ}yaHaHW2MRX?vU}N-Tl&C zeac>0b&pt8moBYtYTaBe>1N3|VA4#HUXkxw+Qt?l=x9H;Ukf8kp2AYsrKODo=?+ib z+m+sSEq5Fkf*`qP$490y!{a!XYpQ>g$dHyy~2SGz82L5J{RpXeX*F%cMRn_G46+?(cAAf-b#JzBHg!* z%P75*QmZiLm!^E`Gtn;Cc@IA;acwWoztO!QTlb*>7oS}R+v1e6m`VSPg-QB|@Ae@H z!c#KPGJY!N6^NNOR8o4@PL+4Vc?|db>AdfmwtAhYJ5DUC&Nb5pu4;*VCMZc-A-rPW z=fdT+CY|{Cms54#WE>en4SDUv;*k%unouF{TZaAS3oLwG;}H;DS-{(EyV*(W;2Fcf ze0T>m>@sMo8+(LwYJlE*S{gW=Zfr8-dfoMDky^P)&wd7zdlJ_*R-avN@XR=L_*P5J z*71{KV<^}U;dpFUIJi{VRQx<(F+Qk4e=FEi<}kowPsP7xWAu3^hW{Z9DD4VQr_#N@7mMT7*dZTWEBQJ;8_UZ&r_LE*f$u4AXpTsyB=piw(KjwL3B&ms zy|xMuY_Hy#T$#RD)yQs**xemyuW+VoEiNCU7}u0i261l`q=5kaA4Bm^K(L8R?W)kg%KSfNOOqo zH~Yp_!c%8mhfw)~vxw1#b?SVUAF5p{t4CaT3(`fzIrvA&uH}u-+He~EwLp@8ZHjU1 zjBj4XZutOMnTFQL8+7gDJ;szJwhNAGSP0v1-Bqum9He|Y5zejPnREGmvIe~$5@bME#PEACX)rFD~*vgmLX zk9Z>kbA3{ERWz$wpOa?30{>^0alz>W-E6{hV;)W$aLTf)BN@OjJUIJe>4VHm;FC#{ zioC&@j84@wXj>7Qhwfi$_1QWZftQLr_**}rncA`d${0;N^wYR+#;JhL(9pb{?qD&GVSgQ zo{sFaj=O_jmk%j^W&wIc6u}|?GRL1-}h zzmDMBTrY&SWaZ4C;IsX6%eIR@`glP@eRmupLtRm=ou?+@Kd;`UjRHCbDW3+L`|a~% zBK!2Z+7o9Czj3XF&9=D*SuOr1W<)spV?iO4&EAQKxXJuHey6wEjk_abzip-DIT}mnpx%H|9%o zJ>@Il4!f|?*_J)gQe>P+GmYCXTflu)ySb|Ax)#wr{ zQHga_wyUCt`6y1UKe}J+Jl59M*HqTR8e<==oa|wj#j|SGH%^!~B9x|G?$%?S&R7Ha zyALN)yk`3D#VI+HoIx4|4&8TN`+1?VNcOc+RhG15TN+e$+sJq4CV6#L$FZPbU(@=a z>-6;R#QPWOMl`b>5hdx`skU;$z5-J1pBG;E3WrIX#|XLXuck>_1uYfTMb&sp(LZNM zxzO_W>|~|pynsN0tnp>V#&j*~OtnE*ujCon1Y=3^z~6m6^3T;5vqewB+T8(Nv0czf62 zH%>y1!wLL;d~>1&lDyxW`J98COY#q{AK!4cziSW?k2x2d_1}GiE7$**Aq@XL^Yo4D zKcOc}iT{f`>_51N|D>G$-|nzI>aSdn{%u+OsLb__@2BrI&6(fZb8ia4<{II<+CXJS z(ncfd^Tt-wD+|quyE65X?{^wJu$o2M*-D;Ul(7M*erQZJLDoI(%L0a4C2n2osTkv7 zf>Ov|!AK4T24!lw#{0Hcr5ky8qe-NOL)chw#IK zo15uCO25aPyH{}f?F{wWuz|i?+H;FkCvzLM1bwWtBkx?dA9b=P<}Sik1Q9!!6oS2z z9Cc5sebLNYWz5EdX1roN+Klx5Tq~s8L&?Ynd}#oPF~DeLbZ^E^P|uCLr$zams%*Yj zSq4egvHp0G`Gb$C_NaB!<~J`~;+&-``H{&SrVw&hKKQ|bl>*aH<8cKfHS^3Z!TanU zjQM89j_T45(9d1eI^EM%vXmUlToqFvKa3YnIqw?c1s|?4&(cGwH{_0a_WeGDzf;mB z?2A3xNlM1E!;eF+8Te}Hy-jJW0Uo7~TQZ8>Zb^sYER6kv6yHCycy5)Kzd=d-i`pwI z1GyB~^@v41H}KW~tbefnUf7%Ao&prPb@`VbE#M-!#N6yCon(Zb>{@GxV~xt#|8n6L z|M7>9-b#ZO|Ln_qdNQLw#nG%+JpjriDTzY-=q{EC4(c9^+n8my>|*TG$hRlGD`eoe zixc`oJil?(WG{=(5)scFbIWIxMzKir{emiktv7qneLt!b8xxY1wqjEntv~o0L0OrE zm=TY+>yffS6qg-+EdqR7OCGlFP|k3MGqdj5BT)eAvNM2x=I7SX>wVIkM36f+^n8zB z2Gl9}DJ{8oz$$hNg?=?Yi-<|6gZey+>ytOh#jT5dXuK41aYtC2mF;_PUG{Z}ZPw?% zIW|Q5rB@xHce;bX2p`Q}y+Cn6EvAULpQ}||)1^=7oLJ`v)JegL^3pP9|M8B3y$y9Y z@*d~~G6?wjV|js*V|i)Egpu>-*j00@$^5ksRXZ)=N$068*;$Q)Z9kJ5gd!+BrB6z zZJ(%YwqNy}glM7Z#|FUWj!pr|GL?p2H+`_h7|A5YYF&V5bKxkx&nxI{#Y`A))k39} z-MB(j9*=!*ds`YsVm^)`EuRs0m-nDyxAV@ORayTaeODv3xJHs%;^%f+vqSV{M7`B< z4?l~>&Xo94?YEryxgh-Ukhacj4r6Ysf0NtBDfe7Yq!WPKVe(UlS45WZXg(-fD=1sN zqt;o&z^WwxO_?t;|3WByX{4|1!+Ixgyypv^Vb`k{ittVXZL|bo{`P45G6mpW3dGSKlBKBq!6or;c9skwcqGibk% zEdaZa2}=}?Pt)iWx6vDc3qiqT+FDBT%w|jfZdVmd{mbm%L(j|to{ufJY@Q7_HuJMU zElG?2q9sr00wfp}cj2l`Qpqt)30^g3uV-E(w-|n$%yV$f(Uk34AqS?Efb>!!c6pyb zRpk?2ZD!7b54E~_(4&0m=9U-DFEs~N(09iMe8i4imj(&>*(2@4&zmpZF5fO(kDe{H zKbyY<=*Qpqgj%qXt(IIY~xJo=rA6@nV^f z7-i0tqqLv+QuR9TNp=4}#rd$j7m*({ zUR!a9jJw2lPR7_0j$~SP1&WQ0?=4e3BkR_xOSLbq(EpptoPZE z+$JI|mhzL5;igjxqiSzv0C=}5Zq{PP&_+$ScbnAn?T|DG?zl%YXmu5+z9ekaqYhdc z#@lzk&t`e7ULL{YM{3C=PeYF)#^x_gFYtN$yGUa@dGVWBiqlFH^BvJo+)Pr}wC{!) zxgX&FTr|t8t_K>lw^ltSatsn_VR(N?&xgTvY2hV%>X~m`9RgGfq{UzQ`l`!V&z)~v zLSDV#r|UOl=krxpIQKK{V7m_6#1B<0Wq5OyjB~6Aa#MAw?q*|egt^r(GGL<)L}``P zHsZummHNTN+BcupS(kV{Ib!Fq;B2=Pa<2~vIPdW0nYUMX1}|O5FQ?8u1o%*y?VWs@ z=ObUrw~4(IMqHQyIc7}L`FI6GTNYu5lH^Af?i6yD)n z;n}p010YCyDz281J(MTwZ8x*3I{N5lZat5pg_*dv(pfCw>=Y+&l>aQJ!(287SHg~aseC1H`jo;mXehr09Q-|slZ$4}oC3_d{)9`C!T@SEfx ze{-x0KYp)3I1crQAAUXeZ!yA&K>l_$APXPX~=O_e~>&em%#5V{zIn92}Zp`ODz&Qq<7!D(rKT)(X{4{p;*uyj!NO11S#} za61PR$jht61PC$%zwX!;eR%DRz8C`P64!$vhitSfXYkPMjs{C)HQnj0%9czgTOJCs zztD{d7gfH zVNcWIg-jpPzxFD6tJ%biC7(9b*zbl3cLJ$^Iz3-3E&eE@1B-s331^p4^SMR)qJN za1^-hIA@__Refx^CQ4)t=oxFP`091TF;#``?xLpK8-W-nj}|A`7>taM?lo%^*_uM# z)H!q|w@9lrklQu|L)~_gIY%{3$M$-DM!6p1#hRzG9dc@|=bxfP0enl~$}e}O@$6Wgq=Tx0`B+3Lkhx1?bm$}c*24@t+DWJ5GMqHvW;1<-czL|s zZiRAc?iws@S)IjOG+>pPf$@*oPC9;yEGS>8f_=za)VrQ|k*?N$wru~|QEz5C!k2#v zH&%L`VyEQL$`RXrdH2XZy(+8G!JO)fBHMwQzVwZYtxPqkUMo@PZFkEWHnx1`aE5r{ znd)4Um*rN1-l&)`#-6q;MlU-?DjMK>!O&NCD+|y2dyosUbO1C->sRW;YX*j;RgjQx1pa zJ~^tP0Dh5Sy*+)!LzbB1W4gePNeG0G>2; zeGZygyC@0jWGEKGZ#*}l5T#5<6lJxtxhpAsL-LbypGfnH+Fn~v@B_8Lfq%wQ?NsLn z!$(t8#sXg;5lB}^T?sy3;k2U{>4M5u#jzM1UCo@`9enykrPrjcUZ3>R^q<*7V$V!T z?n8tW@VPta&xXB@qlM`tnrEhFnk<^TgS*R@b-id8j7E6obRuA5tNv0E$lRx^3=jAt#Jt|arS*Lb=S9nFcz8_=V{=o zoCXvvtXeTr^$CM@LwNEMPr*-A*lC9j=cEtrd8B?~X{nVff%t7WSR~e8``F$494GzF z;oIA}MZ-PiVK6n52Jk;Ke-G}q`2$gnJ?kTO$+0xpl#jZHBt3j|xnfvH} zs!&H|AIm-6TL5>y24K=Ql~{Jw^PlL4FBOVdyyaLCUa^<9yF>{K?ZMq+J6NvaNOwYC zNvnExbxqH~3R+3LH97dU4axsClY#th;}`Vz+EP zyT9Uls<9|2Fqn%z)z2~DzM%-uR3!HeAf^vi$(e&)kC01i%IZJ63L$6SOU)To(^%&{Ak4cqjsqfcRGN*XWLgj8 zcXl)qgai6u!%I?*3kp@eT>#J*5`XyY^;I!zDod*eDaCLJcwV2jr(@`_h|G@5eE-tj zK|JzuS5Ul;5o?=Gr2*mtm`l?SbIo(=)U!pG6#G_n8YRWT`YLOT_B>~e){gtyv|mno z9EyA2mxq(NAO7hbr&lu^pQiRkrGEVCkdHiv2zO+nK$p&JlyV2U595gKD+>7(eWX*p z6mD*ko9}lvom*r^SC{h`8-+)J{H)(XpM0!I&G9j4yNZZ7_l}$6?EH=bUH=HyV3iT%H-4wb3J99Mz?O7A#NR#Xn!8Qc*tSNlAHgy`pyU+l|t-2lgMV z@x4i?Y_{$|0c`YePt%c+5S^ARAB9HqU)L5JbzKiMx7ScomXhSwY=_MO=lGzv@Mxp!#Z4Oqoz^F1VqBo`wozc z3{io&!ENh?lXcZw3O$c?_cf-S9q)30liCqL3LhYhkZM(Jg`}F*nJ4tc4M~Wc1{03^ zyMp)%Rs)yAS+WMo3(~zQtAllcbb0&7`IQaxlM=|K-zO-MF%koUYbTe(d)$`=OVryf zQr;SQC!Zwvkx7uo+srrH3H~}iYyJ`%n4&Yr;vd#wf}u`JroAd8h&C1r+EOETV!XUb zK`)HqwE=ct5Mw(47YDmMq0&gZ=^cmuuZUbI3HG!xBlxq&u(-}*hL13I3}-gB>Dv&S zr^YvxLKmewz(6kFxV+7FAT^$*x?E7wdrufbU7Y0ZR;T2rB7eV$ zOl{2*Z6JIh&J@ycgaCB&dUVkx;1hRI@N6ng;E5!+l-ZLSM zqFEnDNADE5I!~)?Pi9}fkimKmibBm~z2#>~3l|TL3Kl_PrZz7e{(V|&hs$asHTJkc z4p?KY7~VASLAVpV04g#gGd0Hc46HuVS)@894;KNqu$~b<)_V|jCcgfo&C2+%E51ue zaUqrE65WvDIhQ)qJPfM@E;DZT) z%xr*0=m*j3=ZoEOD&g0(W%cze8=Gnx7MWpK6*rtVTHN(vf^uHd%sjWlm7Zg#Uj9;^ z0=V6=FGz$EuN`(@xUiss*4z#daE^8#77m}F=T-LVn>XK{kR0M|A}plU-44wn9hxy) zW^wMA!DT#OqTl?aXcNoOaUUU3q1yD(ys;;dzw8w(g-_u1l#aO0!G17F z*#+Es;@!L$Y$Ho2vnB%NL+EFB)!B&LoB@hWhNuFMi?p3*&CPc1zJ8A9WjJo^&h1O*oU!c&+-Z9@#(3O5re&5{vv-N=6q6c%o?;IZh8z_y&U2EDagUy0zvQ;k0Dy>8HMmr!Yy znxklShRL?)q}tk(i#rw?uDNDBV6$q!B|_EL&6f@#i>m(0uFo_Vlx+^t0fQaNcRuv4YF@cEMsrDFuN-grjdGNFk_rdgcX_ZB&+Onf z=B2mJ1d~*7SygVr71jkeY=vL67r11POc&xR#c-$Ta@ZJ83=X@qkap@LuUAz308_rr z`a3OnjK5?2=0Uz>HPP$0BjEEUc}v zznq$ra}SSvLlv!YE(|%D+bzLkW47rHZn>O5pQD%$^Tcv(S0B%8xge`xC<VURGNU1*h;DxThkE(bEEuKnG4nREWYD^n;=QQOczT8%8XD}g8jshX?dcbTvH#qQ?#j*`W)RI?B#%D>w zUmaR`hn|5)KZuU|q|{ay;-6Q3wUK6J-9iQl8TnTlSvc-e09xh^I-UpVyTNf4%6=Bx zq;|;+JKhrkMBt_xN*;%tautx!6@Eq?Py8g8=DY2$Sza>RH&|=ZVrzGD9vicuIV7#c zr~aAA7ww$0t!YBDI`2=ZOrsT11f^C(PCCfjW3en;McKNvp1@C|h7;F%Y6g%2Q!%;X zf^Zu)aOlPM)w;(VIN$57^S5@0%hF$#O$m7-En?gC?8vN#A^{8p26^5=pl#pWOrTR} zHxfo$bdag7OV~wmKTz(!lDqgwg|B<+bo2p0K7zdUrd4!&M*&u_o?HwTkWngj+>PBq zlj18}CYVyPIvewh*J&8j<2HR;K*HkQ0JXIN$Ly44h(dF=A5)1d@qieX>yF<8zXccI zOzb9zG)O+or)ua7->GWZ4afERF1igb-*WHTXkX}jopkd4QSk?qgji3_U{UVRJFB~y zif2=?uCrxP+?%lh_LuYc$HjCJlvmvw42(s=k}Fx$#d0-{przsI-TCp;;xID<3Jsd<0hjttBSn}s< zH+a6wVdpvCo~w_W&)!z(djE&aZp!%+(b-`?4(c$i%&{Wz?zr@rlxRC~Xd_(V=w2Pg z8-!n+=}nlQN=(d3us#auo|N!^?y8~qq9D$R{u5MduHlkM>D`nf3xTdKw9@k>d&)Jc z3oh4QX+CvjYbFtmzHN;)8MczSuR8R5jfVEEcru-ekPsWp`XI3OS<7izai~gGHO=5< z1*dJQo446e!(N9JADdK<|8o8&q5ocTvVD@y*<@QCuzX~P@;5Hy#~Uv*po*FJo>gwR zO6oyC^KCy(I}L|HXuZAlvdCr+utduQS1>OD#~H3?xk_oDQ=Ukyn4?J!+P z59Bl7E`HHK_L)>k1XHO!jYp8ZDDK6JhY7llh(@PdBE2NIhAlKTN&rOa$)s-)& zN-Ik7cjCU5aVx>&n|?(89b7=@)DFMwG7BA`5YE}22g+3r_VtFNm44TxG(7#Xyu9v` z`3|h!cilKVm38>F6E^A}DzXm{u5N#h5C=#t)fE3}V4J0=(*ye(ac}->4RrJ*zN)Wp zSNM|?a0H`BhF%$~$I1gP2BhYU-6Tp~_U*~Avs;UH);Hvd_8WVZI1KV#ePG>_>wGx} zyEFKj&E1a@4FUk0$}|igH1KEH<^?5ml@vX4A_%*=D(aGSp~df1{rrlfL7Gxeq=k$M z%FEEvZ+-2o5ZEngQ-Y5b)OA-U6%vz*E(#waD}|KT5E4*RuEQPhJ`cHea09jWNyYt5TYdacz(wU?Fo#-%NQ(Dh_d9YWQpW0N@Y@GK9U zC{ltK^_o4+Ka~yn>25(8+B*EJZGwn*oGu89>T&wU#j`ha2g0aVwe)cx0uh1lij zs{GjC?bZs{VlZ)Sh~R!bbQj;}^>lO3qoLi}Vxy=M76b8QaD;J2TYc@llHxDw4|)}0 zCaEbW(vJ_=^y7{2teOMds?K(P%U{Dq*-j$F&bo_yNkygg`fA8YRWqQjq7<&3@WBOx}+~=DuE~w(#X0Eke)R)40QqQUT%M)Rc$X!f%W; z0zpE@RD-ooq?WN;aV?tBGF^ej_Nnu$9&dO32sE29RoP|-qCl*KXiDPv`>*knlYe=p zMLRbh0ojIXmeoM&>)HK#>Du0woF@@{(Nt;yiFvgA% zJW8wc=Z!4fGMLf=S#33LgCMBVnW3y$5$T14V+i^MWL1*sObZe0No5ssDA<-MIs|9& zGEp#1nDHdb3b@Qx@Cz3U@^vtIg{yYpof-fu#dgkm`wFLs_foofN;*D&<6;6pY(DU}!2uioEunzh?X*RGVFi?7> z`xpX6E-!>7HTaTbA3%7i67wZR4Fmc`5MmAGYGzVqqEMXORzHw^<kmQ@R~m($+QM3SVX4Ev=pCId%H|y?64*4i(1I&O%<60Odg+scksrmKb;&?q=&_Hy?vRYwq#1PIYNeF?C*caR(H4`+Puf+188KrwzM2f%0wCg0gRbo2r zpZqct>ZlCcLRZ|VfvKb&7z~L-L-~YRdDr?44wNPeEH8grwi! z{rES}Z(Kk87V&`78q|p6y3dt#{+I79!2eWS{ciuG6nnM#_5aKi`5$AvuFal0opx(+ zR4nV(vD_yFu0LTnetax-q2k^9UKlANJO$cS@yUoic4V_BNAKt5@{ zO4x97+*bmB)@_A}0YJfQ6^oMS0v=D3QeYm41uw>D;^t~^HEq87I0WRWytap1m1D$` zK{A_+#Q4&NJw0tT37u82_If4c+Nh}3c9N%OYu9$dz{&z7yN3S8=fw<41xQjm#|R6M zYDGnP8UBeQf*U5^qq;K65BG>=R|C++rcL8w?`9cia!INNqsU06)iB?p!&CE1sT?6_MO^)fB@30~D-cq?suum#J!I6Qp(#n+$~ z=XpDggipBkbWCZiH|;~RheqtUhMK4{#g4pk|5-yVF!{X~Bkr^;HstS`U>JR<1)}(j zk;_^^iY#Z=`pZdXyMSg=C|{2U zrr$3Xy}o7pR8SECD|DmI@0MWShn@#X?Vh=T3h*uL%4OMt^i)L+AfG0E`Ka!VkN)$%2kl!JV`ml1uA>y7OV60)09|O>T-ei&;|{_PwNVhnkObe zhjIb^Y0lGTISa&ElGF7l79~`JcU7Ut$LlbCef1R(;&Wb>&WwNgRV1?LTcvW z9LMz``pbQ1Qyp;7nI%_?#rl`1ly{5tI*C6&9cnulCgg&!+q=Y2sGg}Z1h!wbp$M|| zk%=R}#B5Vx*rZd#fm6F}#}a}`Hc)P*cP!8cMcxmSktQ1`m%*QxxaH!C?QeO!_k2;p zpPb@}(^W^YfmCT**X@bL6nhK7nZ=vOaO#t?NFizJD^5Ll)54%GxZ&qo!QpRQr?*$M z57Z08I=Bm5Q@q3Nyhzf=jp6`gFDG=MIAC>IV8Sx3F66W$0~NPpt&)Ur<#fY8q%G9` zn3@Kf99eKIrNwG6-|VPDScbn97Rj1AN)|>W$`n>5woOCcx4IzNB_&P#U)#G*QJdS5x7ht^F!l(raCLA!g(5V&k|Y^88l+()QN$-n~pR zj660}H)<+MvHHe!#~|Ip-&4KE)c!(c-_d8 zOsS1IvL(pWJG=$(>ceVwLxjWYU_ zoLTQQ4iPR|_lFqR&D@J?gt|a<|HU_~T=JKW$O>bcw$}!+XZ9P{4`B09z~aaRLSe1i zJ8H!A=#|21U_CIfq8MIp%nG?flrFO zQteCX5EJe-rS+^amdY_^z-k-##F)v6JP2^JdzQ?>kukrmq2r>h&#b)GK{wQII^+2L zt76p$JvVWjKvE5XcKEY(h5{xqtG4w1A^KIp^0WM5JHnd%O-Tn0Jr{nxD<_uwm>nTyHH~5Xq(v9=W zo{XXH^0bSlZKfyfndx~n{Q{?{BZ>w<3DviNFRm)L=sDk$H!Y*uR#N9~VUVfpykJJ|GrkZjJ> zVviy#u}8TA0~Ug(9V63Y!oj)c>y|APYEJC^e@&<$`IpD ze78Y#_9rS_$Zo||+ucE9VzI@TTfaO@Tk#v$)q)Z`ar}oCSA_wyQaW#qPjgM9*eotX zPR-y`!$;@Nluv|nRVZ)ttkr7YcA&%C$!bEB6c>iKE3mn; z{(XtCy|`>8H(+|8r?H{lCSiD+2#RrJ~B-|Is)?`SEKbUQRKT!~NS? z4u780aOAAt{^7dw(BRQkiy>AviPVD9&`xbClggoa>bnIa_$q=PySiW;--#Bqsk$EeEjr1Bj~30_?=qsIg}v`};}ruT0^h-YE$omjb`CSY~F~ zYW(S+3+xCso>m~Nl3!R8IK&LuDHLJ|HY^;p5;CjyZ%!a}(Eiziw9Sw26L8tL3Au+$f^}m1RV*uw`D9&i zq#TsCu^#-e43hU|R7(DNW=TSDdnSEX9+2WeNRc)!lW2=b#<(S7J8zj#R4lZ%Ycmk0 zGby&ij%g(oSuO=-`6+#u{R~`rK09VzoH@Ez;Fl3C7cIBy+1)Z`zuzb<3df0Q@CAvjSz(vTZWLX*nNcYS=15Z(>P=4bn)?rgtrr_VTTDGK;2#HR>1}7M^ z`?yyX%i0>xEV|bAJ`yE}|7_upssBJ@6==Ehb3Zo`Vmg=NI=O$4d4yd@X&4AY9*juVt$fwr~A<#0-D`}?%#kSOh z#Px(Lgk0K&x2je0xbSD9?~YS0X;foRZkA%OY<*{SR@bRBj#%bSwGab{7PRFerVVNg zX%5litrLAIa+|}83k?bZruN5PVY4ns>Ov@i7&J9KxF=udMJ3vqD-TOWuX!0@mrXJ; z#uvCjbtVBWl=R(anbVlTARW6z7{c^W=WQ1tgy*yU>}+2MJ{q217Ad>&{Cc#s_{X%8 z!otH_0P>QGj5Hwx0`rRLo&tpg2cF8%Lzmzh^z@-I%(2GHSf zemm%fJ(tETxLZp?-SJ7vLcn zm7qB4h{J~3SM{X*Lle{SX@-0PtR>DWtTC0;R_sgSb_S`e%+2;NQIgS0mc^REi_v@z zM@`t*DevX$+}rzvI{TqhEph1zT{GK!@2Hg8!ig9?iQqwfN|7~GfW*tYwn(Ec`=o@mKLQxR9E{=7a6WI)dXL)O$ zik4=)Y`S&PqjLJuQQKxoTSME4L`M7DZBHlbfEHy~vZu>R!f;X~3M5*KciZ1=-CgbM zkFvEm>*FfJzFlZPI4tB~I5uov4NoraR$NUUvKWPx@~q4EjLi8>cde~muf+^G^ndA7 zlU<$qj3#sgn~W%zO=FW5YWqUoXKqDWo{HWaM{9E*L#{jXbYkl>oBXV}E%y@ImB z#woVN`N2f>`mQEjzQi`{w)G*!i#!RvEU+n=ytlq_8D%X&&}cEYF`+26Y;K?S@ozRrtdq8^c#kbt&oO|z^H|KpbNmf`B z)?y}n|Mn{lqCcKnX6fpWj$@q@B9zjhg#x;1Gn2ZBE6LjqCSiG=549lPvhNKG&EaKc zog6wP(Jypj5L>yuc6++bw~INmD+?o3*S?3xF6}OBT~~xpI*(Wz^g*skE8iN9mi)j(?vfFWysZh&J>atNZx~pJKmLRaUfvi)ZZE-z*NhF5dGkC@B z<0{kQ7E5?)B$FpKp4jm)vyxc+EnAMb=XzY+1%*U^Wt*{Haz)rco(_6|KD{-~JL%4R zD`)K2HQVK$lH};1UMU*>u}=^%GJW(175(FTkN!nUXMOg+84LjYz6;p0{gX)m-5&rD z{`Q#w;86Lmd?nzwL1ocwfRA#!f(@WjSA3)6<$ss8@#{MKw=;IBqIu`k|A^E*(WVFA zjn%UXc`Eiam5TpQs+$+5fjJ&}3<-<(EXg*!SfS%lQf7h;yozht!uy{!x>FWw8G|5l-OQjM_aPS^YU8SeLcwpO#if z>K2N#V)PT-1^Ia3+#LVNw_WdY2}%?Hy6R#5q23N%g-|5L?M}q7kR=P~L$TeG@c7B^ z`*-d!2$m+yW54<3dU_)D%M``=goEV^X=7ch=BH@5O`Pgti+V!liAWQB5iLf8lAhq5 zd-llMn9M5Dz?5_QwvHbmL9RpF5o3@wp!y>ExWX+ASNJ3&Yc&^z0WS%Zl_RvZj+6CU zl97Emin+0UuwPW=ONFnYdA8D{c6wR;I;BTmod|o}zKoI*M(?C*4tWdkp%!;rTAg=Y zSv7moKg&2|*W$HvsQkeOMjW(t(9yT3vCvctXoi91iD~6a2M7rxt)6YJS3W!2qICA`OTVqaU}IH@ zG+CkLMG-|@eP%!tzs^|mvWI!%~A%lhFuXvi{zn%MQu zNHS)k#D&G~ey`BX!ntGx_c%>YCRx?gXj&kCQGL9>$0W;@CeS>C)VH&rVoDy$p$xDV z#-dr(V{H`dj>}}{F#84eLz{diopVbj_AiqvFn*dUQb(Du0`M{;31j{X-4GB0NJ`x4 zK&;`Gab?SFaejy^aeIN;%Aa=+K5OTeGua8gZOT2rZ#0p&1IanX91IW#)_h|5%H3RB z*|u>SKPM?{ugqiycm=Wsw;CqBTsMaZw5MIXCI$w3Qt~=mWE~g_+_ap~d`dU!|FkYP z_bsH>ImmLY;}_KlI6yYkknS-{A=f1=pxSJZGE&gW3U| zq2B)8-U0&cdtRpbNkkx{(;5e$9xGJ?X$6VzlIbDJr|kKN^5beDBAgu+x1>4eAlp!~ zW&US1F{pmlBw!y>Q&mUZT))(K7Ju0+Qed#U3fG@X15>aeEXEA&BN3d6mh`QpjT5{^ zk^4@5Un5-4F*u@gCxWwMw#R0qPH3D{$MzJ&8aAY!2MwI)8O4CvEZcr$=C@?TRr|rl z`pdE44!vcgm>nTgEjC-Hq+}+JaD&F_D`9I;+TsW<08r1hkSMsfbDk5{P zy{p;i@=PB=zv>rNvv(Ac+tA@&LEm!fA}8A&k;rI!heG|5gNGdYS`pvZXLv70$N3^V z(s7hc4TxcCuvhFWK^<&QpNioGb|}pnmtK9&e>IZ`?U8UwbJ-IArs+!ty~rLD&=U6k zmJQeoY1=OL6*hgTJUP^Cc5W4uTj69sp9-;os3z~LyEOlZp0$q|6X#rwUoOcgR20%i zHR9CzHt){bZxjz)+$36I{jjDz-`Kfg5MH9h)h>LFaCh_D)9;zE2@>*{y8= z>guccya{!!gSy$_FIaR+{0ib*zZ~gLRM2`izBPjZocN;>3U6&(07loEpv|@v zr{$I}$TKLJ(6ZRWZ*6tjg8+@Y3{JutBaFT6wSq^px>0HQ!#?e%(w7gQZp#tZV<4hx zu8Gl_s4zF{HI6}BUsclj+?Cl@oG#vtEbEf6vU`lPX*PcD0p$(hLEvoEM~N(A0-E0J z;xT7`uNF13$>793RQ1|gvW^)}-6)n=aByyf}%fs}$o-mLJQqO$6V(%RdS#WLL)jYPni2GT4Z_|$zq zoj8-jc4k}AF>YJ&mPQhvDK(UQ0KxoVq`TRxRW-*t{rA|Gj7fovDy^`iv3>9@>g>$r ztt=oJT9w2n$Ou#;Ss-|c;()Ib(au`>!fXLhr>%J#7IjURG8P-0j9mh$tKoBtro+OK z_!CA-x{)1`e<+KC0W`$Uv$Zg^YByIAQn|?K+5j2XH(VbFz?Jce=hnl+dFBnB3Fi)Z zcT6&MdqMz4u-fw4+og9Qw9+&^`NfM^^ZcG*m-c!Knw245@f>$4SL1KZeMU#3QA0m8 zH4;#X&7mvb@*J$g^=k@wQ0nnup5V4me^FhcrUFtBDyqkKrGRk{)rCJE{U6lNo8M<{ z0_OnxC;;yRjF=*e{}=7^^WU?lHWktT?z5-xzkQSez?8~qHSIToI!~!snguNRl$+;G zwD#5LH#{%@0P>$tkEuRs1NK-qHL0Fm{F|BTCP08oV`}Sjlrtv9Z5U(z?A}lI^Mfmn zEE22~L_MI$JteKwIzgqXIN~LSpa=txC2=ju)#DY)#c)q;cI2F8ww*1KJ3a@;>Adw; zKjM0okhruBXz@ywQ0rsl4>vF-og_U?IKdUJnRG63cnmFg9%e zoV=)uh|xW@!=YJ}G8l(Q1;GhR{lth_MuWPv@FYI#8&DV9;5v1$B!AaC*G3u4Zyi&RE4G|n$ zYu#V_gq`}#I9J}ccR7rkz75zbDwyn<&CjyU%`Z!;M(93Bve(6hD(Y-G&6);WvPa)8 zzCBZJkhz(}Q%t~pGUGeW35FX^E4>Gze;}(!e?Hrme z)4CfWUibSD;cKyug;0*?2+<~q!E``1_OHuTd#trjmBX_WbzlT@>cRvWq zpoS7J1>SL4^{cD+Q3v5O7-z+2;G3Jo|p`I=8GPF%~pm9LNVpk!_}%x(@fUK*tsq`KJ! zHaR|s;Nlq##2879Wb_TmN}GB>E#L}bBB*t*1&1HH@Fjf7t*}Kn-?2z@>AA@DI7bOs z9Y$aF^BjT3HWSgC>xhrkXBXTAyY@W)mKUJG(qhnJwt{0A_2x+&6;(Wk`rUXkGC=ku zk1ix@YDv5YZRQi|U!|i@{frxLz#yJ(8Q%rU2LvD{i;O9ze)rz)e# zAJ>HEstBA-k#14G6+#AxNO_xg&l`Yf88F7vLNCg znpx0JL+y)!k@1t|c$(E|O)x~zrts*9!7-zwwH?Yg7d|KcP!v2pq_?^uwe2<2Xf!6c zhd51;^0@}7iirv1jjQMXQD|+l|8mU?#u_&6Wt{*+RqXB0w;Dc$d)4K(^qNUBIf_Va z1tr%kvgzWPR45a^tcR?7472P;&<+f7N0|s!nXBxUOAMX$=fUwGC7^9R#UL~_HY)K~ zk5tGUy8gvKq0%2Sck1X{ROkaj*U=dReqeKfJSU2ES~G#^gTpItPQsU6g4Rr|A$^35 zvM0Hxp%XvJ?pIV~sy5tWJZcgVR@X4*{Nl-ND{tF8y-o(VinNIo|rW#=zu+ zRZQ=INr*wEuGiu_S`pU~5VH0R>f(i5?2;@kqK5OJWFSKHaYkIcO{$`k#o(NWmqbu>kBo0!SEgg>p>9m%LV!I%Tuci;6&&0c zS&@8%q*m^Z)-qdNWX|>$6CH7c)s;Mf_j}DezfHE?rYzeY!86={_O=MRKtvTO8O5#V zFkFgdNu{(>HZO~ds&pZnaigi_sK5ADZ?|yV+2~Mbb+?8+DMm{RB|T4ymBv;y#iq5k zTY5yqMZRao1#1K{fqbjjC%^aUqhaCFbtXEktz1!S-bt&*f>{m1qSkPPh%f!vtc^!J z_(kzBzg%8{GQJWX&hAo3kGITt*X07Caq6M=+F3u6~`o-}L&^`(KG^!9IwObcLCta$Xe4<)2D401}LEjP$K%^FTW2*!TyK{@m_$|vNgUKN} z=b2?nwTQZ>h*w%frO`Ka8i`0cx0wj#>S!;SA*myiMTVXR23v!Ll1Zm^$(+jS`f!lu zuJMB)q|KS)iLOiwp1R<;r_ovd*pYdLHwazoHM`(a8WSG12?Kc3UPadGTh`V3+lw)- z!rHuzMX!{?zqHe<0;DAY0BZ|GKq{)gF9LjQ`l~#5f4}*yZ3lDhMlkCXDEWyL?qANP8hR_e!XNYzmD-}7%nSc5#JQF+bvbUQB$nR zTkiHYjYe+c-9~n%cIyWT?}Ie{wBAPFmlSD0N81Rp#jQq+&L)wwo)9k*1yv$$S-ZJE zZa0;mxpN8^l!309RDW(4RLBo{h`(MqX%gb`Yu(A`@Ru9D7IrNA?{ zIh0swc+P)lBm-R#7s0&CY&4AV^hzIQH<^Y zHQ+>Sv3AIIoYW%xjbOCMb&X5Ldz(+O%d2SqL|+y#eYxE?Of2UX=#UEln<+(FswQLg z1&v~pY#IgSW{Ar*TaoL2g(W8%h9mpgOHRd^co!Cjv)rh)liL*6!3?cz_1yGLN+c~X zYzW(ccAqJ1K!y05GWbmbO^=yyUyAe@8lgffjT?8z;HTms`^i4(e!}e;!XL$=ayAm= z?Epfk_-ji##k;XC0`>dilzS;G>IXtJMw_YQ$n^ZU6DaF*1xR zT8R-uC@481aU@|zZ@L?@g{!$ zFO2%_x3MPH-I*fS5rwKrm2)NCCObQ4gvy0(8ksIgEa?QMW+j)}tzH}fT42+3e6wiA zm>1?!+SjoSAm55I+tddbPDwiZ-X|EDZ5*pD(#G+5?l_nsgBm1P^J=FtFoZ$(y^K12 zG<4%S?*M z5pIWE3a7;D=7&2mOeHg}b4N}b{;bKu>sywM_QIA?V;qrYyIQ1K^1_$7AIV*u*>=nC z6-71tkK$pPv=6U{G4I-+5NJOn>I4$aaa$^UduyApSaaSzj&OG(9B`2Lz{%cFch){; zGaKF;)60p5I^lUsG(HZUzWokfSq(B{o^Y-$Syg$mb?yOzS|2GZen!Xo&3<-~ZYozI zv}<)2kVGMGG4?l6?mOI?+w!*t)F!O*-%h<3=DI!Vp=4llB6p|l37B6Gz3yu!}6s&})+1M0ZyMLs=iYpKz`G&N-Al<3ruwzem=HTHKO znAyWmZFkyFWTg_j^rUIj&>S-)nV~LGrT3h?mK_xPfogXk;LD`f~EV0sm`^H zk1{9PrH(`t?@g$-5C!uB!Cp_f(bwB*1Y4B1$NAAWeI ze%@Q_?*jq;I&tDp(WPq?nQobc(b03RJGIX4rM=PEg*54(QoWtpb|M++K}os+cg)_? z%RlInRHb?b1jf7n4vsVcjsPe$sqQ}J0f@nOF24Ee;-@=2Z&m+w`Yt^He!TF)EAn^A zJM!D9ZGZ9O(eIL%Plc;Le73c>W_nE&HZ6K4+pS9?LcYdTB8g|{ zv29FW;a~DWm-V8 z{62;}4|G#wGOXdFo2}RKT7>mcQ7&-`;(H7Q$vNHp`f~6Re!kk;TRA%ABU*H@YF~i^ zRDftgLBGb+x#!adlz|gxl5bm|C7|GCgDPkl^!==H%i7S?)B9sqvvjFYfVf{3t5{JZ z`0&o4;UEsS-d51@3cTgq${E1FJZvY?Z(Ch;yAGYr|GAleW^bx6S57HqjA1D`>PHRl z-Wq=G6k2r#Pnu7sd?n2lX3w8Mbax}J8j@j_qAQsr#-6sL6q)gd0~_M|vM_LNN3;tG+`H)gF=@{QrDcwU9nzJ}d$_3hH7q1TS?%L2GNd_^S(1lXf{4 z!vA+p(ZB5O(cHZBiW)HbT_f%+jaA{B`+uU{mmEF&gsOiw zH)xYvJiJbC#E&?dPI-978%N~!9G;eJCz_=_D)ZBdq%4HhVH}e;N8nf`P~!P>Rr4)5 zXva7H#z+}|`G}@m;|%cV>{+gdUk{Ejy8+Xmleer?i%1<`QhOe#4eOvhntBApVzt3yUP?Fw=GD{1B$Q9WW$*;Xj zlFLi<4_0elfLdr)yq$0#jLis+7g-$o4NJr)v@Ht-#ps5mg%R-qYQt(-8|`_4ZeyGV zfNUmTw02BrYud6|uPo=~#3qY_-Kk6R+_d+XAri3Q6J3C?L7^5CD9lMEF>rKoBsuRZ zGN$ee{0u0RC3*0wH|_Jf0=^JblAI4TFEJ~dGDtJ<#XFk+mHBh)@to5OxzW({RjA5= zlrh4xG@)^!Ty%7Ixl>OYX*7^; zwa$5slWfOYb>Py?fNB5iQ4liWt5sLs%9d`niXS`{T1V4pNf_w>yd@1sWd|{ki>puG z_9n8%+v}|L3E?aWp_v=gY;!Hm0tDz3_vMIjNDzbpzW+SCBZhpGDb!Go}oJ)n% zc&N&!`Rk0B3R*~Ioa^AOw=nYOdg6TqzV+Asz!C{TYhFs-W#+e=X5H4*CPW(dN%aOIuB3cqb_P7_+I_qVs* zkZoxrX32CH?+s)$)ILBAm`-Sf2x^&eb9CWGm^CmTaC?gECr+U)!m*W;9^Fh$>oaBc z;$`EpEBeh8RZwU8**Gq{fR%_&lE^PDcMbE>*_YwZiCs%}$uxC0ju95?E2h!G#|_c! zhgFscs?Pijx;_q`zTgKg% zyFvk#nH)eQoaxRy@Ga_j70YeKYX6~KWSx=!uhGrJm=?OSfvaMU?cyMtOQKKHDW|?> z_P?lxE{_GwJ7h|0ivMXFD5mm2tV%DfwK2K;r(p@k{m|l+zzs5CV`1xWRdsdp1cCvTh#ndgKmS643pWwkiJ6f7zcae4?gPnyT1p{K>hmf zb%84SkMuXdn~PNR|Jbtq{f8)ojR-}UY*%kA&~ z6~Nce^^jIYn&Tfj={g|WDpr3A1(dS_kM}vp`ybQ4xqAQcT_C>xvu^$M_7U)SMs?w? z`UODvEc%b4C?KQA>ClOQGCdLe8`S5erjW4INpjHXrtv_QyX80A4#eR^d_!@MWs2UL zmMCJO5AsQ})dA@SA9bx)-qR?_a!h=IDbgdJmrb;bETX>`f;2lCVgd@CK?mKPSrx29 zDQZGy>l~{#yj_k{(`d0&1ASgpajVg`;2C3R$|m&@>5SnVPrN zOofZ9Q;8xV(G-q;!|Wj+K>0|wNT^Q-5-i@-J)rec@nAw7Lj2jQg_Pr}ut;kIU zSXNBEE{iKMt!Beawskr~=`Of|xe0fF7|~dN)MIV3awM2Jqhs8y`pHwAldx<|zu^a2 z26n|7KbLokF^A)61uUNT>oCRk!M2NN_Jp!geMEwet)gRs{|#uyYV$-hDbHc6ZD3TM zd9$152HmT*cETS!_F|Lk4zj_0vu7nt93IGGd*PAjBtGmRY}Cgb%Yl=#80I_4PcLfc z)PA@NSBw$ijl=}uIAq7%y-sh=7*UcEDig8K9}iAc@FjD0^n>&LZ$Zl@`GeP#$A*iw zKB~zadZPA}`G@Z)8KRb^v4)K1l_!C(bmLg$Da_53#e&wZUsU&So1bfjXX@i3l0Eg1 zx{wc^U3(fKNA}wWwOa4Ta_u7oYvw$|d;FOzyX0DFnPW4mp3QhnY=t=*K)&Y2p!^Zf zPd6x)Hj=Gt$4L*i8|L6Q5+}#;X~n{QV**=)m4-|*waQoOg50Q`l4f2WEn0e*e)o&{ zzC1f1yBOYHKIxm977g4SS#dhd^Yxfz;@HtwcsO7~siFxSy}TQ=^tF2`^eQ?{G* zVdfIajV~<+66U|`%S<=7z66ct#1cYYDc7lK3Z5B@*>mpXqWx?sZl_?^{N~c$gHLXb zVMsxULw>@J;zC_VejPfiJE=KCfH)ffEqu~HrJ7{gavHx-=3}c{0PO!-Vgn%2Q$eI3 zE#5|vyxWDVm7d+`InkN9l*#JHaIvuEFe{=WS)!padg4U>WZP+Q zU%$9dI7{2fYI;9_w8z_GJLoD`=0`}=*0U7i=GiV0SJ4qU;?|3;f1euN6MWw4RNOyJ zq0US+1zB2GaXc$6m}xL|;#zfFS?cgWvSkT7AYUS?Xp% zmf=+aWV5)=avxN!d4tmUy+MzxK-L-A#_8C~?!TWiaSius9dy~^vlMo@+&k@*22-Ir zaQenGH~4}qJb{b?qYp%7XQMUiJ8c>Ebo=4eIcq*ZfBP1$B3BLc9cdPhLu zw3b$89J}51W2I>`ZT;B~4mF)12IpjSS=`mux7S2Qh27wKo3_G6E%&fz8`O95`$n=$ zs*?AiB5SOh(ff6XIR)S3$~z}5CBf8+=LmRbf6%W*Z6_SSI^NEcMy@)Z3h_JCLNu8%!7;P zbc@kY@DL2=_B#H6ncU1ID2Cw*$u>CR$?0BeU;jnrT9-p7BFk5w%Y1k@zqiU$^Wd;s zMw@bYbxAk(W|4++Qf;%(?HRaqyNdGjGU_oi`v+k782P2`dmK#bWrkJyfiKXNOdar_ zxY>>5!tljcA9O||959P;MTwyC`dqF`mX>qzToz3w(#vk=m&_4Z&LJ1*mZg6n*;E?qy&xOb*YDHx09r(fg6`7yw+q%VdTQl5g zIhaNpWzCHKW;-GB^&3stYYt^<(wA^Q9q|Y#J;vBevJdWn<#g8RCmWOBwesz>)#RdN zMj>qj0$WJMyP)kL25(pIrE)rk79A}yI-ji>CX^o=$mV?Dot|Cs(qgc40bGZbp)NGq z(RJo$4F0~x_c?*X5*FFhXmUXB-d2b6hP@LzDQ=PYqA7`eN+xC`Ji#_P&Nkb*M#M($ zwEe?riwcH~^<4jbY8ad%ro4_xtLRSvi_Zd+GC7E7)}DS=p16fMNhWO1hhMp;N~fiL zeQbC;P%zD-qS$^&T_^llndwqvk_H=OEM{^r9m^ZRbXwWIxOc3QX15v~6_=n;9_tqQ zf$*hjya+ikI@CS}bhnk_p(TF$WWnY-+d20$O09Jai`I=B>?8;8y2haDD`k5eiS7w~ zU0Z$1q7{K34u%hIWf)&)z<#N1UG%Oz1&Og}47P&sw*@wpaz{cJdBs>%HCr?b?E8;k zjvfrPQF*amR!cFPFs^rdDv#|?q#=@FoQC>8@TO(H3?>x1r@n0v>-1uu09i3V*>z=4 zarSseWxm-TA?wwU`O6Nmr84c!fm=QMNPng_2qibnX*%romQxbJ+RbI$uYAW8IhYvW zq#&kS)#aD#3i);&vG4EsSYxwNA0<;UrNFNBZS}KlPO$qBbJh$}oO@8)&)zz4m>)~q7ERQzoiezFr^T~@7=e80iP zRom4L9yN2q7r0dzTfTzdwLO)dahN}<|Ep*E)OR}op<=XuY|f9{bT+SA_2|Yz=7-$i zX=X05+=A_Ecj{#;?)S7~QfT<(qpiDf41(m5zt~1ekw{SI2~>0 zn1gDb3stZ>GZg11@E-dGh=C$+_YN?$&iH{^7=3o-_3|2THKj*S93}ZHIv{90i&Qj* zgl9UYJp@BBZ-35yar?gfbFNVpk<=k!9MGtamqR4eUhS0(ZE4E?2#x-9?X63ZoYCE; z4?jDP=375}PrJfVa>6QFC@eQ`6nLA3zw57#dT4M@!x8cOyuH-kKwfx~c&LuH8$232 zRkXC0ZG7!v)F^`|8Vu!`El1hh=zJU9tMUHD%#8N@R+wc}LU2@mt4;8bANFNzT8K>m zj4ywN|4K4Rw*90*z*r}&7Y*VmPMC5NT1cFBnT7ZdvQ)M4TgIBgj?LqPk~qM!s1GGH zBTUi}EY`y4Iu@Ze=Q`5j=eRZaL%3v%tQXJ-CS?yFsY=8mP9BJiIl3`}wE=Clo)?QO zhRu9M)iDsx7E6#o$6soBd)Q`CctVkdj&1(bjSrK9&W|l*U?D>xlNkf0xW??9(dwU7 zzReeF1Fw(@u*6GZ`*>4eTxodLQoVbOQnP>E1v?JN3FP?N&~?e(bP`sYkDW5eUw@%mU-?xR1SeGZ=v-!#hM^myg^$mh^BCP2Rox%4;2<;tNe_^ z#=Pc=x&l)!ScK^sIA@6bq7w1`Jd)>(nJLhT{)xSnV?a_=Z672#F}!^!*PN{?202O` z7DmgEG`pIc>%yb#!()!ulq*N>*H(oLT|CJ};{B&We2)`ef3F1_m^Cjk#J!2Y;a(JsFU7f%#f;=;Vni<|CU!24%(Nns5? zToA5K2`&&Uk4U=5Q(>c`W^M7M*gY4R*0`cVC7<4Z^a@xm*NyP^IhOd{uTMCqiwY}7#!$i-2|8FAJ@BN5YJ zotqn}l(_TyKIUERtR#-a4Yxa%cWgNV?9J4IY2|7_wrI%JMv|trQzWO=C4o(A$+rmC z##89Sd{Sd=vcW5ga_)53_k?Mef_R>)IhA+=@;!$`hXNPW+3QA@UGC>1$B0x%X&5Z% z2wZW9bC(`5LzU4YPsuEy=Je^&ea-6G!}BCT%M+Hq?8F&kJ5BE_;<|4TT54Zir_&-W zau0A|cYiTxJBl?AuBqsl*3DTW719}Kx;e?g#@d$k(J`TR526t|W7{Iub^fCTK6*dv zI7UPLWJ@XUj~35VqJ~!^W6N!`^;S%AmzlglJcAx^GufYmJwH~_C$OS>qYPk!W{PO~ zvH1A5Qra+UtzusoKiUPT_)PN@50QN%9p5QE-#*C5cZ?O@5QBTV?9$0(=9_uFn2ClH zbm=s#g$)Lp3+jlSy9B|#i5O2h`dAY(dW?N90A4%h!$~YTcXsb;k#8Hj28`U8=#6$- zRwg53a96~+OvZgrD|a?}&+7S8EbXsL{tUA}dMIB(Vc&Kb6UUx~%PDjCd$){(HP6++ z=kTBW{LJHfw_((V@Qfqw9z2LLn0>f);NU5hoM;f8o^;3WIPS!}ENit2o;gHgZise!NhAr5+9+P+I?IArs(g zxgt16+uqtbzMkcmaRha8@*w;;QF(wl-nWs{@GO!lsOu)eGGp3q9Ku3hozGgmugl ze&rc66j~!48NpM<#GS5<=WaI9X+ka_nN4@Im11o;wn^d|Ya-eqy0p9^l^Fyz0Hf+= zWa+Y&w7tEg!;xayUzOcAqF{MlSv0y~sk;08*=};Y<$yjkT-nT@rL#ms|5&pD!2+gp z$;Yp-zbReNXpt1>oxw-y}Wp&<~qG_j|To#r76 z7w7r0s=K#(9`SQ0SXp(jU5KQ8;4;=ZR(_4Mjn3A=u>X6(2ildmEfJMhVV=X+MM{UX zUA1GqkmV=Nc@9xk=uAAlF`NXDZ&j1r2GHLO+?~~CO0P&BYC17|J^>v|>BkDk;yh)k zTLoLW668w?8_s+lCiB84J#+x!8^8+5m8LlHMWZvuOcpYJ%*r=lh}`m~h-CZGvW}Kq zBr;dlFW8>*I?hb#We6ormurba1Kl5+$j5`UWO{--6y*X-?vW2?72We|#!Vf%H8(xc zBl2TL&@#R{mGO_ZYec#U1)kJC3?}qc=6=>PLzH)t)nb@TEHZMZv&D{PxG=>S8IIx^ z)Y+Mso&p&QkWGS$@!gdqO|j%_+&mx|N$r6&<1CVkrn6YmZSoBm){U5qiD6Efd=`}8j!srU;0IMsOn#XOZ=RGMUAXsO2j_~p+= zL05q#-jAY~O{^hzZj3ZOeU}=DK~xvz-|*h8xS*~I9A9AO|HS?+-3lM{^5TD5ZT|^A zH>rWL>P_~=`#^NL^6_Ztg!`XaT8|?zWCL_{{~59gZ@2yT_Ff<$s%DiOc>t&*zFGs~ zmc%~ff7*M?rL8B^zq^+Dqz0$d&KrgowPlik!1`6iI+v! z?j0hi=8a;!gt~<>(c)%TSQPW_pzOsxS3!ewsyK$v{_@<2OwAgeax+cTmcyof5yh>Y z0X$+VGeqDah`)I+oPE%xdE)Uf7UAJ2^3l4c+H~LL)dis!`IM|?6-PRgkFU0b*!xzf zO9ZD_RA$OnDhp)9M8HnMR+KN?W`o^#GUr(Prf?tg=@i$`rS{~rAMY9&?^-VnL) zN#qxR1#3>UBRys;HhA650js|`5<`$+IHypV^gxKn*?mXoNuqoLN*5sb=jAy?H@rPS zgL;chE%mf%`58)*9v&u+Zs;KjJ_{CTn_?fDC7{`1Xv9jU>HVKABT~emeS4O`Rks80dZa^DMHKm8-0*NgKc|j!Gn;D?Lyv+BNha{j4eX}KDd^*@qOJR(BelL z#t*EkNyI}=ZHx<;GdRje@)P#CnOko8ney_*8tQb{0;z?XwDV(`Bt1if(^+Q$8H2lS z5^(3Z#IdiMt+e9R5YVoW+TNeqR8!-pmS(QZKbo@|!QwwK9?~ah3M%f5m$AMQ&pVZ) zEOf;+C@M;4Fz`i(ZVR+DoB4PvU*6o`8Eaep$<84-w1@banni}H>I51~ce!goAeUI9 zAuQu*eg-2C)dio^D56&Gq+YU!L2(4q-#)xi1tT_w-S6S~6pF5CKD%GHp&$Z@RQY4$2Ye`)r=%U?9fgPm|nm4^8`Lwu&C@n)XxLllTKYvX=CVFh5dgYBJA%3@o&GPChXIWnl@6^V0 z1k3|}@@?Un8#KLz>61eW>tR;CD^=so15Px@Y6j4`mY7dK29QL-A zHiu$?bC_?oSZJ$=Z&a+5()>dusTB6@S+Dcv$`&EHVD0?Cj7xUzp2`9v;#N9Rc4!0g z^_#Py@cCZruBQX`irSd{B*gbP?2MY0_i{PSP}V>e5=`Mcvpix@II!F5!0qPMHx(*9 z^!M`Y@YAgS+Tt$Ae<=$M65)$Moj$6C#;V%sZ@Hi+()>gcaP;L(gzaq6+E%fAsjM8K zE(p}Sk@_52xn$QbK=zawGU}i*TaSrOb`2sVzW+t_PG0I;gCJR=C&>fk@}ot1*2PiT z?|F5)pWru5xg*(61?hYV5FGZFBuT>-WYZ^*Z&s!=3gsQ(o1~2!)8A}zi1Dj<%;-*b zZN2QW?BO0i0yaHn_m}qX1N3eAYT}3b6z8rMC9K;zM|Yc-)>4bGuW@%m_G&obiMo>N z)X|?=-bzwMwNX9&Th)sPU>8zdcvDXGC)Gvu-(sHpR9C<9yrH^SeDh*(r_cqZtN(-k zfAfsy4{ECavo-(5Km2FU{(4&cU-1unOr5PG;+AK>s8)u3-J1M5{iFB))$@{SyTA*C z3(bq4094}};7+7~#~)M|Z{4M$dc&$ob@9mqf|$CsWQ6x$-Gd^nDIG;>vaud}4A$aT z=k0D|{EZX(wTonqEO>n_{@!o5YbVCFW%E&zx6iJX4#@YB@do}-M^2FjTG~X15i6d# z_{^=oXa;GZ{rlii_zg_R<943{bs5tMMeq$y&488z`vOtk;58`)B!9`o4(=u^KeD}G zA=~M+Cq;0;n;@+|45^gYg_4!pivObJ0!r4b|Z@%*LkoLc? zg~}0fRB&#vSKikJ9~li``Re=P1FSdha56-q#`L9%=12czQqqqi7xoS_#65djHnWE;i0*I{{OEcG>h6O~~6Bvs1Hy!%0jN?;Lx9q}E2I^X4 z6XA-=a&yy0DcbtfI?3Ap+=z$k3wlIdYLhuy-$~xRm*s&)c6Fcv-bT2=y0P?Spg&8G zGNhSf+DkA!VLG^PvZOi?$}jVaD!vrgys@cfj{+-#hm8mj{x@-+>N3Ux1st(n1eyR$ z?tRhRMZ1^^mL+D{`U<|`9?@no7J=<^g0~JM$aF$xr%Oa&Ziv>yG&G3#H7SFqc!-#| ztx78eu2ljR!!?~mYRYY{EhDi9I1yCv2{+UQdVp;fN}-&#LZ0YROsa-AU23wF63irR zStp#|b^E;XrM;N~a;QukIe|t(CSlQw6)B!jqBeE~x1gO1)1XByGHWuXv5Tp9bL?|- zJ~mt*?QMHWLE-aq0=&LMUv>Vct`>@eq#WQ2OWLRb~AgiA?y2+)yk5;BFpIm zCcv8Q-E-v8RmGZHaba##g{pVA#EfVr!uZ--7FdSc19*|q#-_|>PaPtbWEYdzZsDL` zDcyNUP2Y^RY#}|a4-yDt@&QM+KsW|VBl}{f3gy~rQA(;APujCbIFd zhBs4ryUnb-XpV%IIGu75uk6U)AVp4R`}AXuNwU)Sm>2z<^-FH%Q9A8p>Xws(Z7+GP z3Z{>ZqHeR^LZ*6iHg`(1vt6Ar4kB*6#GITVPklSV#hW9GqixeUwu1WXJ2SJW)QMg^ z+)o0xo3*GiT#)7;2QHY^v`I3j-LHrizB&Mh<1lg!aGhbYfvS`=!hlJdYaO|3P$aAr zQW2w-TFq()f^?l(tEUAQ&KZEzp)$uqv98373Y{$_LO@vxuJsjuPyeZXk~ZHIBT91z4MR4%KbggRq>foC^jYO38u5+noJF~ifRPmMXr6MV4Xuf z+nk6P?@}ex?)R7l{Pk{ZWrHxkxCR@~QD#J$bPxNFtY);-c4&y>Z6lC=#dU`=vyKn( zy`Vyu_~zI-JL^N3MQV%Y^S%}v7Kh6X&vp6lKD&G2(>=v&!K05J0Xq}W@%ZOiL-NIi z3xD-rdQWxjYRg=T;>T-mMBh-oc?5e5JVWkYz4#_&R{1{zfRZsh*CO%XlYu|~|Ay87 zWV5yZ$!2f=pDxUJ-(9M|si-uWdGBAi3#8G&@jr_N*g2{GPz|Dg^e3a$lSj5ie0VMO zV&RZED-Rp?0u2?zOU#NQ#rhFaD?rpCne4G>&_H?T&8kWVS9oc2ZOfxuqq?%P?Ka;-m$h2ud#xUR{4#;I^m|am~TNScGpDdxE}YddST&9*t1HPVV2jGe4zI|4Q#= z36?VWyphqRZdsTUbs@O4MRQKDj?}uKhc<*K8jM+*_imf7ug-R1`PZMSN_cfQu$zym+pl9SN#fVvn0lVAqN3O=tjm#wt`khC{Xk z;m+OXV#djbh)zcLD#V^#H%VRCdXR_EY(+#Qd^Q;uY{HM|dLPVxNt62~Hz3%}!H{n` zd3ouFzVkVqTcI#J3aGc)(E94^LXIuvEnO!dR)G>@WdYN)IO^IA_(|)=JR zacT^%@Uyzq(v~^9{fp(pQB~q#|=d|9p4!zbkC4^bpvv~w0Ix|d7N6U zpC}TCZ9lWC1hgy5|Dd3?$15AMR3G|wtr8ASp}XH#A}kWJEgSX^juWLegKsWg9^##i z5os6p@@+{S9Tg`urf6s8pa#vThkG+`gg*Ena#S&o&H);oj98G0xK5r_vFlrmOPH+o z(wYqndhM0ylunaTYF7G}a`TW{5v3p|n%1TswhKXFF3VO!i1%_rDRq%pL3c~KWw~ESW-JUwNa%0)0&UCa@&KkQ?f&>l6>}`wc)_96ocggx*DRrm3F@Sa= z+4ESCpSCY?sND$Ol&;9s?kcZF^0TXn#iP=y<_MCtdq_oTg2jC~$=-I$5ZQ$^rF!n< zoD6k4l&Ma)qVZ|O0#PfYr?FnqO}K#P@L^L;G?}*IzU13TI!B!`kBpAVPPyG5xIg0D zO1ShpLJFO9GjhJK;b%)T_L9p50!alCT^ZsRW0nVAxc1bhL&XN}@r;GK^Zl{7Q`ACDa3E@6+g`klpK%lC)Ccicwu~Y0;j6kj1M?`AN zS>#Li0>+21eI>=}eS&8}pW%|J_Gy#}dGF!zS+)+M0&>FAyY{uPsH_)jlh$9awP9r= z9&=nfOqchIs$q}A zeIL?6tEmUEoEu`YDH=#bkxCXg_+`~Z7q+nGm1zZu3wrBjJevyX_$!M^L+qp)HVoKf{wDk!%oQ9{t(d>#NjL#RcGm3CaD>r7@QTI=uR}W3FEnw3N_{R~?KhbMF zf5h^~EkF+A|E}vn^bc-Py?puqH1-|faCKeVqeMtZ7`;T48NH4!hy;TfC5RS+=%No1 zLJ&k9!sz``M+u@EqD37wdQC(xL6nFP@%%@g`u^Yd{@1^+%RXn$*=G*+-fQ1$t$VG_ z2IQ&$u3LZ@*f1d0el>HM#Df? zh(kmaA)vrt!1Lnugdh+kFi;Ug#YPIbbcZ~aq`0egpv+nOda?VbDN7>@w_1sskN%hV z*{g@v%^yr%)iFDa^Kc*1FdEmZYM7$$7)~`tXbspDtQlcuHJi`kjP+7+TBr^`bWLR! zX$LG@SQn}QO(^KMj-xCAJGQ@MMdnm$m1IeNdn54Y%EFaNS z^9uKnzVTdUdL6Q~mcJm00emx=`W->K305n)vpp-yk9e^U6OSXF-%r!@qoX*L*3U&g z`y3zepx<l@Y1RvCCv8TGJd;WW z+=@VALSY<>F}wvOrjhQlgm=Q<;tPC-|?)8+V1YKB<>Bo;nb|QN|y8w>7IdgS9^OHxd&@b#(y&IoAR}c&DGwgDB zA%XAQT!h1T69h($3OaYQ1+Dj>@4&cWJYuL zT>jlWSJO{zv>Kh?8vf>VCnIfD&u0euLYaG^5H|Pu)Q0GO89=>%%&PQ zpQWhdd=KX_tPR+X0Jsp?DegDFTtR* zwa_yZL}8Q2DI)mt4Z*$I0D5&LfA?2JXVcJ>q+>ogC_?Agg~$kn{3 zHQgIb++rc;N}&7fp;S%$wdF$@SG};8XXykNF3X^4)))5S86Jf%#yD&`;?&;cyM0Zo z8SR%9883dq&tDl=DIZy@&7X=JpoqXkyCrE%vB97Wx%a@p-W3FIK-7P@&ImLBVH2nb zs-n1%Iv~&s@`R3+3l7O&NKwA!I@RIWH`Te@?x#mD{f+5zAo2$iL;TF?8V@wC zcy&{_#C1>Im2u{Jx*dj*In89wGBY5pTyxF7>QDc2!yX&C=`P)BC7V_zTOmDP#wxHw zE-Xe-JLGb;+%FeRod6^y&v&Wq)9EyzDi~BsK<%kGJdwdSaSNV&8Z=}Y)j{yjB|W<3 z?X)jGH>2YDEzO|tZ9iD^L|8;nrbxa%fSQaS8=7IUpkb59U0TJDZ$T3OWbg5s`|3*k zP_W3Db~)?dNaEO(7uTmLHEBLQL1Efp;wSS^mectuKSz3+>-*wD*=Ia@gZg{wPr3uT zo!lpfzLsS0=h(p#-fI4catooMH$9xcORJQh?z=1)VudZAgEW--9wrpvMoLm9jx6GP zc*@OL%?V!@*SYOsO`qi-`yUJ?rB08tG*pg7EHAv;eJo~RpO7*B=s{0LdO+&vSl92M zTmJ8wB_<4L3K)@iA(cK?10HFd=SXJ7+Ua84UVhcZNRo?|jS5KGZAUcmf;!TD zfm`-SdM0I+r}?L0+?eaUeI1KmE2pIMYnPI|H20Yl?j)g*-_k=+XVn9T&8-IucMcwl zp8974yqSj|Gx4|2s)Jc|eV@Hq#sS^@F^+Pk^Sxi?_6JXPD8GZAdilZ4>Fv9-J7G6G z^rBYA#9nd9{0@4snSGg?_S}0K2D9Q_=4uf$QIC2`x9T#wgSaKEwu4)3&bc(Cjc4Xm zuuC~nQAx{sk1j|iPA2zj4pY!5m)@1X$sERNrp?mKyY=H~P1D_hr=w#AA4f`EqqaPK zsVlu-M^E*6r1|(cw+=+p+m$+-*szu=6};5XPkwu_zyDFAK7QkDI5F(<%`lp&^CZs+ zfU6*VoRs9=+s>6Vt-N}G|7tyUDD&_`l`k8P!OksxXw@bgjC>kfu=S`VU_K%Hye(Ng zS~r>QsqOQRK0Di23U-bO40U*?OcHGC4yB)(7q@7d7Zl#}<%JJ}Z|8v4Ar{fyOc$tM5RW0UaDp6==%3DAV|>v6La_I%?8Kl<{V z%nfO#+B4R3J?EQB3xG^WQPxPNL{wvfVPt4_p-zj$_{kLHCu2<3+;5nKmF5#c{=$=* zpT(4hPjn}L0y{jI*3_yiFk^lPH44(RES=Z7`wya79w$pk|6o|Z(cnkE38|A7k~ZWtGS0x>&&aKY5MwH?)>$t$pNPCd(Ky8x`)o1K3yyJ zbNc#EVm5FZjL{OcgDz2SLH;kphZKv-VK5`4jZ3qUNbk+R4VrY0(tm_)1Pmj}&mr&GlO4-1o z9HTKd-L(l*%`!Su;jXLa<}6qEc0fw#!<8fXxtV19sntVizRAG})1}Av_n-ZgES4_o zM~n}P=1s~ibL(lBE<1Xlc80aaAUNPNUbXu=E>z!%%c&cFMwcc5i$^Pk>IZ4l^{hz(snjZlM80JU3k(PE68Oc zLu`8d;tPb1PL%HSSHm8jrO%!RC_hm#D_vaGN7Lre;>qvTnQ!A98!&65M-upMMxB^1 z8L6QegNS>_%kcJ8U^)4Zl z<5+KZ1CNwU50&}A9P(%Xw}t!M9Wu723#qz0`qUZKEO34yz13YMM1`vZk0vDjJR!;B zdy?mA>ePcOKMAR|DnL)YpNQ0x8_jgyUYi615%R>7@l~_v_{OQXWtH`NM3v@_4S1p5 z{sl9J{Fw4|)7?IOH{W3?gOZa1D0cVJ$lzkD`}7EWzE~aqjAxM!I#$ZJT9i%xR=r$1 zZX^$B%9b>3W}4hkTu`};+Ig&dT3o-H!KtBNCjX>E5GyQD7Io@NZKVEXGdNpQuWn!n zuP(fb|o)keI*|xr6T9U2(gUdL%#^axHY~AhrJWEKrG~3lSkbpJDewnUg8L? zu67)6FYdbi7(dn?6)Bn|##{1{CH2u${_e*L8dI8?t53SDk|bg8CtSkxUvlMeN2p?J zjts{bmAJX}v5vQVhvmK>GQTy~yP2KLrNH^hSe~cum=JMiV_+ z591FxnYh%6YrF!Ywx`Btw-X+24|1qWdhx#^ocp9dq8iF-Box5=)_vS>$OV~YQvA7> zgW7f6ZeT9b2a~g+O2wx}wunqS(*QCI4tR%oV|egZw@7AalleL+DF}bg8Tk?t&75&fNCjmG>gYdcXxJfaqHr2!N!?DA59-_aaG+H zJz*BQHIM1`xXF#93AH(zqAy)@LDNZo7p0{i3p)_I=RdJ=AKW7b;rO=TAz&A)Oiqm^ zY!8N~b0!$sPw-Sl()FYwzORZuaVnF*A!em)RJpbqPIjPu=JR>azZr5f=P>J@*ESw! zHt`v)cpTC2mNR;;oJCmn$VxhFZ`zpu4uXlfSFkcKzxrXO;hghM6V+b&vNQViR8h1; z_uIFJ@v=}hl2F5i-;)#DV!C7Y5hmBzi<*4UBEONz;O)$r2!ah;+x7Z43;(jFUfK< z3G`pNRQV@Kf2o+R@Aq%R7XJ27N5k(b0uYRfjg5ggkS&&QgAEh}$TrZg*9rv!XTV)D z&>eCNp@eZVp5>=Lb%F`1eH7#Re$3{I>70enT&cUOK&g$N8TP4ueg*mRR!$Sr{*<8- zcc1&lUe=Xo8PM=A+`X+l4PASx0lL-22;=m*{(*k?foI+d&<7pKlDO*Z;*t6E-9<9w zPm`h_jnyrS*SL?YK9A2(fJ?6R8m0j3YhApX($Yz6y%xRgH?tDhmm`=*Ef8YF>v7D9YSn~uGCyMR9GmJ9K0o*w=h;19 zCTzP^3rmbM(7HXolyg05?TKe*{4MD6`07!#ah3wJKi8K6xId3B7FvJfz$0Kfb)l$^ zMQAN=3NTqBvn6i*7p+nlfui#(s(pHAX#Vl~LaE9-Wu{Ann--jxT zyOj;LG|b5=;CbEuj#4S1bk$ktrVn^I%Y+JzQk3dmzJ4{pl2iJK(dWS58a#=LoQ=%m zL8-(vF(6qEE9o9erVI5h=Sog-oEwBWMNJT zt8S~%lVxeilUd`XhfK%hr-{qf*uh7qsrx?b3XjC+x?*ww0{z5_Xc-l=BS4&b_v46&I=+c65|N?1Ox*tOwxnw>c(NtXO(Pa&+Wyq0m3 zI9W(|=BwnfZdBKs;nQTPpS|}w`r!a|@yP7sLsHJB^RVoJ@S%JD^skg$Z+PuMKMW6| zMAIC+r@)&K6{{5FXCeCao7Uy-*gJFIE*aF_qR*f(FvN7C$|4XOK+3QuuBCF32KRG6B4i|0qik^~8%Oqc z7kNhbq0DqV;RP2 za{}dGXVA@ za$dIzjve*8e7Zi#+DdVPcgxH7t%U7YdbEqOa=RiBuJT?gvkn`ZNpo_^Uk`OwssW-& zLHQgv(_{4OWK)!?owli?A1vF)&d3n&Yivd9U(QuExi($*5npZU$NRKQwU0^)uH|SG za^D(f1*}Ym9ApsNv>uTn9TsVBbXWf+M(BoWZOZ<(QgRBT%xQ{D-enlH`FQ0kj*3W#J{JFB^_#2D=NN}@?jSzj}X0R=0rdwUc2KuUV;M>t^ zSQ2<=){Iu*KulArWmWoP2A$bVx8@rlUzy!^F>PoR_=DiFnD*6xw)gq@>81m@vnEp8 zUy)1Sf!w*{?$@Q?&1{hfn#wzq8E3^68Ty67|0*jkuYg_k}F_2 zBT}W`di7g5+L&C3r-etM#SA@_!B|^NXZ67Zf2?=L_%cy?JsQ~AR(JJ__{!YXf*aWp z!fj4tm+5dJQDkt3z1Qn$$bhhKB2S!C6d<@$3aFiRD+zh^Q51WXB-r9r=B!v6~_ z5ZnL#%)d_vPLlz}r4TNZNqv zA|{CPJ%}WTj4f9trH>Kr4bp3jfgE_$GHGxp#@9{im~BZfO+k>7h=bWoRy{|jj52u8 zK&^Q-cX<;cZw#uwIlB6CSJWyWzmhXO8}ob_IsS6y8qWK5JbrV%aG2fPj$e#dJQp`) z+B!d!cHrYEewzB>!{!#s+<ij>)ZVJteJJKKv4Pcj;!~@tl7Ak)!|Q7f>uaG^Y8vf$8K7s9wubdQ zw9ZJaqb;fsiB3@IGj{baOU{8LiS-t+AJLG+6T;cVoupT4e)%b`N}{1lUDzS}az~0sz5UB*aVK;^ z=7Nwi&nYwKje7nY956@TFY(gYz+CHG${0g2@w<$q{p?5*!4w>|Lm|rh1B(5@vOwN7 zIqs~78j!vHJ%3y0$MCbA9z8%OzmM$N>|@q^ZeQL~RKeKr7K=;wdARCNhbtKS8Bx1G z`p7ZZ`-6hR5iEf^D$}H=#BWIn@g1jP+4VNJ@igzs+M+<$Vw|!ydY(28Rc3Ybfr44- z`*&<}!0(nP_tS6vG+9gsBgIe#7&(Ex-f=LN2j8gZmV0OD3i|C<*Bp{+tRs4GC&a{M zCowk#-@z2&WJpWHVqOYT8ZW=?^X{9NvpU0v z2f+@U8g^m0&}BzIftiPc+tGsi)T9kBJ?(KlPMHt6N-vX?g4fJ*x67jVjK0o3SEcJZ zT~F}&0R8UhQH`yS5V*mKIWta^F!(0PMWPr&&5=(GOqdJf!EeBUjgUYQI1qLE9X*fp z`a5Ns;SBw+kcrEH&432D|33@CY5c#e1c3YLVzM&-Wof<88~||uWKO?Fsdf<~aZ!jt z#Pqt?0Z3t;ga=gS3Lv=QF75pDX+i~S~NDF zC8xIX)u!$aSSC6~nw#VvNFcOO{d`i#8QT8VGAB&yXVQBWWoMmI?dGQit(Ejv3PRys z)eLg0El8AD=gAAf?MG868_3U4ITNaT-)zB2RDvnBv9GzggZ28qr+@bnb?fgne&EgE z{?5Pf94zhE+vKQUK<+V}qRQGJS+q^#)|Y-a1~Qd$uf6j1o1{g_c1{Qy!@w32K`diI zHbidPN_6?L>7Z`)64F4k4l&^Rz9W2c9d||bR=fw@kl;^7Q9m>K;hYM{A=i`!`g1(G zs`Z;y7Bs6gm4~9tO$+Eia_45_waT^{y)%%3|HW_8-kmcvS zaAa>TxE;d)WrKo)ls~;8Yg5Z0U?8IoQbb>)W)GBTNNGsdFj8^A;&hK+;zlqkj-|Ro zU8-~^yL+jFjk-g6K8LE0PqWUcv!D3wgAMZHiyFQ6;gSI#Zq66*KX&aZjQpmahkc@n zP4IX^qyCjsPvcQ>q3~n(og~dxb=??thOTEEVIEMyj}#M@`~#7LcxrM^zcMWVxS1DPNH2>eSIm4z7?*wpfhyMkBhOB9fB!&l zH>J8eGFfF!>%hm>2<2m46it2~wdWF|i_peGf5ejtxb`1fYDb*XE0N*xD=9*x`z-&d!&CW~dl@(bG* zS=9bZc0#<8kGMl3YDZCU4O6sf=#)mU=<>8Xy+fg5PZY{K5Km5-DnqCXrPM-%DZ*93 zcZ?~my3tT!4rfN3)1dsW86!B$hiM@vD&7O!6M7cyvv7m_FlF%&UiW6eP%UFMh{HOL z1dfc0gqcQBC3x8yb%;i!t!@h-QVu06Xr3FcIKth%9n#Q~_kclFX-ES{Gw(gb5?y8_Lze zD0AGS4kQDlx>{n88?l8&tPY7FW9$f3S^nb;%^fDEC;?hQb7pgijaWW6e<>XX2b4t8=Zb_nc8@u=Lk9$`?Fq_Ufq>g9NX(mg?88&CLiUupB zNA1Clx;HBBXQ$c}~@!B32uC&gfJ zM!Qq;Rf8iwydwgylNNi6rX$#?0m2i{e~vwXlKPjjb>ZOuOABi|djsMK{LiL;e)``O zujhZzQ#Ak?4dS6x4*Avem*0#KL`eBd(Nhj=CysIYt?mxx2>|*)%Qwez=Ob0bZgfv$B!t2nR z%{7!K%uFKo{TeX1x_u{~-n`7NrP2ZETo3xvqY~YL>^3{g?Xdu#&jb80hov)8ZTFQd z!R9YwmtDkmM4YC0A6+$o@9xJ)#otXLj|U{@+&p?4euw6|Cz+BG%^PbRfYI$|-K@i7 zy48vjY^Z6hW8vzI<#7JE1O$4yvF#4^_LhNh*Ybpdea53@tP-))D}mcjbMooBjr#hI zQx0r(yh%ABA_yV~I*v5K2nr5|sSsPoMPLYZARPC|$-!6HfShLSD-)bP!4XvO@9tbW z^&0j27F$DC$+5+u)b>Sdb8!Efc$vmb785)8TGCUQ#`LFaOfk<-@5i?fWwKCz$YgdJ zddg##(U|&J6rJ!aeOVwkl*QOGwZ2Ymw9hwv5fT|K$9R9?o-8fWBc~vXD@I=%wIjJ# zW0WMX5aSrqO5#={n;AD*GG7)GgEWTxOA20e`Cm&n2uP#)7bdtME&l};Ec}f?REifZ z|3|r5{0qPT2???|e?fbIw}0W6EmRZ%+Jpb(nGgm7V5Imr;EE#X5+#TXe**-H=w%}V z(GabWuH9k5bA!m$A%Se&a9aTiO`DW-CQ+D3JA;^gB8$M$mFGsw%RkXRhtORxp=H&c zMwZ!pQmbY4rm*(mzC9zONZ+o7V2}2cyGlRI3eOE4pAk(~?tGkH( zPqv7B<3_Bx7 z#9FfsFJ&D}MY*RJ4_pt#GjdQvNg;Sj1P?|L7!-mLz-+cagB2N4>TCxXsmi?_Q3ZJM zevCFr6bI%5-8HUK&CWTEl`Z=WkMXZ{QzRc(Z+$MawohCYL?4_9V4(_JY13aH6m{9X zPALAq+L;!p#0l*4qB;WzHwOt}7hy`GzMUbj4 zXK2KU(0m65>B{I#fHt?0uVZC!RyZI+h0=tA^*) ziDTpd!&L5E)M{*TY3!VkNa&v{)5={a0S^Xno68U)*sdvph(#FiQObc5cqS)&bDrr~ zJC)MCS<@QX;1JE89aEjBEJ8hI@}d_NC^D~m!N_{dDb$>i}5V&q4CY_nLB zx66g)BI~B!rNd3a%l7EvmsDkJv{SuOv8)!(6&-^8B9$E3bUuuCovvk~@FSyh?}p^* zQ21x*j5fAC`D3PODjP_i#SNDgFWzLZtW23QokE-fi;6$D5M1lJ^*iJj?zgt~`-I@2nvtz*8y;y$zco5lHI;wVQ0()Rk0 zo-m7O7a0kw&VB}oy@QVkpd*n=U2ifDkgqR0lar)r=Irn5eOjCA9rvGtJ)6T9TGJtE0LL zB<5Sp%2K_oD&JNIgP6vM=|OaF(I6oe3^fczB&k3qmP10o`2Bg%<_i4su~ zvH|8Q>i1xMOA$3|hFC2UNg_Ig@+S*&V-jysB099j9uuXbeS4lGT#)jyBTsvt7x@D* z197)q>*dO{@U|#NLsCa8Thi_h_-uxT^i!$5l)co^zR6shj>>E@Ip|1T?G5~**dkQf ze(u_h2pw3vbO)1U4^hZ)h}-sk%Q#*}o=^RRoUGvcZZ?5&nmHW0P!3&E0B~f0V=3b> zfH^^jj1h7dN`qTG15Cog*TBXqa2XS>R_RsDmxl2ZDBxJqP8Cy)ubs-9_{RSe+ z0R{t57%nyS`rmP)Px;lh&CCJ>QkB|ZW(l{r6R8|@4@!w(!x*f9Ba7cDU$cI}ASz}n z%@(LdYM~?w;{<`(QyBnZAdz6*+s3p9C*QARfQhyT zqD9trmyis6g*ZqTgiEDVBSx?#z9CfPAqWJ80`?G_AQ56J3>?&TAmV5xnI_6M2I-rW z5i%B1cQUNBi?fueA99tMUVnI1x-M>_5%(ozW~dSY^U)+2ZUGXptFPqn3-vEK?005v$Y$xZy`M>{zja@|;ZRmah!ZmCZfj3o zMM!!f^Z^$T;41#_<|L+9{U9P>H2iPnC-Ps@8t^Xvqd)Ib!2VE`13^*0{d$yu1O$FT z#HI$!by7eDH`%=z3u?ci{_9>FBk1mj6{*Y!qy<~B0KClT=hNQA772TdKQ28tj9F>n)qB1Ky2uz`qw zBaTIYNPaC%#4h15QQ#j0v)r?`>=ktoQ(k&AILLrd$e|HNPkNhObHPSqyKqyDVD(*( zib_@9T?ZE5Jc*<_pr?Hv8{IPK6PCYbTk^I(EZaRPH)hZ&t|BZQuBQcKALKdY)Ig2M zELd|K97Ep{r@8Db3{xJILzCczm;*2|E zq#vvrpf1;SgEP^SV_>A@x|Jm#OgT0J4 Date: Wed, 29 Oct 2025 15:17:36 -0700 Subject: [PATCH 15/20] scripts(evals): simplify eval configuration and allow RunnerArgs body --- scripts/evals/eval.py | 42 +++-- scripts/evals/run-release-evals.py | 276 ++++++++++++++--------------- 2 files changed, 158 insertions(+), 160 deletions(-) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 5a164ad84..550b50809 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -10,9 +10,10 @@ import os import re import time import wave +from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple import aiofiles from deepgram import LiveOptions @@ -53,6 +54,14 @@ EVAL_TIMEOUT_SECS = 120 EvalPrompt = str | Tuple[str, ImageFile] +@dataclass +class EvalConfig: + prompt: EvalPrompt + eval: str + eval_speaks_first: bool = False + runner_args_body: Optional[Any] = None + + class EvalRunner: def __init__( self, @@ -93,9 +102,7 @@ class EvalRunner: async def run_eval( self, example_file: str, - prompt: EvalPrompt, - eval: str, - user_speaks_first: bool = False, + eval_config: EvalConfig, ): if not re.match(self._pattern, example_file): return @@ -112,10 +119,8 @@ class EvalRunner: try: tasks = [ - asyncio.create_task(run_example_pipeline(script_path)), - asyncio.create_task( - run_eval_pipeline(self, example_file, prompt, eval, user_speaks_first) - ), + asyncio.create_task(run_example_pipeline(script_path, eval_config)), + asyncio.create_task(run_eval_pipeline(self, example_file, eval_config)), ] _, pending = await asyncio.wait(tasks, timeout=EVAL_TIMEOUT_SECS) if pending: @@ -177,7 +182,7 @@ class EvalRunner: return os.path.join(self._recordings_dir, f"{base_name}.wav") -async def run_example_pipeline(script_path: Path): +async def run_example_pipeline(script_path: Path, eval_config: EvalConfig): room_url = os.getenv("DAILY_SAMPLE_ROOM_URL") module = load_module_from_path(script_path) @@ -196,6 +201,7 @@ async def run_example_pipeline(script_path: Path): runner_args = RunnerArguments() runner_args.pipeline_idle_timeout_secs = PIPELINE_IDLE_TIMEOUT_SECS + runner_args.body = eval_config.runner_args_body await module.run_bot(transport, runner_args) @@ -203,9 +209,7 @@ async def run_example_pipeline(script_path: Path): async def run_eval_pipeline( eval_runner: EvalRunner, example_file: str, - prompt: EvalPrompt, - eval: str, - user_speaks_first: bool = False, + eval_config: EvalConfig, ): logger.info(f"Starting eval bot") @@ -262,17 +266,17 @@ async def run_eval_pipeline( # Load example prompt depending on image. example_prompt = "" example_image: Optional[ImageFile] = None - if isinstance(prompt, str): - example_prompt = prompt - elif isinstance(prompt, tuple): - example_prompt, example_image = prompt + if isinstance(eval_config.prompt, str): + example_prompt = eval_config.prompt + elif isinstance(eval_config.prompt, tuple): + example_prompt, example_image = eval_config.prompt eval_prompt = f"The answer is correct if it matches: {eval}." common_system_prompt = ( "The user might say things other than the answer and that's allowed. " f"You should only call the eval function with your assessment when the user actually answers the question. {eval_prompt}" ) - if user_speaks_first: + if eval_config.eval_speaks_first: system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" else: system_prompt = f"You are an LLM eval, be extremly brief. Your goal is to first ask one question: {example_prompt}. {common_system_prompt}" @@ -330,9 +334,9 @@ async def run_eval_pipeline( # Default behavior is for the bot to speak first # If the eval bot speaks first, we append the prompt to the messages - if user_speaks_first: + if eval_config.eval_speaks_first: messages.append( - {"role": "user", "content": f"Start by saying this exactly: '{prompt}'"} + {"role": "user", "content": f"Start by saying this exactly: '{eval_config.prompt}'"} ) await task.queue_frames([LLMRunFrame()]) diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index ce380158b..2f6038c14 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -11,7 +11,7 @@ from datetime import datetime, timezone from pathlib import Path from dotenv import load_dotenv -from eval import EvalRunner +from eval import EvalConfig, EvalRunner from loguru import logger from PIL import Image from utils import check_env_variables @@ -24,190 +24,184 @@ ASSETS_DIR = SCRIPT_DIR / "assets" FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" -# Speaking order constants -USER_SPEAKS_FIRST = True -BOT_SPEAKS_FIRST = False - -# Math -PROMPT_SIMPLE_MATH = "A simple math addition." -EVAL_SIMPLE_MATH = "Correct math addition." - -# Weather -PROMPT_WEATHER = "What's the weather in San Francisco?" -EVAL_WEATHER = ( - "Something specific about the current weather in San Francisco, including the degrees." +EVAL_SIMPLE_MATH = EvalConfig( + prompt="A simple math addition.", + eval="Correct math addition.", ) -# Online search -PROMPT_ONLINE_SEARCH = "What's the date right now in London?" -EVAL_ONLINE_SEARCH = f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}." +EVAL_WEATHER = EvalConfig( + prompt="What's the weather in San Francisco?", + eval="Something specific about the current weather in San Francisco, including the degrees.", +) -# Switch language -PROMPT_SWITCH_LANGUAGE = "Say something in Spanish." -EVAL_SWITCH_LANGUAGE = "The user is now talking in Spanish." +EVAL_ONLINE_SEARCH = EvalConfig( + prompt="What's the date right now in London?", + eval=f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}.", +) -# Vision -PROMPT_VISION = ("Briefly describe what you see.", Image.open(ASSETS_DIR / "cat.jpg")) -EVAL_VISION = "A cat description." +EVAL_SWITCH_LANGUAGE = EvalConfig( + prompt="Say something in Spanish.", + eval="The user is now talking in Spanish.", +) + +EVAL_VISION_CAMERA = EvalConfig( + prompt=("Briefly describe what you see.", Image.open(ASSETS_DIR / "cat.jpg")), + eval="A cat description.", +) + +EVAL_VISION_IMAGE = EvalConfig( + prompt="Briefly describe this image.", + eval="A cat description.", + eval_speaks_first=True, + runner_args_body={ + "image_path": ASSETS_DIR / "cat.jpg", + "question": "Briefly describe this image.", + }, +) + +EVAL_VOICEMAIL = EvalConfig( + prompt="Please leave a message after the beep.", + eval="Assess the conversation and determine if it is a voicemail.", + eval_speaks_first=True, +) + +EVAL_CONVERSATION = EvalConfig( + prompt="Hello, this is Mark.", + eval="A start of a conversation, not a voicemail.", + eval_speaks_first=True, +) -# Voicemail -PROMPT_VOICEMAIL = "Please leave a message after the beep." -EVAL_VOICEMAIL = "Assess the conversation and determine if it is a voicemail." -PROMPT_CONVERSATION = "Hello, this is Mark." -EVAL_CONVERSATION = "A start of a conversation, not a voicemail." TESTS_07 = [ # 07 series - ("07-interruptible.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07-interruptible-cartesia-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07a-interruptible-speechmatics.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07aa-interruptible-soniox.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ab-interruptible-inworld-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ac-interruptible-asyncai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ac-interruptible-asyncai-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07b-interruptible-langchain.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07c-interruptible-deepgram.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07c-interruptible-deepgram-flux.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07d-interruptible-elevenlabs.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "07d-interruptible-elevenlabs-http.py", - PROMPT_SIMPLE_MATH, - EVAL_SIMPLE_MATH, - BOT_SPEAKS_FIRST, - ), - ("07f-interruptible-azure.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07g-interruptible-openai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07h-interruptible-openpipe.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07j-interruptible-gladia.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07k-interruptible-lmnt.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07l-interruptible-groq.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07m-interruptible-aws.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07m-interruptible-aws-strands.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("07n-interruptible-gemini.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07n-interruptible-google.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07o-interruptible-assemblyai.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07q-interruptible-rime.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07q-interruptible-rime-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07r-interruptible-riva-nim.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "07s-interruptible-google-audio-in.py", - PROMPT_SIMPLE_MATH, - EVAL_SIMPLE_MATH, - BOT_SPEAKS_FIRST, - ), - ("07t-interruptible-fish.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07v-interruptible-neuphonic.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07v-interruptible-neuphonic-http.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07w-interruptible-fal.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07y-interruptible-minimax.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07z-interruptible-sarvam.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ("07ae-interruptible-hume.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("07-interruptible.py", EVAL_SIMPLE_MATH), + ("07-interruptible-cartesia-http.py", EVAL_SIMPLE_MATH), + ("07a-interruptible-speechmatics.py", EVAL_SIMPLE_MATH), + ("07aa-interruptible-soniox.py", EVAL_SIMPLE_MATH), + ("07ab-interruptible-inworld-http.py", EVAL_SIMPLE_MATH), + ("07ac-interruptible-asyncai.py", EVAL_SIMPLE_MATH), + ("07ac-interruptible-asyncai-http.py", EVAL_SIMPLE_MATH), + ("07b-interruptible-langchain.py", EVAL_SIMPLE_MATH), + ("07c-interruptible-deepgram.py", EVAL_SIMPLE_MATH), + ("07c-interruptible-deepgram-flux.py", EVAL_SIMPLE_MATH), + ("07d-interruptible-elevenlabs.py", EVAL_SIMPLE_MATH), + ("07d-interruptible-elevenlabs-http.py", EVAL_SIMPLE_MATH), + ("07f-interruptible-azure.py", EVAL_SIMPLE_MATH), + ("07g-interruptible-openai.py", EVAL_SIMPLE_MATH), + ("07h-interruptible-openpipe.py", EVAL_SIMPLE_MATH), + ("07j-interruptible-gladia.py", EVAL_SIMPLE_MATH), + ("07k-interruptible-lmnt.py", EVAL_SIMPLE_MATH), + ("07l-interruptible-groq.py", EVAL_SIMPLE_MATH), + ("07m-interruptible-aws.py", EVAL_SIMPLE_MATH), + ("07m-interruptible-aws-strands.py", EVAL_WEATHER), + ("07n-interruptible-gemini.py", EVAL_SIMPLE_MATH), + ("07n-interruptible-google.py", EVAL_SIMPLE_MATH), + ("07o-interruptible-assemblyai.py", EVAL_SIMPLE_MATH), + ("07q-interruptible-rime.py", EVAL_SIMPLE_MATH), + ("07q-interruptible-rime-http.py", EVAL_SIMPLE_MATH), + ("07r-interruptible-riva-nim.py", EVAL_SIMPLE_MATH), + ("07s-interruptible-google-audio-in.py", EVAL_SIMPLE_MATH), + ("07t-interruptible-fish.py", EVAL_SIMPLE_MATH), + ("07v-interruptible-neuphonic.py", EVAL_SIMPLE_MATH), + ("07v-interruptible-neuphonic-http.py", EVAL_SIMPLE_MATH), + ("07w-interruptible-fal.py", EVAL_SIMPLE_MATH), + ("07y-interruptible-minimax.py", EVAL_SIMPLE_MATH), + ("07z-interruptible-sarvam.py", EVAL_SIMPLE_MATH), + ("07ae-interruptible-hume.py", EVAL_SIMPLE_MATH), # Needs a local XTTS docker instance running. - # ("07i-interruptible-xtts.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("07i-interruptible-xtts.py", EVAL_SIMPLE_MATH), # Needs a Krisp license. - # ("07p-interruptible-krisp.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("07p-interruptible-krisp.py", EVAL_SIMPLE_MATH), # Needs GPU resources. - # ("07u-interruptible-ultravox.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + # ("07u-interruptible-ultravox.py", EVAL_SIMPLE_MATH), +] + +TESTS_12 = [ + ("12-describe-image-openai.py", EVAL_VISION_IMAGE), + ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE), + ("12b-describe-image-aws.py", EVAL_VISION_IMAGE), + ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE), ] TESTS_14 = [ - ("14-function-calling.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14a-function-calling-anthropic.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14e-function-calling-google.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14f-function-calling-groq.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14g-function-calling-grok.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14h-function-calling-azure.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14i-function-calling-fireworks.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14j-function-calling-nim.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14k-function-calling-cerebras.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14m-function-calling-openrouter.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14n-function-calling-perplexity.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14p-function-calling-gemini-vertex-ai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14q-function-calling-qwen.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14r-function-calling-aws.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14v-function-calling-openai.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14w-function-calling-mistral.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("14x-function-calling-openpipe.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("14-function-calling.py", EVAL_WEATHER), + ("14a-function-calling-anthropic.py", EVAL_WEATHER), + ("14e-function-calling-google.py", EVAL_WEATHER), + ("14f-function-calling-groq.py", EVAL_WEATHER), + ("14g-function-calling-grok.py", EVAL_WEATHER), + ("14h-function-calling-azure.py", EVAL_WEATHER), + ("14i-function-calling-fireworks.py", EVAL_WEATHER), + ("14j-function-calling-nim.py", EVAL_WEATHER), + ("14k-function-calling-cerebras.py", EVAL_WEATHER), + ("14m-function-calling-openrouter.py", EVAL_WEATHER), + ("14n-function-calling-perplexity.py", EVAL_WEATHER), + ("14p-function-calling-gemini-vertex-ai.py", EVAL_WEATHER), + ("14q-function-calling-qwen.py", EVAL_WEATHER), + ("14r-function-calling-aws.py", EVAL_WEATHER), + ("14v-function-calling-openai.py", EVAL_WEATHER), + ("14w-function-calling-mistral.py", EVAL_WEATHER), + ("14x-function-calling-openpipe.py", EVAL_WEATHER), # Video - ("14d-function-calling-anthropic-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-aws-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-gemini-flash-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-moondream-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), - ("14d-function-calling-openai-video.py", PROMPT_VISION, EVAL_VISION, BOT_SPEAKS_FIRST), + ("14d-function-calling-anthropic-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-aws-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-gemini-flash-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-moondream-video.py", EVAL_VISION_CAMERA), + ("14d-function-calling-openai-video.py", EVAL_VISION_CAMERA), # Currently not working. - # ("14c-function-calling-together.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - # ("14l-function-calling-deepseek.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - # ("14o-function-calling-gemini-openai-format.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("14c-function-calling-together.py", EVAL_WEATHER), + # ("14l-function-calling-deepseek.py", EVAL_WEATHER), + # ("14o-function-calling-gemini-openai-format.py", EVAL_WEATHER), ] TESTS_15 = [ - ("15a-switch-languages.py", PROMPT_SWITCH_LANGUAGE, EVAL_SWITCH_LANGUAGE, BOT_SPEAKS_FIRST), + ("15a-switch-languages.py", EVAL_SWITCH_LANGUAGE), ] TESTS_19 = [ - ("19-openai-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19-openai-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + ("19-openai-realtime.py", EVAL_WEATHER), + ("19-openai-realtime-beta.py", EVAL_WEATHER), # OpenAI Realtime not released on Azure yet - # ("19a-azure-realtime.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19a-azure-realtime-beta.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19b-openai-realtime-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), - ("19b-openai-realtime-beta-text.py", PROMPT_WEATHER, EVAL_WEATHER, BOT_SPEAKS_FIRST), + # ("19a-azure-realtime.py", EVAL_WEATHER), + ("19a-azure-realtime-beta.py", EVAL_WEATHER), + ("19b-openai-realtime-text.py", EVAL_WEATHER), + ("19b-openai-realtime-beta-text.py", EVAL_WEATHER), ] TESTS_21 = [ - ("21a-tavus-video-service.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("21a-tavus-video-service.py", EVAL_SIMPLE_MATH), ] TESTS_26 = [ - ("26-gemini-multimodal-live.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "26a-gemini-live-transcription.py", - PROMPT_SIMPLE_MATH, - EVAL_SIMPLE_MATH, - BOT_SPEAKS_FIRST, - ), - ( - "26b-gemini-live-function-calling.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), - ("26c-gemini-live-video.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "26e-gemini-multimodal-google-search.py", - PROMPT_ONLINE_SEARCH, - EVAL_ONLINE_SEARCH, - BOT_SPEAKS_FIRST, - ), + ("26-gemini-multimodal-live.py", EVAL_SIMPLE_MATH), + ("26a-gemini-live-transcription.py", EVAL_SIMPLE_MATH), + ("26b-gemini-live-function-calling.py", EVAL_WEATHER), + ("26c-gemini-live-video.py", EVAL_SIMPLE_MATH), + ("26e-gemini-multimodal-google-search.py", EVAL_ONLINE_SEARCH), + ("26h-gemini-live-vertex-function-calling.py", EVAL_WEATHER), # Currently not working. - # ("26d-gemini-live-text.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), - ( - "26h-gemini-live-vertex-function-calling.py", - PROMPT_WEATHER, - EVAL_WEATHER, - BOT_SPEAKS_FIRST, - ), + # ("26d-gemini-live-text.py", EVAL_SIMPLE_MATH), ] TESTS_27 = [ - ("27-simli-layer.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("27-simli-layer.py", EVAL_SIMPLE_MATH), ] TESTS_40 = [ - ("40-aws-nova-sonic.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("40-aws-nova-sonic.py", EVAL_SIMPLE_MATH), ] TESTS_43 = [ - ("43a-heygen-video-service.py", PROMPT_SIMPLE_MATH, EVAL_SIMPLE_MATH, BOT_SPEAKS_FIRST), + ("43a-heygen-video-service.py", EVAL_SIMPLE_MATH), ] TESTS_44 = [ - ("44-voicemail-detection.py", PROMPT_VOICEMAIL, EVAL_VOICEMAIL, USER_SPEAKS_FIRST), - ("44-voicemail-detection.py", PROMPT_CONVERSATION, EVAL_CONVERSATION, USER_SPEAKS_FIRST), + ("44-voicemail-detection.py", EVAL_VOICEMAIL), + ("44-voicemail-detection.py", EVAL_CONVERSATION), ] TESTS = [ *TESTS_07, + *TESTS_12, *TESTS_14, *TESTS_15, *TESTS_19, @@ -240,9 +234,9 @@ async def main(args: argparse.Namespace): # Parse test config: (test, prompt, eval, user_speaks_first) for test_config in TESTS: - test, prompt, eval, user_speaks_first = test_config + test, eval_config = test_config - await runner.run_eval(test, prompt, eval, user_speaks_first) + await runner.run_eval(test, eval_config) runner.print_results() From 8fa6cbac51c51aaccd538df0474680aace0be0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 29 Oct 2025 15:23:04 -0700 Subject: [PATCH 16/20] examples(foundational): added 14d docstrings --- .../14d-function-calling-anthropic-video.py | 8 ++++++++ .../foundational/14d-function-calling-aws-video.py | 8 ++++++++ .../14d-function-calling-gemini-flash-video.py | 8 ++++++++ .../14d-function-calling-moondream-video.py | 14 +++++++++++++- .../14d-function-calling-openai-video.py | 8 ++++++++ 5 files changed, 45 insertions(+), 1 deletion(-) diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index a4daed481..bf35b9648 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -38,6 +38,14 @@ load_dotenv(override=True) async def fetch_user_image(params: FunctionCallParams): + """Fetch the user image and push it to the LLM. + + When called, this function pushes a UserImageRequestFrame upstream to the + transport. As a result, the transport will request the user image and push a + UserImageRawFrame downstream associated to this request. When the + UserImageRawFrame reaches the LLM assistant aggregator, the image will be + added to the context. + """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index 78bacbedf..20367bad2 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -38,6 +38,14 @@ load_dotenv(override=True) async def fetch_user_image(params: FunctionCallParams): + """Fetch the user image and push it to the LLM. + + When called, this function pushes a UserImageRequestFrame upstream to the + transport. As a result, the transport will request the user image and push a + UserImageRawFrame downstream associated to this request. When the + UserImageRawFrame reaches the LLM assistant aggregator, the image will be + added to the context. + """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index a669e1e46..867fc3180 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -38,6 +38,14 @@ load_dotenv(override=True) async def fetch_user_image(params: FunctionCallParams): + """Fetch the user image and push it to the LLM. + + When called, this function pushes a UserImageRequestFrame upstream to the + transport. As a result, the transport will request the user image and push a + UserImageRawFrame downstream associated to this request. When the + UserImageRawFrame reaches the LLM assistant aggregator, the image will be + added to the context. + """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index 0bad950ed..eb06556a9 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -47,6 +47,12 @@ load_dotenv(override=True) async def fetch_user_image(params: FunctionCallParams): + """Fetch the user image. + + When called, this function pushes a UserImageRequestFrame upstream to the + transport. As a result, the transport will request the user image and push a + UserImageRawFrame downstream. + """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") @@ -61,7 +67,13 @@ async def fetch_user_image(params: FunctionCallParams): class UserImageProcessor(FrameProcessor): - """Converts incoming user images into context frames.""" + """Converts incoming user images into vision frames. + + This processor handles the UserImageRawFrame from the transport, converts it + to a VisionImageRawFrame and pushes it downstream so it can be handled by a + vision service. + + """ async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index c6320b1ec..c09d39231 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -39,6 +39,14 @@ load_dotenv(override=True) async def fetch_user_image(params: FunctionCallParams): + """Fetch the user image and push it to the LLM. + + When called, this function pushes a UserImageRequestFrame upstream to the + transport. As a result, the transport will request the user image and push a + UserImageRawFrame downstream associated to this request. When the + UserImageRawFrame reaches the LLM assistant aggregator, the image will be + added to the context. + """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") From 74fb6e767601f9c848405091c7dfe1e65acfeb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 29 Oct 2025 18:09:10 -0700 Subject: [PATCH 17/20] scripts(evals): improve eval prompting --- scripts/evals/eval.py | 3 +-- scripts/evals/run-release-evals.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/evals/eval.py b/scripts/evals/eval.py index 550b50809..e23a99420 100644 --- a/scripts/evals/eval.py +++ b/scripts/evals/eval.py @@ -271,10 +271,9 @@ async def run_eval_pipeline( elif isinstance(eval_config.prompt, tuple): example_prompt, example_image = eval_config.prompt - eval_prompt = f"The answer is correct if it matches: {eval}." common_system_prompt = ( "The user might say things other than the answer and that's allowed. " - f"You should only call the eval function with your assessment when the user actually answers the question. {eval_prompt}" + f"You should only call the eval function when the user: {eval_config.eval}" ) if eval_config.eval_speaks_first: system_prompt = f"You are an LLM eval, be extremly brief. You will start the conversation by saying: '{example_prompt}'. {common_system_prompt}" diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 2f6038c14..9ef8d533f 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -26,32 +26,32 @@ FOUNDATIONAL_DIR = SCRIPT_DIR.parent.parent / "examples" / "foundational" EVAL_SIMPLE_MATH = EvalConfig( prompt="A simple math addition.", - eval="Correct math addition.", + eval="The user answers the math addition correctly.", ) EVAL_WEATHER = EvalConfig( prompt="What's the weather in San Francisco?", - eval="Something specific about the current weather in San Francisco, including the degrees.", + eval="The user says something specific about the current weather in San Francisco, including the degrees.", ) EVAL_ONLINE_SEARCH = EvalConfig( prompt="What's the date right now in London?", - eval=f"Today is {datetime.now(timezone.utc).strftime('%B %d, %Y')}.", + eval=f"The user says today is {datetime.now(timezone.utc).strftime('%B %d, %Y')} in London.", ) EVAL_SWITCH_LANGUAGE = EvalConfig( prompt="Say something in Spanish.", - eval="The user is now talking in Spanish.", + eval="The user talks in Spanish.", ) EVAL_VISION_CAMERA = EvalConfig( prompt=("Briefly describe what you see.", Image.open(ASSETS_DIR / "cat.jpg")), - eval="A cat description.", + eval="The user provides a cat description.", ) EVAL_VISION_IMAGE = EvalConfig( prompt="Briefly describe this image.", - eval="A cat description.", + eval="The user provides a cat description.", eval_speaks_first=True, runner_args_body={ "image_path": ASSETS_DIR / "cat.jpg", @@ -60,14 +60,14 @@ EVAL_VISION_IMAGE = EvalConfig( ) EVAL_VOICEMAIL = EvalConfig( - prompt="Please leave a message after the beep.", - eval="Assess the conversation and determine if it is a voicemail.", + prompt="Please leave a message.", + eval="The user leaves a voicemail message.", eval_speaks_first=True, ) EVAL_CONVERSATION = EvalConfig( prompt="Hello, this is Mark.", - eval="A start of a conversation, not a voicemail.", + eval="The user replies with a greeting.", eval_speaks_first=True, ) @@ -172,11 +172,11 @@ TESTS_21 = [ ] TESTS_26 = [ - ("26-gemini-multimodal-live.py", EVAL_SIMPLE_MATH), + ("26-gemini-live.py", EVAL_SIMPLE_MATH), ("26a-gemini-live-transcription.py", EVAL_SIMPLE_MATH), ("26b-gemini-live-function-calling.py", EVAL_WEATHER), ("26c-gemini-live-video.py", EVAL_SIMPLE_MATH), - ("26e-gemini-multimodal-google-search.py", EVAL_ONLINE_SEARCH), + ("26e-gemini-live-google-search.py", EVAL_ONLINE_SEARCH), ("26h-gemini-live-vertex-function-calling.py", EVAL_WEATHER), # Currently not working. # ("26d-gemini-live-text.py", EVAL_SIMPLE_MATH), From ec95618b940ab8d91005b54019de730af3d3095b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 Oct 2025 12:42:47 -0700 Subject: [PATCH 18/20] don't tie UserImageRawFrame with function calls --- CHANGELOG.md | 21 +++++--- .../14d-function-calling-anthropic-video.py | 20 ++++---- .../14d-function-calling-aws-video.py | 20 ++++---- ...14d-function-calling-gemini-flash-video.py | 20 ++++---- .../14d-function-calling-moondream-video.py | 48 ++++--------------- .../14d-function-calling-openai-video.py | 20 ++++---- src/pipecat/frames/frames.py | 45 +++++------------ .../aggregators/llm_response_universal.py | 19 ++------ src/pipecat/services/llm_service.py | 24 +++++----- src/pipecat/services/moondream/vision.py | 11 +++-- src/pipecat/services/vision_service.py | 10 ++-- src/pipecat/transports/daily/transport.py | 3 +- .../transports/smallwebrtc/transport.py | 45 +++++++++-------- 13 files changed, 126 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78620da43..32952afd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added `VisionImageRawFrame`. This is an input image frame with an associated - text. It is usually processed by vision services (e.g. Moondream). The text - guides the vision service on how to analyze the image. - - Added support for including images or audio to LLM context messages using `LLMContext.create_image_message()` or `LLMContext.create_image_url_message()` (not all LLMs support URLs) and `LLMContext.create_audio_message()`. For @@ -168,9 +164,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- Updated `MoondreamService` to process `VisionImageRawFrame`. +- `UserImageRawFrame` new fields `add_to_context` and `text`. The + `add_to_context` field indicates if this image and text should be added to the + LLM context (by the LLM assistant aggregator). The `text` field, if set, might + also guide the LLM or the vision service on how to analyze the image. -- `VisionService` expects `VisionImageRawFrame` in order to analyze images. +- `UserImageRequestFrame` new fiels `add_to_context` and `text`. Both fields + will be used to set the same fields on the captured `UserImageRawFrame`. + +- `UserImageRequestFrame` don't require function call name and ID anymore. + +- Updated `MoondreamService` to process `UserImageRawFrame`. + +- `VisionService` expects `UserImageRawFrame` in order to analyze images. - `DailyTransport` triggers `on_error` event if transcription can't be started or stopped. @@ -196,6 +202,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- `LLMService.request_image_frame()` is deprecated, push a + `UserImageRequestFrame` instead. + - `UserResponseAggregator` is deprecated and will be removed in a future version. - The `send_transcription_frames` argument to `OpenAIRealtimeLLMService` is diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index bf35b9648..db4c75c48 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -15,12 +15,13 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -42,21 +43,18 @@ async def fetch_user_image(params: FunctionCallParams): When called, this function pushes a UserImageRequestFrame upstream to the transport. As a result, the transport will request the user image and push a - UserImageRawFrame downstream associated to this request. When the - UserImageRawFrame reaches the LLM assistant aggregator, the image will be - added to the context. + UserImageRawFrame downstream which will be added to the context by the LLM + assistant aggregator. """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the user image frame. Note that this image is associated to a - # function call and will be handled by the LLM assistant aggregators. - await params.llm.request_image_frame( - user_id=user_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, + # Request a user image frame and indicate that it should be added to the + # context. + await params.llm.push_frame( + UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + FrameDirection.UPSTREAM, ) await params.result_callback(None) diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index 20367bad2..dcd8cf202 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -15,12 +15,13 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -42,21 +43,18 @@ async def fetch_user_image(params: FunctionCallParams): When called, this function pushes a UserImageRequestFrame upstream to the transport. As a result, the transport will request the user image and push a - UserImageRawFrame downstream associated to this request. When the - UserImageRawFrame reaches the LLM assistant aggregator, the image will be - added to the context. + UserImageRawFrame downstream which will be added to the context by the LLM + assistant aggregator. """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the user image frame. Note that this image is associated to a - # function call and will be handled by the LLM assistant aggregators. - await params.llm.request_image_frame( - user_id=user_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, + # Request a user image frame and indicate that it should be added to the + # context. + await params.llm.push_frame( + UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + FrameDirection.UPSTREAM, ) await params.result_callback(None) diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index 867fc3180..7a8eecca1 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -15,12 +15,13 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -42,21 +43,18 @@ async def fetch_user_image(params: FunctionCallParams): When called, this function pushes a UserImageRequestFrame upstream to the transport. As a result, the transport will request the user image and push a - UserImageRawFrame downstream associated to this request. When the - UserImageRawFrame reaches the LLM assistant aggregator, the image will be - added to the context. + UserImageRawFrame downstream which will be added to the context by the LLM + assistant aggregator. """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the user image frame. Note that this image is associated to a - # function call and will be handled by the LLM assistant aggregators. - await params.llm.request_image_frame( - user_id=user_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, + # Request a user image frame and indicate that it should be added to the + # context. + await params.llm.push_frame( + UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + FrameDirection.UPSTREAM, ) await params.result_callback(None) diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index eb06556a9..a14ba14e5 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -15,20 +15,14 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import ( - Frame, - LLMRunFrame, - UserImageRawFrame, - UserImageRequestFrame, - VisionImageRawFrame, -) +from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -57,40 +51,17 @@ async def fetch_user_image(params: FunctionCallParams): question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the user image frame frame. In this case we don't use - # `llm.request_image_frame()` because we don't want the LLM to analyze it. + # Request a user image frame. In this case, we don't want the requested + # image to be added to the context because we will process it with + # Moondream. await params.llm.push_frame( - UserImageRequestFrame(user_id=user_id, context=question), FrameDirection.UPSTREAM + UserImageRequestFrame(user_id=user_id, text=question, add_to_context=False), + FrameDirection.UPSTREAM, ) await params.result_callback(None) -class UserImageProcessor(FrameProcessor): - """Converts incoming user images into vision frames. - - This processor handles the UserImageRawFrame from the transport, converts it - to a VisionImageRawFrame and pushes it downstream so it can be handled by a - vision service. - - """ - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, UserImageRawFrame): - if frame.request and frame.request.context: - frame = VisionImageRawFrame( - image=frame.image, - text=frame.request.context, - size=frame.size, - format=frame.format, - ) - await self.push_frame(frame) - else: - await self.push_frame(frame, direction) - - # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. @@ -152,9 +123,6 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context = LLMContext(messages, tools) context_aggregator = LLMContextAggregatorPair(context) - # This will get the get the user image frame and push it to the LLM. - image_processor = UserImageProcessor() - # If you run into weird description, try with use_cpu=True moondream = MoondreamService() @@ -165,7 +133,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): context_aggregator.user(), # User responses ParallelPipeline( [llm], # LLM - [image_processor, moondream], + [moondream], ), tts, # TTS transport.output(), # Transport bot output diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index c09d39231..2dbdc7e63 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -16,12 +16,13 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame +from pipecat.frames.frames import LLMRunFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.processors.frame_processor import FrameDirection from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, @@ -43,21 +44,18 @@ async def fetch_user_image(params: FunctionCallParams): When called, this function pushes a UserImageRequestFrame upstream to the transport. As a result, the transport will request the user image and push a - UserImageRawFrame downstream associated to this request. When the - UserImageRawFrame reaches the LLM assistant aggregator, the image will be - added to the context. + UserImageRawFrame downstream which will be added to the context by the LLM + assistant aggregator. """ user_id = params.arguments["user_id"] question = params.arguments["question"] logger.debug(f"Requesting image with user_id={user_id}, question={question}") - # Request the user image frame. Note that this image is associated to a - # function call and will be handled by the LLM assistant aggregators. - await params.llm.request_image_frame( - user_id=user_id, - function_name=params.function_name, - tool_call_id=params.tool_call_id, - text_content=question, + # Request a user image frame and indicate that it should be added to the + # context. + await params.llm.push_frame( + UserImageRequestFrame(user_id=user_id, text=question, add_to_context=True), + FrameDirection.UPSTREAM, ) await params.result_callback(None) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index afa647b1c..70975d262 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -11,7 +11,6 @@ including data frames, system frames, and control frames for audio, video, text, and LLM processing. """ -import asyncio from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, @@ -1202,27 +1201,23 @@ class TransportMessageUrgentFrame(OutputTransportMessageUrgentFrame): class UserImageRequestFrame(SystemFrame): """Frame requesting an image from a specific user. - A frame to request an image from the given user. The frame might be - generated by a function call in which case the corresponding fields will be - properly set. + A frame to request an image from the given user. The request might come with + a text that can be later used to describe the requested image. Parameters: user_id: Identifier of the user to request image from. - context: Optional context for the image request. - function_name: Name of function that generated this request (if any). - tool_call_id: Tool call ID if generated by function call. + text: An optional text associated to the image request. + add_to_context: Whether the requested image should be added to an LLM context. video_source: Specific video source to capture from. """ user_id: str - context: Optional[Any] = None - function_name: Optional[str] = None - tool_call_id: Optional[str] = None + text: Optional[str] = None + add_to_context: Optional[bool] = None video_source: Optional[str] = None - request_event: Optional[asyncio.Event] = None def __str__(self): - return f"{self.name}(user: {self.user_id}, video_source: {self.video_source}, function: {self.function_name}, request: {self.tool_call_id})" + return f"{self.name}(user: {self.user_id}, text: {self.text}, add_to_context: {self.add_to_context}, {self.video_source})" @dataclass @@ -1296,33 +1291,17 @@ class UserImageRawFrame(InputImageRawFrame): Parameters: user_id: Identifier of the user who provided this image. - request: The original image request frame if this is a response. + text: An optional text associated to this image. + add_to_context: Whether this image should be added to an LLM context. """ user_id: str = "" - request: Optional[UserImageRequestFrame] = None + text: Optional[str] = None + add_to_context: Optional[bool] = None def __str__(self): pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, request: {self.request})" - - -@dataclass -class VisionImageRawFrame(InputImageRawFrame): - """Raw image input frame to be analyzed by vision services. - - This is just an image with an associated text describing how the vision - service should analyze the image. - - Parameters: - text: Description of how the vision service should analyze the image. - """ - - text: str - - def __str__(self): - pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text})" + return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, add_to_context: {self.add_to_context})" @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response_universal.py b/src/pipecat/processors/aggregators/llm_response_universal.py index 920775853..44c534b9b 100644 --- a/src/pipecat/processors/aggregators/llm_response_universal.py +++ b/src/pipecat/processors/aggregators/llm_response_universal.py @@ -616,7 +616,7 @@ class LLMAssistantAggregator(LLMContextAggregator): await self._handle_function_call_result(frame) elif isinstance(frame, FunctionCallCancelFrame): await self._handle_function_call_cancel(frame) - elif isinstance(frame, UserImageRawFrame) and frame.request and frame.request.tool_call_id: + elif isinstance(frame, UserImageRawFrame): await self._handle_user_image_frame(frame) elif isinstance(frame, BotStoppedSpeakingFrame): await self.push_aggregation() @@ -767,30 +767,21 @@ class LLMAssistantAggregator(LLMContextAggregator): message["content"] = result async def _handle_user_image_frame(self, frame: UserImageRawFrame): - logger.debug( - f"{self} UserImageRawFrame: [{frame.request.function_name}:{frame.request.tool_call_id}]" - ) - - if frame.request.tool_call_id not in self._function_calls_in_progress: - logger.warning( - f"UserImageRawFrame tool_call_id [{frame.request.tool_call_id}] is not running" - ) + if not frame.add_to_context: return + logger.debug(f"{self} Adding UserImageRawFrame to LLM context (size: {frame.size})") + self._context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, - text=frame.request.context, + text=frame.text, ) await self.push_aggregation() await self.push_context_frame(FrameDirection.UPSTREAM) - # Notify who ever requested the image that we have added it to the context. - if frame.request and frame.request.request_event: - frame.request.request_event.set() - async def _handle_llm_start(self, _: LLMFullResponseStartFrame): self._started += 1 diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index c75893ea3..0a1a835f7 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -503,6 +503,9 @@ class LLMService(AIService): the image. If you expect the image to be processed by a vision service, you might want to push a UserImageRequestFrame upstream directly. + .. deprecated:: 0.0.92 + This method is deprecated, push a `UserImageRequestFrame` instead. + Args: user_id: The ID of the user to request an image from. function_name: Optional function name associated with the request. @@ -512,24 +515,19 @@ class LLMService(AIService): timeout: Optional timeout for the requested image to be added to the LLM context. """ - request_event = asyncio.Event() if timeout else None + import warnings + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Method `request_image_frame()` is deprecated, push a `UserImageRequestFrame` instead.", + DeprecationWarning, + ) await self.push_frame( - UserImageRequestFrame( - user_id=user_id, - function_name=function_name, - tool_call_id=tool_call_id, - context=text_content, - video_source=video_source, - request_event=request_event, - ), + UserImageRequestFrame(user_id=user_id, text=text_content), FrameDirection.UPSTREAM, ) - # Wait for the requested image to be added to the context. - if request_event: - await asyncio.wait_for(request_event.wait(), timeout=timeout) - async def _create_sequential_runner_task(self): if not self._sequential_runner_task: self._sequential_runner_queue = asyncio.Queue() diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index 6e470071e..b7527f76c 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -16,7 +16,12 @@ from typing import AsyncGenerator, Optional from loguru import logger from PIL import Image -from pipecat.frames.frames import ErrorFrame, Frame, TextFrame, VisionImageRawFrame +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TextFrame, + UserImageRawFrame, +) from pipecat.services.vision_service import VisionService try: @@ -94,11 +99,11 @@ class MoondreamService(VisionService): logger.debug("Loaded Moondream model") - async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: + async def run_vision(self, frame: UserImageRawFrame) -> AsyncGenerator[Frame, None]: """Analyze an image and generate a description. Args: - frame: The vision image frame to process. + frame: The image frame to process. Yields: Frame: TextFrame containing the generated image description, or ErrorFrame diff --git a/src/pipecat/services/vision_service.py b/src/pipecat/services/vision_service.py index 16b25277c..5de282896 100644 --- a/src/pipecat/services/vision_service.py +++ b/src/pipecat/services/vision_service.py @@ -14,7 +14,7 @@ visual content. from abc import abstractmethod from typing import AsyncGenerator -from pipecat.frames.frames import Frame, VisionImageRawFrame +from pipecat.frames.frames import Frame, UserImageRawFrame from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService @@ -37,7 +37,7 @@ class VisionService(AIService): self._describe_text = None @abstractmethod - async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: + async def run_vision(self, frame: UserImageRawFrame) -> AsyncGenerator[Frame, None]: """Process the given vision image and generate results. This method must be implemented by subclasses to provide actual computer @@ -45,7 +45,7 @@ class VisionService(AIService): visual question answering. Args: - frame: The vision image frame to process. + frame: The image frame to process. Yields: Frame: Frames containing the vision analysis results, typically TextFrame @@ -56,7 +56,7 @@ class VisionService(AIService): async def process_frame(self, frame: Frame, direction: FrameDirection): """Process frames, handling vision image frames for analysis. - Automatically processes VisionImageRawFrame objects by calling run_vision + Automatically processes UserImageRawFrame objects by calling run_vision and handles metrics tracking. Other frames are passed through unchanged. Args: @@ -65,7 +65,7 @@ class VisionService(AIService): """ await super().process_frame(frame, direction) - if isinstance(frame, VisionImageRawFrame): + if isinstance(frame, UserImageRawFrame) and frame.text: await self.start_processing_metrics() await self.process_generator(self.run_vision(frame)) await self.stop_processing_metrics() diff --git a/src/pipecat/transports/daily/transport.py b/src/pipecat/transports/daily/transport.py index 18c36ee56..c6dd06986 100644 --- a/src/pipecat/transports/daily/transport.py +++ b/src/pipecat/transports/daily/transport.py @@ -1839,10 +1839,11 @@ class DailyInputTransport(BaseInputTransport): if render_frame: frame = UserImageRawFrame( user_id=participant_id, - request=request_frame, image=video_frame.buffer, size=(video_frame.width, video_frame.height), format=video_frame.color_format, + text=request_frame.text if request_frame else None, + add_to_context=request_frame.add_to_context if request_frame else None, ) frame.transport_source = video_source await self.push_video_frame(frame) diff --git a/src/pipecat/transports/smallwebrtc/transport.py b/src/pipecat/transports/smallwebrtc/transport.py index cdefa976f..4bd18c9ef 100644 --- a/src/pipecat/transports/smallwebrtc/transport.py +++ b/src/pipecat/transports/smallwebrtc/transport.py @@ -15,7 +15,7 @@ import asyncio import fractions import time from collections import deque -from typing import Any, Awaitable, Callable, Optional +from typing import Any, Awaitable, Callable, List, Optional import numpy as np from loguru import logger @@ -567,7 +567,7 @@ class SmallWebRTCInputTransport(BaseInputTransport): self._receive_audio_task = None self._receive_video_task = None self._receive_screen_video_task = None - self._image_requests = {} + self._image_requests: List[UserImageRequestFrame] = [] # Whether we have seen a StartFrame already. self._initialized = False @@ -657,23 +657,27 @@ class SmallWebRTCInputTransport(BaseInputTransport): if video_frame: await self.push_video_frame(video_frame) - # Check if there are any pending image requests and create UserImageRawFrame - if self._image_requests: - for req_id, request_frame in list(self._image_requests.items()): - if request_frame.video_source == video_source: - # Create UserImageRawFrame using the current video frame - image_frame = UserImageRawFrame( - user_id=request_frame.user_id, - request=request_frame, - image=video_frame.image, - size=video_frame.size, - format=video_frame.format, - ) - image_frame.transport_source = video_source - # Push the frame to the pipeline - await self.push_video_frame(image_frame) - # Remove from pending requests - del self._image_requests[req_id] + # Check if there are any pending image requests and create + # UserImageRawFrame. Use a shallow copy so we can remove + # elements. + for request_frame in self._image_requests[:]: + if request_frame.video_source == video_source: + # Create UserImageRawFrame using the current video frame + image_frame = UserImageRawFrame( + user_id=request_frame.user_id, + image=video_frame.image, + size=video_frame.size, + format=video_frame.format, + text=request_frame.text if request_frame else None, + add_to_context=request_frame.add_to_context + if request_frame + else None, + ) + image_frame.transport_source = video_source + # Push the frame to the pipeline + await self.push_video_frame(image_frame) + # Remove from pending requests + self._image_requests.remove(request_frame) except Exception as e: logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})") @@ -701,8 +705,7 @@ class SmallWebRTCInputTransport(BaseInputTransport): logger.debug(f"Requesting image from participant: {frame.user_id}") # Store the request - request_id = f"{frame.function_name}:{frame.tool_call_id}" - self._image_requests[request_id] = frame + self._image_requests.append(frame) # Default to camera if no source specified if frame.video_source is None: From 19f046a338fdc2babe72dca6f805144a7f758089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 Oct 2025 13:06:12 -0700 Subject: [PATCH 19/20] examples(foundational): add 12d-describe-image-moondream --- CHANGELOG.md | 3 +- .../12d-describe-image-moondream.py | 122 ++++++++++++++++++ scripts/evals/run-release-evals.py | 30 +++-- 3 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 examples/foundational/12d-describe-image-moondream.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 32952afd0..0ec6ad556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -240,8 +240,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other -- Updated all vision 12-series foundational examples to load images from a file - and push `LLMMessagesAppendFrame` with the loaded image. +- Updated all vision 12-series foundational examples to load images from a file. - Added 14-series video examples for different services. These new examples request an image from the user camera through a function call. diff --git a/examples/foundational/12d-describe-image-moondream.py b/examples/foundational/12d-describe-image-moondream.py new file mode 100644 index 000000000..ee6f328f1 --- /dev/null +++ b/examples/foundational/12d-describe-image-moondream.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger +from PIL import Image + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import UserImageRawFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.moondream.vision import MoondreamService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + vision = MoondreamService() + + pipeline = Pipeline( + [ + vision, # Vision + tts, # TTS + transport.output(), # Transport bot output + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + + if not runner_args.body: + script_dir = os.path.dirname(__file__) + runner_args.body = { + "image_path": os.path.join(script_dir, "assets", "cat.jpg"), + "question": "Describe this image", + } + + image_path = runner_args.body["image_path"] + question = runner_args.body["question"] + + # Describe the image. + image = Image.open(image_path) + await task.queue_frames( + [ + UserImageRawFrame( + image=image.tobytes(), + format="RGB", + size=image.size, + text=question, + ) + ] + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 9ef8d533f..ce0a32dd6 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -49,15 +49,18 @@ EVAL_VISION_CAMERA = EvalConfig( eval="The user provides a cat description.", ) -EVAL_VISION_IMAGE = EvalConfig( - prompt="Briefly describe this image.", - eval="The user provides a cat description.", - eval_speaks_first=True, - runner_args_body={ - "image_path": ASSETS_DIR / "cat.jpg", - "question": "Briefly describe this image.", - }, -) + +def EVAL_VISION_IMAGE(*, eval_speaks_first: bool = False): + return EvalConfig( + prompt="Briefly describe this image.", + eval="The user provides a cat description.", + eval_speaks_first=eval_speaks_first, + runner_args_body={ + "image_path": ASSETS_DIR / "cat.jpg", + "question": "Briefly describe this image.", + }, + ) + EVAL_VOICEMAIL = EvalConfig( prompt="Please leave a message.", @@ -117,10 +120,11 @@ TESTS_07 = [ ] TESTS_12 = [ - ("12-describe-image-openai.py", EVAL_VISION_IMAGE), - ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE), - ("12b-describe-image-aws.py", EVAL_VISION_IMAGE), - ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE), + ("12-describe-image-openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12b-describe-image-aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12d-describe-image-moondream.py", EVAL_VISION_IMAGE()), ] TESTS_14 = [ From 42f0490414d662c7af5ec9eb56fbe0a3d5c30ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 30 Oct 2025 13:23:36 -0700 Subject: [PATCH 20/20] examples(foundational): 14-* show how to tell the LLM we are capturing an image --- examples/foundational/14d-function-calling-anthropic-video.py | 4 ++++ examples/foundational/14d-function-calling-aws-video.py | 4 ++++ .../foundational/14d-function-calling-gemini-flash-video.py | 4 ++++ examples/foundational/14d-function-calling-moondream-video.py | 4 ++++ examples/foundational/14d-function-calling-openai-video.py | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/examples/foundational/14d-function-calling-anthropic-video.py b/examples/foundational/14d-function-calling-anthropic-video.py index db4c75c48..b21b9fdba 100644 --- a/examples/foundational/14d-function-calling-anthropic-video.py +++ b/examples/foundational/14d-function-calling-anthropic-video.py @@ -59,6 +59,10 @@ async def fetch_user_image(params: FunctionCallParams): await params.result_callback(None) + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets diff --git a/examples/foundational/14d-function-calling-aws-video.py b/examples/foundational/14d-function-calling-aws-video.py index dcd8cf202..dffee193c 100644 --- a/examples/foundational/14d-function-calling-aws-video.py +++ b/examples/foundational/14d-function-calling-aws-video.py @@ -59,6 +59,10 @@ async def fetch_user_image(params: FunctionCallParams): await params.result_callback(None) + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets diff --git a/examples/foundational/14d-function-calling-gemini-flash-video.py b/examples/foundational/14d-function-calling-gemini-flash-video.py index 7a8eecca1..acb977c68 100644 --- a/examples/foundational/14d-function-calling-gemini-flash-video.py +++ b/examples/foundational/14d-function-calling-gemini-flash-video.py @@ -59,6 +59,10 @@ async def fetch_user_image(params: FunctionCallParams): await params.result_callback(None) + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets diff --git a/examples/foundational/14d-function-calling-moondream-video.py b/examples/foundational/14d-function-calling-moondream-video.py index a14ba14e5..3a7889c00 100644 --- a/examples/foundational/14d-function-calling-moondream-video.py +++ b/examples/foundational/14d-function-calling-moondream-video.py @@ -61,6 +61,10 @@ async def fetch_user_image(params: FunctionCallParams): await params.result_callback(None) + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets diff --git a/examples/foundational/14d-function-calling-openai-video.py b/examples/foundational/14d-function-calling-openai-video.py index 2dbdc7e63..ec6fb008b 100644 --- a/examples/foundational/14d-function-calling-openai-video.py +++ b/examples/foundational/14d-function-calling-openai-video.py @@ -60,6 +60,10 @@ async def fetch_user_image(params: FunctionCallParams): await params.result_callback(None) + # Instead of None, it's possible to also provide a tool call answer to + # tell the LLM that we are grabbing the image to analyze. + # await params.result_callback({"result": "Image is being captured."}) + # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets