diff --git a/CHANGELOG.md b/CHANGELOG.md index 32952afd0..0ec6ad556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -240,8 +240,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other -- Updated all vision 12-series foundational examples to load images from a file - and push `LLMMessagesAppendFrame` with the loaded image. +- Updated all vision 12-series foundational examples to load images from a file. - Added 14-series video examples for different services. These new examples request an image from the user camera through a function call. diff --git a/examples/foundational/12d-describe-image-moondream.py b/examples/foundational/12d-describe-image-moondream.py new file mode 100644 index 000000000..ee6f328f1 --- /dev/null +++ b/examples/foundational/12d-describe-image-moondream.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger +from PIL import Image + +from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams +from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import UserImageRawFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import create_transport +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.moondream.vision import MoondreamService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), + "webrtc": lambda: TransportParams( + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + vision = MoondreamService() + + pipeline = Pipeline( + [ + vision, # Vision + tts, # TTS + transport.output(), # Transport bot output + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + + if not runner_args.body: + script_dir = os.path.dirname(__file__) + runner_args.body = { + "image_path": os.path.join(script_dir, "assets", "cat.jpg"), + "question": "Describe this image", + } + + image_path = runner_args.body["image_path"] + question = runner_args.body["question"] + + # Describe the image. + image = Image.open(image_path) + await task.queue_frames( + [ + UserImageRawFrame( + image=image.tobytes(), + format="RGB", + size=image.size, + text=question, + ) + ] + ) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 9ef8d533f..ce0a32dd6 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -49,15 +49,18 @@ EVAL_VISION_CAMERA = EvalConfig( eval="The user provides a cat description.", ) -EVAL_VISION_IMAGE = EvalConfig( - prompt="Briefly describe this image.", - eval="The user provides a cat description.", - eval_speaks_first=True, - runner_args_body={ - "image_path": ASSETS_DIR / "cat.jpg", - "question": "Briefly describe this image.", - }, -) + +def EVAL_VISION_IMAGE(*, eval_speaks_first: bool = False): + return EvalConfig( + prompt="Briefly describe this image.", + eval="The user provides a cat description.", + eval_speaks_first=eval_speaks_first, + runner_args_body={ + "image_path": ASSETS_DIR / "cat.jpg", + "question": "Briefly describe this image.", + }, + ) + EVAL_VOICEMAIL = EvalConfig( prompt="Please leave a message.", @@ -117,10 +120,11 @@ TESTS_07 = [ ] TESTS_12 = [ - ("12-describe-image-openai.py", EVAL_VISION_IMAGE), - ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE), - ("12b-describe-image-aws.py", EVAL_VISION_IMAGE), - ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE), + ("12-describe-image-openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12b-describe-image-aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)), + ("12d-describe-image-moondream.py", EVAL_VISION_IMAGE()), ] TESTS_14 = [