examples(foundational): add 12d-describe-image-moondream

CHANGELOG.md

@@ -240,8 +240,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Other
 
-- Updated all vision 12-series foundational examples to load images from a file
+- Updated all vision 12-series foundational examples to load images from a file.
-  and push `LLMMessagesAppendFrame` with the loaded image.
 
 - Added 14-series video examples for different services. These new examples
   request an image from the user camera through a function call.

examples/foundational/12d-describe-image-moondream.py

@@ -0,0 +1,122 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+from PIL import Image
+
+from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import UserImageRawFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.runner.types import RunnerArguments
+from pipecat.runner.utils import create_transport
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.moondream.vision import MoondreamService
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.transports.daily.transport import DailyParams
+
+load_dotenv(override=True)
+
+
+# We store functions so objects (e.g. SileroVADAnalyzer) don't get
+# instantiated. The function will be called when the desired transport gets
+# selected.
+transport_params = {
+    "daily": lambda: DailyParams(
+        audio_out_enabled=True,
+        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+        turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+    ),
+    "webrtc": lambda: TransportParams(
+        audio_out_enabled=True,
+        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+        turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+    ),
+}
+
+
+async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
+    logger.info(f"Starting bot")
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    vision = MoondreamService()
+
+    pipeline = Pipeline(
+        [
+            vision,  # Vision
+            tts,  # TTS
+            transport.output(),  # Transport bot output
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
+    )
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected")
+
+        if not runner_args.body:
+            script_dir = os.path.dirname(__file__)
+            runner_args.body = {
+                "image_path": os.path.join(script_dir, "assets", "cat.jpg"),
+                "question": "Describe this image",
+            }
+
+        image_path = runner_args.body["image_path"]
+        question = runner_args.body["question"]
+
+        # Describe the image.
+        image = Image.open(image_path)
+        await task.queue_frames(
+            [
+                UserImageRawFrame(
+                    image=image.tobytes(),
+                    format="RGB",
+                    size=image.size,
+                    text=question,
+                )
+            ]
+        )
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
+
+    await runner.run(task)
+
+
+async def bot(runner_args: RunnerArguments):
+    """Main bot entry point compatible with Pipecat Cloud."""
+    transport = await create_transport(runner_args, transport_params)
+    await run_bot(transport, runner_args)
+
+
+if __name__ == "__main__":
+    from pipecat.runner.run import main
+
+    main()

scripts/evals/run-release-evals.py

@@ -49,15 +49,18 @@ EVAL_VISION_CAMERA = EvalConfig(
     eval="The user provides a cat description.",
 )
 
-EVAL_VISION_IMAGE = EvalConfig(
+
-    prompt="Briefly describe this image.",
+def EVAL_VISION_IMAGE(*, eval_speaks_first: bool = False):
-    eval="The user provides a cat description.",
+    return EvalConfig(
-    eval_speaks_first=True,
+        prompt="Briefly describe this image.",
-    runner_args_body={
+        eval="The user provides a cat description.",
-        "image_path": ASSETS_DIR / "cat.jpg",
+        eval_speaks_first=eval_speaks_first,
-        "question": "Briefly describe this image.",
+        runner_args_body={
-    },
+            "image_path": ASSETS_DIR / "cat.jpg",
-)
+            "question": "Briefly describe this image.",
+        },
+    )
+
 
 EVAL_VOICEMAIL = EvalConfig(
     prompt="Please leave a message.",
@@ -117,10 +120,11 @@ TESTS_07 = [
 ]
 
 TESTS_12 = [
-    ("12-describe-image-openai.py", EVAL_VISION_IMAGE),
+    ("12-describe-image-openai.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
-    ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE),
+    ("12a-describe-image-anthropic.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
-    ("12b-describe-image-aws.py", EVAL_VISION_IMAGE),
+    ("12b-describe-image-aws.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
-    ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE),
+    ("12c-describe-image-gemini-flash.py", EVAL_VISION_IMAGE(eval_speaks_first=True)),
+    ("12d-describe-image-moondream.py", EVAL_VISION_IMAGE()),
 ]
 
 TESTS_14 = [