More Storybot updates (#1116)

* initial changes for gemini storybot

* storybot updates for gemini

* more storybot updates

* interim interruptible commit

* cleanup

* cleanup

* cleanup

* first draft

* wip

* more storybot fixes

* more storybot updates WIP

* committing before changing the image prompting strategy

* wip

* prompt updating

* cleanup

* cleanup

* cleanup

* readme cleanup

* fixup

examples/foundational/03b-still-frame-imagen.py

@@ -0,0 +1,64 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import os
+import sys
+
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from runner import configure
+
+from pipecat.frames.frames import EndFrame, TextFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.services.google import GoogleImageGenService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+logger.remove(0)
+logger.add(sys.stderr, level="DEBUG")
+
+
+async def main():
+    async with aiohttp.ClientSession() as session:
+        (room_url, _) = await configure(session)
+
+        transport = DailyTransport(
+            room_url,
+            None,
+            "Show a still frame image",
+            DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024),
+        )
+
+        imagegen = GoogleImageGenService(
+            api_key=os.getenv("GOOGLE_API_KEY"),
+        )
+
+        runner = PipelineRunner()
+
+        task = PipelineTask(
+            Pipeline([imagegen, transport.output()]), PipelineParams(enable_metrics=True)
+        )
+
+        @transport.event_handler("on_first_participant_joined")
+        async def on_first_participant_joined(transport, participant):
+            await task.queue_frame(TextFrame("a cat in the style of picasso"))
+            await task.queue_frame(TextFrame("a dog in the style of picasso"))
+            await task.queue_frame(TextFrame("a fish in the style of picasso"))
+
+        @transport.event_handler("on_participant_left")
+        async def on_participant_left(transport, participant, reason):
+            await task.queue_frame(EndFrame())
+
+        await runner.run(task)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/storytelling-chatbot/README.md

@@ -7,7 +7,7 @@
 This example shows how to build a voice-driven interactive storytelling experience.
 It periodically prompts the user for input for a 'choose your own adventure' style experience.
 
-We add visual elements to the story by generating images at lightning speed using Fal.
+We use Gemini 2.0 for creating the story and image prompts, and we add visual elements to the story by generating images using Google's Imagen.
 
 
 ---
@@ -18,7 +18,7 @@ We add visual elements to the story by generating images at lightning speed usin
 
 Transcribes inbound participant voice media to text.
 
-**OpenAI (GPT4) - LLM**
+**Google Gemini 2.0 - LLM**
 
 Our creative writer LLM. You can see the context used to prompt it [here](src/prompts.py)
 
@@ -26,9 +26,9 @@ Our creative writer LLM. You can see the context used to prompt it [here](src/pr
 
 Converts and streams the LLM response from text to audio
 
-**Fal.ai - Image Generation**
+**Google Imagen - Image Generation**
 
-Adds pictures to our story (really fast!) Prompting is quite key for style consistency, so we task the LLM to turn each story page into a short image prompt.
+Adds pictures to our story. Prompting is quite key for style consistency, so we task the LLM to turn each story page into a short image prompt.
 
 ---
 

examples/storytelling-chatbot/env.example

@@ -2,8 +2,6 @@ DAILY_API_KEY=
 DAILY_SAMPLE_ROOM_URL=
 ELEVENLABS_API_KEY=
 ELEVENLABS_VOICE_ID=
-FAL_KEY=
-OPENAI_API_KEY=
 GOOGLE_API_KEY=
 
 ENV= # dev | production

examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css

@@ -6,8 +6,8 @@
 
 .videoTile{
     @apply bg-gray-950;
-    width: 560px;
-    height: 560px;
+    width: 760px;
+    height: 760px;
     mask-image: url('/alpha-mask.gif');
     mask-size: cover;
     mask-repeat: no-repeat;

examples/storytelling-chatbot/requirements.txt

@@ -2,4 +2,5 @@ async_timeout
 fastapi
 uvicorn
 python-dotenv
-pipecat-ai[daily,openai,fal,google,cartesia]
+-e "../..[daily,silero,openai,fal,cartesia,google]"
+-e "../../../python-genai"

examples/storytelling-chatbot/src/bot.py

@@ -51,8 +51,8 @@ async def main(room_url, token=None):
             DailyParams(
                 audio_out_enabled=True,
                 camera_out_enabled=True,
-                camera_out_width=768,
-                camera_out_height=768,
+                camera_out_width=1024,
+                camera_out_height=1024,
                 transcription_enabled=True,
                 vad_analyzer=SileroVADAnalyzer(),
                 vad_enabled=True,
@@ -69,16 +69,7 @@ async def main(room_url, token=None):
             api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")
         )
 
-        fal_service_params = FalImageGenService.InputParams(
-            image_size={"width": 768, "height": 768}
-        )
-
-        fal_service = FalImageGenService(
-            aiohttp_session=session,
-            model="fal-ai/stable-diffusion-v35-medium",
-            params=fal_service_params,
-            key=os.getenv("FAL_KEY"),
-        )
+        image_gen = GoogleImageGenService(api_key=os.getenv("GOOGLE_API_KEY"))
 
         # --------------- Setup ----------------- #
 
@@ -92,14 +83,13 @@ async def main(room_url, token=None):
         # -------------- Processors ------------- #
 
         story_processor = StoryProcessor(message_history, story_pages)
-        image_processor = StoryImageProcessor(fal_service)
+        image_processor = StoryImageProcessor(image_gen)
 
         # -------------- Story Loop ------------- #
 
         runner = PipelineRunner()
 
         logger.debug("Waiting for participant...")
-
         main_pipeline = Pipeline(
             [
                 transport.input(),
@@ -119,7 +109,6 @@ async def main(room_url, token=None):
                 allow_interruptions=True,
                 enable_metrics=True,
                 enable_usage_metrics=True,
-                report_only_initial_ttfb=True,
             ),
         )
 

examples/storytelling-chatbot/src/bot_runner.py

@@ -217,7 +217,6 @@ if __name__ == "__main__":
     required_env_vars = [
         "GOOGLE_API_KEY",
         "DAILY_API_KEY",
-        "FAL_KEY",
         "ELEVENLABS_VOICE_ID",
         "ELEVENLABS_API_KEY",
     ]

examples/storytelling-chatbot/src/processors.py

@@ -1,16 +1,27 @@
+import os
 import re
 
+import google.ai.generativelanguage as glm
 from async_timeout import timeout
-from prompts import CUE_ASSISTANT_TURN, CUE_USER_TURN, IMAGE_GEN_PROMPT
+from loguru import logger
+from prompts import (
+    CUE_ASSISTANT_TURN,
+    CUE_USER_TURN,
+    FIRST_IMAGE_PROMPT,
+    IMAGE_GEN_PROMPT,
+    NEXT_IMAGE_PROMPT,
+)
 from utils.helpers import load_sounds
 
 from pipecat.frames.frames import (
     Frame,
     LLMFullResponseEndFrame,
+    LLMMessagesFrame,
     TextFrame,
     UserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.google import GoogleImageGenService, GoogleLLMContext, GoogleLLMService
 from pipecat.transports.services.daily import DailyTransportMessageFrame
 
 sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"])
@@ -44,24 +55,56 @@ class StoryImageProcessor(FrameProcessor):
     The processed frames are then yielded back.
 
     Attributes:
-        _fal_service (FALService): The FAL service, generates the images (fast fast!).
+        _image_gen_service: The FAL service, generates the images (fast fast!).
     """
 
-    def __init__(self, fal_service):
+    def __init__(self, image_gen_service):
         super().__init__()
-        self._fal_service = fal_service
+        self._image_gen_service = image_gen_service
+        # Create a new LLM service to use a different system prompt, etc
+        self._llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
+
+        self.pages = []
+        self.image_descriptions = []
+
+    def can_generate_metrics(self) -> bool:
+        return True
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
 
-        if isinstance(frame, StoryImageFrame):
+        if isinstance(frame, StoryPageFrame):
+            # Special syntax for the first page
+            if self.pages == []:
+                prompt = FIRST_IMAGE_PROMPT % frame.text
+            else:
+                prompt = NEXT_IMAGE_PROMPT % (
+                    " ".join(self.pages),
+                    "; ".join(self.image_descriptions),
+                    frame.text,
+                )
+
+            await self.start_ttfb_metrics()
+            # TODO: This is coupled to google implementation now
+            txt = glm.Content(role="user", parts=[glm.Part(text=prompt)])
+            llm_response = await self._llm_service._client.generate_content_async(
+                contents=[txt], stream=False
+            )
+            image_description = llm_response.text
+            self.pages.append(frame.text)
+            self.image_descriptions.append(image_description)
             try:
-                async with timeout(7):
-                    async for i in self._fal_service.run_image_gen(IMAGE_GEN_PROMPT % frame.text):
+                async with timeout(15):
+                    async for i in self._image_gen_service.run_image_gen(
+                        IMAGE_GEN_PROMPT % image_description
+                    ):
                         await self.push_frame(i)
             except TimeoutError:
+                logger.debug("Image gen timeout")
                 pass
-            pass
+            await self.stop_ttfb_metrics()
+            # Push the StoryPageFrame so it gets TTS
+            await self.push_frame(frame)
         else:
             await self.push_frame(frame)
 
@@ -96,7 +139,8 @@ class StoryProcessor(FrameProcessor):
 
         elif isinstance(frame, TextFrame):
             # Add new text to the buffer
-            self._text += frame.text
+            # (character replace hack to fix TTS sequencing)
+            self._text += frame.text.replace(";", "—")
             # Process any complete patterns in the order they appear
             await self.process_text_content()
 

examples/storytelling-chatbot/src/prompts.py

@@ -4,35 +4,65 @@ LLM_BASE_PROMPT = {
         Your goal is to craft an engaging and fun story.
         Keep all responses short and no more than a few sentences.
         Start by asking the user what kind of story they'd like to hear. Don't provide any examples.
-        After they've answered the question, start telling the story. Include [break] after each sentence of the story.
+        After they've answered the question, start telling the story. Include three story sentences in your response. Add [break] after each sentence of the story.
         
-        Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene.
+        EXAMPLE OUTPUT FORMAT:
+        story sentence 1 [break]
+        story sentence 2 [break]
+        story sentence 3 [break]
+        How would you like the story to continue?
+        END OF EXAMPLE OUTPUT
+
+        Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want.
+        Please ensure your responses are less than 5 sentences long.
+        Please refrain from using any explicit language or content. Do not tell scary stories.
+        Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include [break] between story sentences. DO NOT RESPOND without story sentences and break tags.""",
+}
+
+
+IMAGE_GEN_PROMPT = "an illustration of %s. colorful, whimsical, painterly, concept art."
+
+CUE_USER_TURN = {"cue": "user_turn"}
+CUE_ASSISTANT_TURN = {"cue": "assistant_turn"}
+
+
+"""        Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene.
         Image prompts should always be wrapped in triangle braces, like this: <image prompt goes here>.
         You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
         For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>.
         Please do not include any character names in the image prompts, just their descriptions.
         Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
         Please use the following structure for your image prompts: characters, setting, action, and mood.
-        Image prompts should be less than 150-200 characters and start in lowercase.
+        Image prompts should be less than 150-200 characters and start in lowercase."""
 
-        STORY SENTENCE OUTPUT FORMAT:
-        <image description 1>
-        story sentence 1 [break]
-        <image description 2>
-        story sentence 2 [break]
-        <image description 3>
-        story sentence 3 [break]
-        How would you like the story to continue?
-        END OF EXAMPLE OUTPUT
+FIRST_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book.
 
-        Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want. \
-        Please ensure your responses are less than 5 sentences long. \
-        Please refrain from using any explicit language or content. Do not tell scary stories.
-        Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include <image descriptions> and [break] between story sentences. DO NOT RESPOND without image descriptions and break tags.""",
-}
+You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+For any recurring characters, you should provide a description of them in the image prompt each time, for example: <a brown fluffy dog ...>.
+Please do not include any character names in the image prompts, just their descriptions.
+Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
+Please use the following structure for your image prompts: characters, setting, action, and mood.
+Image prompts should be less than 150-200 characters and start in lowercase.
 
 
-IMAGE_GEN_PROMPT = "illustrative art of %s. In the style of Studio Ghibli. colorful, whimsical, painterly, concept art."
+Here's the first page of the story:
+%s
+"""
 
-CUE_USER_TURN = {"cue": "user_turn"}
-CUE_ASSISTANT_TURN = {"cue": "assistant_turn"}
+NEXT_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book.
+
+Here is the text of the story so far:
+%s
+
+Here are the previous image prompts:
+%s
+
+You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence.
+For any recurring characters, you should try to use the same description of them in the image prompt each time.
+Please do not include any character names in the image prompts, just their descriptions.
+Image prompts should focus on key visual attributes of all characters each time, for example <a brown fluffy dog and the tiny red cat ...>.
+Please use the following structure for your image prompts: characters, setting, action, and mood.
+Image prompts should be less than 150-200 characters and start in lowercase.
+Here's the next page of the story:
+%s
+"""

pyproject.toml

@@ -54,7 +54,7 @@ elevenlabs = [ "websockets~=13.1" ]
 fal = [ "fal-client~=0.5.6" ]
 fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ]
 gladia = [ "websockets~=13.1" ]
-google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.24.0" ]
+google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.24.0", "google-genai~=0.7.0" ]
 grok = [ "openai~=1.59.6" ]
 groq = [ "openai~=1.59.6" ]
 gstreamer = [ "pygobject~=3.50.0" ]

src/pipecat/services/google/google.py

@@ -29,6 +29,7 @@ from pipecat.frames.frames import (
     TTSAudioRawFrame,
     TTSStartedFrame,
     TTSStoppedFrame,
+    URLImageRawFrame,
     VisionImageRawFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
@@ -37,7 +38,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContextFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.ai_services import LLMService, TTSService
+from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
 from pipecat.services.google.frames import LLMSearchResponseFrame
 from pipecat.services.openai import (
     OpenAIAssistantContextAggregator,
@@ -49,7 +50,9 @@ from pipecat.utils.time import time_now_iso8601
 try:
     import google.ai.generativelanguage as glm
     import google.generativeai as gai
+    from google import genai
     from google.cloud import texttospeech_v1
+    from google.genai import types
     from google.generativeai.types import GenerationConfig
     from google.oauth2 import service_account
 except ModuleNotFoundError as e:
@@ -711,7 +714,6 @@ class GoogleLLMService(LLMService):
             tool_config = None
             if self._tool_config:
                 tool_config = self._tool_config
-
             response = await self._client.generate_content_async(
                 contents=messages,
                 tools=tools,
@@ -738,7 +740,7 @@ class GoogleLLMService(LLMService):
                             search_result += c.text
                             await self.push_frame(LLMTextFrame(c.text))
                         elif c.function_call:
-                            logger.debug(f"!!! Function call: {c.function_call}")
+                            logger.debug(f"Function call: {c.function_call}")
                             args = type(c.function_call).to_dict(c.function_call).get("args", {})
                             await self.call_function(
                                 context=context,
@@ -1029,3 +1031,70 @@ class GoogleTTSService(TTSService):
             yield ErrorFrame(error=error_message)
         finally:
             yield TTSStoppedFrame()
+
+
+class GoogleImageGenService(ImageGenService):
+    class InputParams(BaseModel):
+        number_of_images: int = Field(default=1, ge=1, le=8)
+        model: str = Field(default="imagen-3.0-generate-002")
+        negative_prompt: str = Field(default=None)
+
+    def __init__(
+        self,
+        *,
+        params: InputParams = InputParams(),
+        api_key: str,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.set_model_name(params.model)
+        self._params = params
+        self._client = genai.Client(api_key=api_key)
+
+    def can_generate_metrics(self) -> bool:
+        return True
+
+    async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
+        """Generate images from a text prompt using Google's Imagen model.
+
+        Args:
+            prompt (str): The text description to generate images from.
+
+        Yields:
+            Frame: Generated image frames or error frames.
+        """
+        logger.debug(f"Generating image from prompt: {prompt}")
+        await self.start_ttfb_metrics()
+
+        try:
+            response = await self._client.aio.models.generate_images(
+                model=self._params.model,
+                prompt=prompt,
+                config=types.GenerateImagesConfig(
+                    number_of_images=self._params.number_of_images,
+                    negative_prompt=self._params.negative_prompt,
+                ),
+            )
+            await self.stop_ttfb_metrics()
+
+            if not response or not response.generated_images:
+                logger.error(f"{self} error: image generation failed")
+                yield ErrorFrame("Image generation failed")
+                return
+
+            for img_response in response.generated_images:
+                # Google returns the image data directly
+                image_bytes = img_response.image.image_bytes
+                image = Image.open(io.BytesIO(image_bytes))
+
+                frame = URLImageRawFrame(
+                    url=None,  # Google doesn't provide URLs, only image data
+                    image=image.tobytes(),
+                    size=image.size,
+                    format=image.format,
+                )
+                yield frame
+
+        except Exception as e:
+            logger.error(f"{self} error generating image: {e}")
+            yield ErrorFrame(f"Image generation error: {str(e)}")