diff --git a/examples/foundational/03b-still-frame-imagen.py b/examples/foundational/03b-still-frame-imagen.py new file mode 100644 index 000000000..87e73950e --- /dev/null +++ b/examples/foundational/03b-still-frame-imagen.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.frames.frames import EndFrame, TextFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.google import GoogleImageGenService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, _) = await configure(session) + + transport = DailyTransport( + room_url, + None, + "Show a still frame image", + DailyParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024), + ) + + imagegen = GoogleImageGenService( + api_key=os.getenv("GOOGLE_API_KEY"), + ) + + runner = PipelineRunner() + + task = PipelineTask( + Pipeline([imagegen, transport.output()]), PipelineParams(enable_metrics=True) + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await task.queue_frame(TextFrame("a cat in the style of picasso")) + await task.queue_frame(TextFrame("a dog in the style of picasso")) + await task.queue_frame(TextFrame("a fish in the style of picasso")) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.queue_frame(EndFrame()) + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/storytelling-chatbot/README.md b/examples/storytelling-chatbot/README.md index 355608b20..8230deba8 100644 --- a/examples/storytelling-chatbot/README.md +++ b/examples/storytelling-chatbot/README.md @@ -7,7 +7,7 @@ This example shows how to build a voice-driven interactive storytelling experience. It periodically prompts the user for input for a 'choose your own adventure' style experience. -We add visual elements to the story by generating images at lightning speed using Fal. +We use Gemini 2.0 for creating the story and image prompts, and we add visual elements to the story by generating images using Google's Imagen. --- @@ -18,7 +18,7 @@ We add visual elements to the story by generating images at lightning speed usin Transcribes inbound participant voice media to text. -**OpenAI (GPT4) - LLM** +**Google Gemini 2.0 - LLM** Our creative writer LLM. You can see the context used to prompt it [here](src/prompts.py) @@ -26,9 +26,9 @@ Our creative writer LLM. You can see the context used to prompt it [here](src/pr Converts and streams the LLM response from text to audio -**Fal.ai - Image Generation** +**Google Imagen - Image Generation** -Adds pictures to our story (really fast!) Prompting is quite key for style consistency, so we task the LLM to turn each story page into a short image prompt. +Adds pictures to our story. Prompting is quite key for style consistency, so we task the LLM to turn each story page into a short image prompt. --- diff --git a/examples/storytelling-chatbot/env.example b/examples/storytelling-chatbot/env.example index cdff19149..3b6e66316 100644 --- a/examples/storytelling-chatbot/env.example +++ b/examples/storytelling-chatbot/env.example @@ -2,8 +2,6 @@ DAILY_API_KEY= DAILY_SAMPLE_ROOM_URL= ELEVENLABS_API_KEY= ELEVENLABS_VOICE_ID= -FAL_KEY= -OPENAI_API_KEY= GOOGLE_API_KEY= ENV= # dev | production diff --git a/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css b/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css index 7158ca10d..975c8a5bb 100644 --- a/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css +++ b/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css @@ -6,8 +6,8 @@ .videoTile{ @apply bg-gray-950; - width: 560px; - height: 560px; + width: 760px; + height: 760px; mask-image: url('/alpha-mask.gif'); mask-size: cover; mask-repeat: no-repeat; diff --git a/examples/storytelling-chatbot/requirements.txt b/examples/storytelling-chatbot/requirements.txt index 970d3898c..c8cd82646 100644 --- a/examples/storytelling-chatbot/requirements.txt +++ b/examples/storytelling-chatbot/requirements.txt @@ -2,4 +2,5 @@ async_timeout fastapi uvicorn python-dotenv -pipecat-ai[daily,openai,fal,google,cartesia] +-e "../..[daily,silero,openai,fal,cartesia,google]" +-e "../../../python-genai" diff --git a/examples/storytelling-chatbot/src/assets/book1.png b/examples/storytelling-chatbot/src/assets/book1.png index 4148dbaa7..9c1b11329 100644 Binary files a/examples/storytelling-chatbot/src/assets/book1.png and b/examples/storytelling-chatbot/src/assets/book1.png differ diff --git a/examples/storytelling-chatbot/src/assets/book2.png b/examples/storytelling-chatbot/src/assets/book2.png index c7dee447b..173947446 100644 Binary files a/examples/storytelling-chatbot/src/assets/book2.png and b/examples/storytelling-chatbot/src/assets/book2.png differ diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/src/bot.py index 8c896dc86..05f8974bb 100644 --- a/examples/storytelling-chatbot/src/bot.py +++ b/examples/storytelling-chatbot/src/bot.py @@ -51,8 +51,8 @@ async def main(room_url, token=None): DailyParams( audio_out_enabled=True, camera_out_enabled=True, - camera_out_width=768, - camera_out_height=768, + camera_out_width=1024, + camera_out_height=1024, transcription_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_enabled=True, @@ -69,16 +69,7 @@ async def main(room_url, token=None): api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID") ) - fal_service_params = FalImageGenService.InputParams( - image_size={"width": 768, "height": 768} - ) - - fal_service = FalImageGenService( - aiohttp_session=session, - model="fal-ai/stable-diffusion-v35-medium", - params=fal_service_params, - key=os.getenv("FAL_KEY"), - ) + image_gen = GoogleImageGenService(api_key=os.getenv("GOOGLE_API_KEY")) # --------------- Setup ----------------- # @@ -92,14 +83,13 @@ async def main(room_url, token=None): # -------------- Processors ------------- # story_processor = StoryProcessor(message_history, story_pages) - image_processor = StoryImageProcessor(fal_service) + image_processor = StoryImageProcessor(image_gen) # -------------- Story Loop ------------- # runner = PipelineRunner() logger.debug("Waiting for participant...") - main_pipeline = Pipeline( [ transport.input(), @@ -119,7 +109,6 @@ async def main(room_url, token=None): allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, - report_only_initial_ttfb=True, ), ) diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/src/bot_runner.py index 2d949c768..591b2d598 100644 --- a/examples/storytelling-chatbot/src/bot_runner.py +++ b/examples/storytelling-chatbot/src/bot_runner.py @@ -217,7 +217,6 @@ if __name__ == "__main__": required_env_vars = [ "GOOGLE_API_KEY", "DAILY_API_KEY", - "FAL_KEY", "ELEVENLABS_VOICE_ID", "ELEVENLABS_API_KEY", ] diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py index 6d30b543c..62a8fcee4 100644 --- a/examples/storytelling-chatbot/src/processors.py +++ b/examples/storytelling-chatbot/src/processors.py @@ -1,16 +1,27 @@ +import os import re +import google.ai.generativelanguage as glm from async_timeout import timeout -from prompts import CUE_ASSISTANT_TURN, CUE_USER_TURN, IMAGE_GEN_PROMPT +from loguru import logger +from prompts import ( + CUE_ASSISTANT_TURN, + CUE_USER_TURN, + FIRST_IMAGE_PROMPT, + IMAGE_GEN_PROMPT, + NEXT_IMAGE_PROMPT, +) from utils.helpers import load_sounds from pipecat.frames.frames import ( Frame, LLMFullResponseEndFrame, + LLMMessagesFrame, TextFrame, UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.google import GoogleImageGenService, GoogleLLMContext, GoogleLLMService from pipecat.transports.services.daily import DailyTransportMessageFrame sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"]) @@ -44,24 +55,56 @@ class StoryImageProcessor(FrameProcessor): The processed frames are then yielded back. Attributes: - _fal_service (FALService): The FAL service, generates the images (fast fast!). + _image_gen_service: The FAL service, generates the images (fast fast!). """ - def __init__(self, fal_service): + def __init__(self, image_gen_service): super().__init__() - self._fal_service = fal_service + self._image_gen_service = image_gen_service + # Create a new LLM service to use a different system prompt, etc + self._llm_service = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + + self.pages = [] + self.image_descriptions = [] + + def can_generate_metrics(self) -> bool: + return True async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, StoryImageFrame): + if isinstance(frame, StoryPageFrame): + # Special syntax for the first page + if self.pages == []: + prompt = FIRST_IMAGE_PROMPT % frame.text + else: + prompt = NEXT_IMAGE_PROMPT % ( + " ".join(self.pages), + "; ".join(self.image_descriptions), + frame.text, + ) + + await self.start_ttfb_metrics() + # TODO: This is coupled to google implementation now + txt = glm.Content(role="user", parts=[glm.Part(text=prompt)]) + llm_response = await self._llm_service._client.generate_content_async( + contents=[txt], stream=False + ) + image_description = llm_response.text + self.pages.append(frame.text) + self.image_descriptions.append(image_description) try: - async with timeout(7): - async for i in self._fal_service.run_image_gen(IMAGE_GEN_PROMPT % frame.text): + async with timeout(15): + async for i in self._image_gen_service.run_image_gen( + IMAGE_GEN_PROMPT % image_description + ): await self.push_frame(i) except TimeoutError: + logger.debug("Image gen timeout") pass - pass + await self.stop_ttfb_metrics() + # Push the StoryPageFrame so it gets TTS + await self.push_frame(frame) else: await self.push_frame(frame) @@ -96,7 +139,8 @@ class StoryProcessor(FrameProcessor): elif isinstance(frame, TextFrame): # Add new text to the buffer - self._text += frame.text + # (character replace hack to fix TTS sequencing) + self._text += frame.text.replace(";", "—") # Process any complete patterns in the order they appear await self.process_text_content() diff --git a/examples/storytelling-chatbot/src/prompts.py b/examples/storytelling-chatbot/src/prompts.py index 186828932..068c60e55 100644 --- a/examples/storytelling-chatbot/src/prompts.py +++ b/examples/storytelling-chatbot/src/prompts.py @@ -4,35 +4,65 @@ LLM_BASE_PROMPT = { Your goal is to craft an engaging and fun story. Keep all responses short and no more than a few sentences. Start by asking the user what kind of story they'd like to hear. Don't provide any examples. - After they've answered the question, start telling the story. Include [break] after each sentence of the story. + After they've answered the question, start telling the story. Include three story sentences in your response. Add [break] after each sentence of the story. - Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. + EXAMPLE OUTPUT FORMAT: + story sentence 1 [break] + story sentence 2 [break] + story sentence 3 [break] + How would you like the story to continue? + END OF EXAMPLE OUTPUT + + Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want. + Please ensure your responses are less than 5 sentences long. + Please refrain from using any explicit language or content. Do not tell scary stories. + Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include [break] between story sentences. DO NOT RESPOND without story sentences and break tags.""", +} + + +IMAGE_GEN_PROMPT = "an illustration of %s. colorful, whimsical, painterly, concept art." + +CUE_USER_TURN = {"cue": "user_turn"} +CUE_ASSISTANT_TURN = {"cue": "assistant_turn"} + + +""" Start each sentence with an image prompt, wrapped in triangle braces, that I can use to generate an illustration representing the upcoming scene. Image prompts should always be wrapped in triangle braces, like this: . You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. For any recurring characters, you should provide a description of them in the image prompt each time, for example: . Please do not include any character names in the image prompts, just their descriptions. Image prompts should focus on key visual attributes of all characters each time, for example . Please use the following structure for your image prompts: characters, setting, action, and mood. - Image prompts should be less than 150-200 characters and start in lowercase. + Image prompts should be less than 150-200 characters and start in lowercase.""" - STORY SENTENCE OUTPUT FORMAT: - - story sentence 1 [break] - - story sentence 2 [break] - - story sentence 3 [break] - How would you like the story to continue? - END OF EXAMPLE OUTPUT +FIRST_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book. - Generate three story sentences, then ask what should happen next and wait for my input. You can propose an idea for how the story should proceed, but make sure to tell me I can suggest whatever I want. \ - Please ensure your responses are less than 5 sentences long. \ - Please refrain from using any explicit language or content. Do not tell scary stories. - Once you've started telling the story, EVERY RESPONSE should follow the story sentence output format. It is VERY IMPORTANT that you continue to include and [break] between story sentences. DO NOT RESPOND without image descriptions and break tags.""", -} +You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. +For any recurring characters, you should provide a description of them in the image prompt each time, for example: . +Please do not include any character names in the image prompts, just their descriptions. +Image prompts should focus on key visual attributes of all characters each time, for example . +Please use the following structure for your image prompts: characters, setting, action, and mood. +Image prompts should be less than 150-200 characters and start in lowercase. -IMAGE_GEN_PROMPT = "illustrative art of %s. In the style of Studio Ghibli. colorful, whimsical, painterly, concept art." +Here's the first page of the story: +%s +""" -CUE_USER_TURN = {"cue": "user_turn"} -CUE_ASSISTANT_TURN = {"cue": "assistant_turn"} +NEXT_IMAGE_PROMPT = """You are creating a prompt to generate an image for a child's story book. + +Here is the text of the story so far: +%s + +Here are the previous image prompts: +%s + +You should provide as much descriptive detail in your image prompt as you can to help recreate the current scene depicted by the sentence. +For any recurring characters, you should try to use the same description of them in the image prompt each time. +Please do not include any character names in the image prompts, just their descriptions. +Image prompts should focus on key visual attributes of all characters each time, for example . +Please use the following structure for your image prompts: characters, setting, action, and mood. +Image prompts should be less than 150-200 characters and start in lowercase. +Here's the next page of the story: +%s +""" diff --git a/pyproject.toml b/pyproject.toml index 0d5e605d4..801948c8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.6" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] -google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.24.0" ] +google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.24.0", "google-genai~=0.7.0" ] grok = [ "openai~=1.59.6" ] groq = [ "openai~=1.59.6" ] gstreamer = [ "pygobject~=3.50.0" ] diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index a0ff8c337..d705bec47 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, + URLImageRawFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -37,7 +38,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import LLMService, TTSService +from pipecat.services.ai_services import ImageGenService, LLMService, TTSService from pipecat.services.google.frames import LLMSearchResponseFrame from pipecat.services.openai import ( OpenAIAssistantContextAggregator, @@ -49,7 +50,9 @@ from pipecat.utils.time import time_now_iso8601 try: import google.ai.generativelanguage as glm import google.generativeai as gai + from google import genai from google.cloud import texttospeech_v1 + from google.genai import types from google.generativeai.types import GenerationConfig from google.oauth2 import service_account except ModuleNotFoundError as e: @@ -711,7 +714,6 @@ class GoogleLLMService(LLMService): tool_config = None if self._tool_config: tool_config = self._tool_config - response = await self._client.generate_content_async( contents=messages, tools=tools, @@ -738,7 +740,7 @@ class GoogleLLMService(LLMService): search_result += c.text await self.push_frame(LLMTextFrame(c.text)) elif c.function_call: - logger.debug(f"!!! Function call: {c.function_call}") + logger.debug(f"Function call: {c.function_call}") args = type(c.function_call).to_dict(c.function_call).get("args", {}) await self.call_function( context=context, @@ -1029,3 +1031,70 @@ class GoogleTTSService(TTSService): yield ErrorFrame(error=error_message) finally: yield TTSStoppedFrame() + + +class GoogleImageGenService(ImageGenService): + class InputParams(BaseModel): + number_of_images: int = Field(default=1, ge=1, le=8) + model: str = Field(default="imagen-3.0-generate-002") + negative_prompt: str = Field(default=None) + + def __init__( + self, + *, + params: InputParams = InputParams(), + api_key: str, + **kwargs, + ): + super().__init__(**kwargs) + self.set_model_name(params.model) + self._params = params + self._client = genai.Client(api_key=api_key) + + def can_generate_metrics(self) -> bool: + return True + + async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate images from a text prompt using Google's Imagen model. + + Args: + prompt (str): The text description to generate images from. + + Yields: + Frame: Generated image frames or error frames. + """ + logger.debug(f"Generating image from prompt: {prompt}") + await self.start_ttfb_metrics() + + try: + response = await self._client.aio.models.generate_images( + model=self._params.model, + prompt=prompt, + config=types.GenerateImagesConfig( + number_of_images=self._params.number_of_images, + negative_prompt=self._params.negative_prompt, + ), + ) + await self.stop_ttfb_metrics() + + if not response or not response.generated_images: + logger.error(f"{self} error: image generation failed") + yield ErrorFrame("Image generation failed") + return + + for img_response in response.generated_images: + # Google returns the image data directly + image_bytes = img_response.image.image_bytes + image = Image.open(io.BytesIO(image_bytes)) + + frame = URLImageRawFrame( + url=None, # Google doesn't provide URLs, only image data + image=image.tobytes(), + size=image.size, + format=image.format, + ) + yield frame + + except Exception as e: + logger.error(f"{self} error generating image: {e}") + yield ErrorFrame(f"Image generation error: {str(e)}")