pipecat/examples/foundational/10-wake-word.py

#
# Copyright (c) 2024, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#

import asyncio
import aiohttp
import os
import random
import sys

from PIL import Image

from pipecat.frames.frames import (
    Frame,
    SystemFrame,
    TextFrame,
    ImageRawFrame,
    SpriteFrame,
    TranscriptionFrame,
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.llm_context import (
    LLMUserContextAggregator,
    LLMAssistantContextAggregator,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.services.openai import OpenAILLMService
from pipecat.services.elevenlabs import ElevenLabsTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport

from runner import configure

from loguru import logger

from dotenv import load_dotenv
load_dotenv(override=True)

logger.remove(0)
logger.add(sys.stderr, level="DEBUG")


sprites = {}
image_files = [
    "sc-default.png",
    "sc-talk.png",
    "sc-listen-1.png",
    "sc-think-1.png",
    "sc-think-2.png",
    "sc-think-3.png",
    "sc-think-4.png",
]

script_dir = os.path.dirname(__file__)

for file in image_files:
    # Build the full path to the image file
    full_path = os.path.join(script_dir, "assets", file)
    # Get the filename without the extension to use as the dictionary key
    filename = os.path.splitext(os.path.basename(full_path))[0]
    # Open the image and convert it to bytes
    with Image.open(full_path) as img:
        sprites[file] = ImageRawFrame(image=img.tobytes(), size=img.size, format=img.format)

# When the bot isn't talking, show a static image of the cat listening
quiet_frame = sprites["sc-listen-1.png"]

# When the bot is talking, build an animation from two sprites
talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
talking = [random.choice(talking_list) for x in range(30)]
talking_frame = SpriteFrame(talking)

# TODO: Support "thinking" as soon as we get a valid transcript, while LLM
# is processing
thinking_list = [
    sprites["sc-think-1.png"],
    sprites["sc-think-2.png"],
    sprites["sc-think-3.png"],
    sprites["sc-think-4.png"],
]
thinking_frame = SpriteFrame(thinking_list)


class NameCheckFilter(FrameProcessor):
    def __init__(self, names: list[str]):
        super().__init__()
        self._names = names
        self._sentence = ""

    async def process_frame(self, frame: Frame, direction: FrameDirection):
        if isinstance(frame, SystemFrame):
            await self.push_frame(frame, direction)
            return

        content: str = ""

        # TODO: split up transcription by participant
        if isinstance(frame, TranscriptionFrame):
            content = frame.text
            self._sentence += content
            if self._sentence.endswith((".", "?", "!")):
                if any(name in self._sentence for name in self._names):
                    await self.push_frame(TextFrame(self._sentence))
                    self._sentence = ""
                else:
                    self._sentence = ""
        else:
            await self.push_frame(frame, direction)


class ImageSyncAggregator(FrameProcessor):

    async def process_frame(self, frame: Frame, direction: FrameDirection):
        await self.push_frame(talking_frame)
        await self.push_frame(frame)
        await self.push_frame(quiet_frame)


async def main(room_url: str, token):
    async with aiohttp.ClientSession() as session:
        transport = DailyTransport(
            room_url,
            token,
            "Santa Cat",
            DailyParams(
                audio_out_enabled=True,
                camera_out_enabled=True,
                camera_out_width=720,
                camera_out_height=1280,
                camera_out_framerate=10,
                transcription_enabled=True
            )
        )

        llm = OpenAILLMService(
            api_key=os.getenv("OPENAI_API_KEY"),
            model="gpt-4-turbo-preview")

        tts = ElevenLabsTTSService(
            aiohttp_session=session,
            api_key=os.getenv("ELEVENLABS_API_KEY"),
            voice_id="jBpfuIE2acCO8z3wKNLl",
        )
        isa = ImageSyncAggregator()

        messages = [
            {
                "role": "system",
                "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while.  Your responses should only be a few sentences long.",
            },
        ]

        tma_in = LLMUserContextAggregator(messages)
        tma_out = LLMAssistantContextAggregator(messages)
        ncf = NameCheckFilter(["Santa Cat", "Santa"])

        pipeline = Pipeline([transport.input(), isa, ncf, tma_in,
                            llm, tma_out, tts, transport.output()])

        @transport.event_handler("on_first_participant_joined")
        async def on_first_participant_joined(transport, participant):
            # Send some greeting at the beginning.
            await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.")
            transport.capture_participant_transcription(participant["id"])

        async def starting_image():
            await transport.send_image(quiet_frame)

        runner = PipelineRunner()

        task = PipelineTask(pipeline)

        await asyncio.gather(runner.run(task), starting_image())


if __name__ == "__main__":
    (url, token) = configure()
    asyncio.run(main(url, token))