wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #50 from daily-co/khk/launch-samples

Browse Source 
Khk/launch samples
This commit is contained in:
Moishe Lettvin
2024-03-11 12:50:38 -04:00
committed by GitHub
parent c5bb0e14ab 4e16e514dd
commit de026ccc8a
6 changed files with 83 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/dailyai/services/fal_ai_services.py
View File

@@ -9,17 +9,19 @@ from dailyai.services.ai_services import ImageGenService
from dailyai.services.ai_services import ImageGenService
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
class FalImageGenService(ImageGenService):
def __init__(
self,
*,
image_size,
aiohttp_session: aiohttp.ClientSession,
key_id=None,
key_secret=None):
self,
*,
image_size,
aiohttp_session: aiohttp.ClientSession,
key_id=None,
key_secret=None
):
super().__init__(image_size)
self._aiohttp_session = aiohttp_session
if key_id:
@@ -30,10 +32,9 @@ class FalImageGenService(ImageGenService):
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
def get_image_url(sentence, size):
handler = fal.apps.submit(
"110602490-fast-sdxl",
arguments={
"prompt": sentence
},
# "110602490-fast-sdxl",
"fal-ai/fast-sdxl",
arguments={"prompt": sentence},
)
for event in handler.iter_events():
if isinstance(event, fal.apps.InProgress):
@@ -46,6 +47,7 @@ class FalImageGenService(ImageGenService):
raise Exception("Image generation failed")
return image_url
image_url = await asyncio.to_thread(get_image_url, sentence, self.image_size)
# Load the image from the url
async with self._aiohttp_session.get(image_url) as response:

17
src/examples/foundational/01-say-one-thing.py
View File

@@ -5,7 +5,6 @@ import os
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.playht_ai_service import PlayHTAIService
from examples.support.runner import configure
@@ -13,19 +12,14 @@ logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url):
async with aiohttp.ClientSession() as session:
# create a transport service object using environment variables for
# the transport service's API key, room url, and any other configuration.
# services can all define and document the environment variables they use.
# services all also take an optional config object that is used instead of
# environment variables.
#
# the abstract transport service APIs presumably can map pretty closely
# to the daily-python basic API
meeting_duration_minutes = 5
transport = DailyTransportService(
room_url, None, "Say One Thing", meeting_duration_minutes, mic_enabled=True
room_url,
None,
"Say One Thing",
mic_enabled=True,
)
tts = ElevenLabsTTSService(
@@ -37,7 +31,6 @@ async def main(room_url):
# Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant):
nonlocal tts
if participant["info"]["isLocal"]:
return

19
src/examples/foundational/02-llm-say-one-thing.py
View File

@@ -6,24 +6,22 @@ import aiohttp
from dailyai.pipeline.frames import LLMMessagesQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url):
async with aiohttp.ClientSession() as session:
meeting_duration_minutes = 1
transport = DailyTransportService(
room_url,
None,
"Say One Thing From an LLM",
duration_minutes=meeting_duration_minutes,
mic_enabled=True,
)
@@ -32,25 +30,24 @@ async def main(room_url):
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
)
messages = [
{
"role": "system",
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
}
]
tts_task = asyncio.create_task(
tts.run_to_queue(
transport.send_queue,
llm.run([LLMMessagesQueueFrame(messages)]),
)
)
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts_task
await tts.run_to_queue(
transport.send_queue,
llm.run([LLMMessagesQueueFrame(messages)]),
)
await transport.stop_when_done()
await transport.run()

19
src/examples/foundational/03-still-frame.py
View File

@@ -6,48 +6,37 @@ import os
from dailyai.pipeline.frames import TextFrame
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService
from dailyai.services.azure_ai_services import AzureImageGenServiceREST
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
local_joined = False
participant_joined = False
async def main(room_url):
async with aiohttp.ClientSession() as session:
meeting_duration_minutes = 1
transport = DailyTransportService(
room_url,
None,
"Show a still frame image",
duration_minutes=meeting_duration_minutes,
mic_enabled=False,
camera_enabled=True,
camera_width=1024,
camera_height=1024,
)
imagegen = FalImageGenService(
image_size="1024x1024",
image_size="square_hd",
aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET"),
)
image_task = asyncio.create_task(
imagegen.run_to_queue(
transport.send_queue, [TextFrame("a cat in the style of picasso")]
)
)
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await image_task
await imagegen.run_to_queue(
transport.send_queue, [TextFrame("a cat in the style of picasso")]
)
await transport.run()

17
src/examples/foundational/04-utterance-and-speech.py
View File

@@ -7,6 +7,7 @@ from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from examples.support.runner import configure
@@ -25,7 +26,6 @@ async def main(room_url: str):
duration_minutes=1,
mic_enabled=True,
mic_sample_rate=16000,
camera_enabled=False,
)
llm = AzureLLMService(
@@ -37,6 +37,11 @@ async def main(room_url: str):
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"),
)
deepgram_tts = DeepgramTTSService(
aiohttp_session=session,
api_key=os.getenv("DEEPGRAM_API_KEY"),
)
elevenlabs_tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
@@ -64,6 +69,15 @@ async def main(room_url: str):
transport.send_queue,
)
# khk: deepgram_tts.say() doesn't seem to put bytes in the transport
# queue. I get a debug log line that indicates we're set up okay, but
# no further log lines or audio bytes. debug this later:
# 20 2024-03-10 13:24:46,235 Running deepgram tts for My friend the LLM is now going to tell a joke about llamas.
# await deepgram_tts.say(
# "My friend the LLM is now going to tell a joke about llamas.",
# transport.send_queue,
# )
async def buffer_to_send_queue():
while True:
frame = await buffer_queue.get()
@@ -73,7 +87,6 @@ async def main(room_url: str):
break
await asyncio.gather(pipeline_run_task, buffer_to_send_queue())
await transport.stop_when_done()
await transport.run()

49
src/examples/foundational/05-sync-speech-and-image.py
View File

@@ -4,6 +4,9 @@ import aiohttp
import os
import logging
from dataclasses import dataclass
from typing import AsyncGenerator
from dailyai.pipeline.aggregators import (
GatedAggregator,
LLMFullResponseAggregator,
@@ -11,17 +14,20 @@ from dailyai.pipeline.aggregators import (
SentenceAggregator,
)
from dailyai.pipeline.frames import (
AudioFrame,
Frame,
TextFrame,
EndFrame,
ImageFrame,
LLMMessagesQueueFrame,
LLMResponseStartFrame,
)
from dailyai.pipeline.frame_processor import FrameProcessor
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from examples.support.runner import configure
@@ -30,14 +36,35 @@ logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
@dataclass
class MonthFrame(Frame):
month: str
class MonthPrepender(FrameProcessor):
def __init__(self):
self.most_recent_month = "Placeholder, month frame not yet received"
self.prepend_to_next_text_frame = False
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, MonthFrame):
self.most_recent_month = frame.month
elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame):
yield TextFrame(f"{self.most_recent_month}: {frame.text}")
self.prepend_to_next_text_frame = False
elif isinstance(frame, LLMResponseStartFrame):
self.prepend_to_next_text_frame = True
yield frame
else:
yield frame
async def main(room_url):
async with aiohttp.ClientSession() as session:
meeting_duration_minutes = 5
transport = DailyTransportService(
room_url,
None,
"Month Narration Bot",
duration_minutes=meeting_duration_minutes,
mic_enabled=True,
camera_enabled=True,
mic_sample_rate=16000,
@@ -55,8 +82,8 @@ async def main(room_url):
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
)
dalle = FalImageGenService(
image_size="1024x1024",
imagegen = FalImageGenService(
image_size="square_hd",
aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET"),
@@ -84,6 +111,7 @@ async def main(room_url):
"content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
}
]
await source_queue.put(MonthFrame(month))
await source_queue.put(LLMMessagesQueueFrame(messages))
await source_queue.put(EndFrame())
@@ -95,6 +123,7 @@ async def main(room_url):
)
sentence_aggregator = SentenceAggregator()
month_prepender = MonthPrepender()
llm_full_response_aggregator = LLMFullResponseAggregator()
pipeline = Pipeline(
@@ -103,7 +132,9 @@ async def main(room_url):
processors=[
llm,
sentence_aggregator,
ParallelPipeline([[tts], [llm_full_response_aggregator, dalle]]),
ParallelPipeline(
[[month_prepender, tts], [llm_full_response_aggregator, imagegen]]
),
gated_aggregator,
],
)
@@ -112,8 +143,6 @@ async def main(room_url):
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await pipeline_task
# wait for the output queue to be empty, then leave the meeting
await transport.stop_when_done()
await transport.run()