diff --git a/examples/daily-multi-translation/Dockerfile b/examples/daily-multi-translation/Dockerfile new file mode 100644 index 000000000..419adca34 --- /dev/null +++ b/examples/daily-multi-translation/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.10-bullseye + +RUN mkdir /app +RUN mkdir /app/assets +RUN mkdir /app/utils +COPY *.py /app/ +COPY requirements.txt /app/ + + +WORKDIR /app +RUN pip3 install -r requirements.txt + +EXPOSE 7860 + +CMD ["python3", "server.py"] diff --git a/examples/daily-multi-translation/README.md b/examples/daily-multi-translation/README.md new file mode 100644 index 000000000..7e27cb217 --- /dev/null +++ b/examples/daily-multi-translation/README.md @@ -0,0 +1,39 @@ +# Daily Multi Translation + +This example shows how to use Daily to stream multiple simultaneous translations using a single transport. Daily provides custom tracks and in this example we will simultaneously translate incoming audio in English to Spanish, French and German, each of them being sent to a custom track. + +## Get started + +```python +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt + +cp env.example .env # and add your credentials + +``` + +## Run the server + +```bash +python server.py +``` + +Then, visit `http://localhost:7860/` in your browser. This will open a Daily Prebuilt room where you will speak in English (make sure you are not muted). + +## Open client + +Next, you need to open the client that will listen to the translations. + +```bash +open index.html +``` + +Once the client is opened, copy the URL of the Daily room created above and join it. You should be able to select which translation you want to hear. + +## Build and test the Docker image + +``` +docker build -t daily-multi-translation . +docker run --env-file .env -p 7860:7860 daily-multi-translation +``` diff --git a/examples/daily-multi-translation/bot.py b/examples/daily-multi-translation/bot.py new file mode 100644 index 000000000..32a157096 --- /dev/null +++ b/examples/daily-multi-translation/bot.py @@ -0,0 +1,164 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +BACKGROUND_SOUND_FILE = "office-ambience-mono-16000.mp3" + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Multi translation bot", + DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_mixer={ + "spanish": SoundfileMixer( + sound_files={"office": BACKGROUND_SOUND_FILE}, default_sound="office" + ), + "french": SoundfileMixer( + sound_files={"office": BACKGROUND_SOUND_FILE}, default_sound="office" + ), + "german": SoundfileMixer( + sound_files={"office": BACKGROUND_SOUND_FILE}, default_sound="office" + ), + }, + audio_out_destinations=["spanish", "french", "german"], + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts_spanish = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="cefcb124-080b-4655-b31f-932f3ee743de", + destination="spanish", + ) + tts_french = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="8832a0b5-47b2-4751-bb22-6a8e2149303d", + destination="french", + ) + tts_german = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="38aabb6a-f52b-4fb0-a3d1-988518f4dc06", + destination="german", + ) + + messages_spanish = [ + { + "role": "system", + "content": "You will be provided with a sentence in English, and your task is to only translate it into Spanish.", + }, + ] + messages_french = [ + { + "role": "system", + "content": "You will be provided with a sentence in English, and your task is to only translate it into French.", + }, + ] + messages_german = [ + { + "role": "system", + "content": "You will be provided with a sentence in English, and your task is to only translate it into German.", + }, + ] + + llm_spanish = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm_french = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm_german = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + context_spanish = OpenAILLMContext(messages_spanish) + context_aggregator_spanish = llm_spanish.create_context_aggregator(context_spanish) + + context_french = OpenAILLMContext(messages_french) + context_aggregator_french = llm_french.create_context_aggregator(context_french) + + context_german = OpenAILLMContext(messages_german) + context_aggregator_german = llm_german.create_context_aggregator(context_german) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + ParallelPipeline( + # Spanish pipeline. + [ + context_aggregator_spanish.user(), + llm_spanish, + tts_spanish, + context_aggregator_spanish.assistant(), + ], + # French pipeline. + [ + context_aggregator_french.user(), + llm_french, + tts_french, + context_aggregator_french.assistant(), + ], + # German pipeline. + [ + context_aggregator_german.user(), + llm_german, + tts_german, + context_aggregator_german.assistant(), + ], + ), + transport.output(), # Transport bot output + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + audio_in_sample_rate=16000, + audio_out_sample_rate=16000, + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + observers=[TranscriptionLogObserver()], + ) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/daily-multi-translation/env.example b/examples/daily-multi-translation/env.example new file mode 100644 index 000000000..a780ec7d8 --- /dev/null +++ b/examples/daily-multi-translation/env.example @@ -0,0 +1,5 @@ +DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev) +DAILY_API_KEY=7df... +OPENAI_API_KEY=sk-PL... +DEEPGRAM_API_KEY=efb... +CARTESIA_API_KEY=aeb... diff --git a/examples/daily-multi-translation/index.html b/examples/daily-multi-translation/index.html new file mode 100644 index 000000000..52fd3d488 --- /dev/null +++ b/examples/daily-multi-translation/index.html @@ -0,0 +1,202 @@ + + + video layers demo + + + + + + + + +
+
+
+
+ + +
+
+
+
+
+ + +
+
+
+

+
+
+
+
+
+
+
+ + diff --git a/examples/daily-multi-translation/office-ambience-mono-16000.mp3 b/examples/daily-multi-translation/office-ambience-mono-16000.mp3 new file mode 100644 index 000000000..ea98082c7 Binary files /dev/null and b/examples/daily-multi-translation/office-ambience-mono-16000.mp3 differ diff --git a/examples/daily-multi-translation/requirements.txt b/examples/daily-multi-translation/requirements.txt new file mode 100644 index 000000000..e20c41d5a --- /dev/null +++ b/examples/daily-multi-translation/requirements.txt @@ -0,0 +1,5 @@ +aiofiles +python-dotenv +fastapi[all] +uvicorn +pipecat-ai[daily,deepgram,openai,silero,cartesia] diff --git a/examples/daily-multi-translation/runner.py b/examples/daily-multi-translation/runner.py new file mode 100644 index 000000000..50743fd09 --- /dev/null +++ b/examples/daily-multi-translation/runner.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +import aiohttp + +from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper + + +async def configure(aiohttp_session: aiohttp.ClientSession): + parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=False, help="URL of the Daily room to join" + ) + parser.add_argument( + "-k", + "--apikey", + type=str, + required=False, + help="Daily API Key (needed to create an owner token for the room)", + ) + + args, unknown = parser.parse_known_args() + + url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL") + key = args.apikey or os.getenv("DAILY_API_KEY") + + if not url: + raise Exception( + "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL." + ) + + if not key: + raise Exception( + "No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers." + ) + + daily_rest_helper = DailyRESTHelper( + daily_api_key=key, + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=aiohttp_session, + ) + + # Create a meeting token for the given room with an expiration 1 hour in + # the future. + expiry_time: float = 60 * 60 + + token = await daily_rest_helper.get_token(url, expiry_time) + + return (url, token) + return (url, token) diff --git a/examples/daily-multi-translation/server.py b/examples/daily-multi-translation/server.py new file mode 100644 index 000000000..a0f38854c --- /dev/null +++ b/examples/daily-multi-translation/server.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os +import subprocess +from contextlib import asynccontextmanager + +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, RedirectResponse + +from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams + +MAX_BOTS_PER_ROOM = 1 + +# Bot sub-process dict for status reporting and concurrency control +bot_procs = {} + +daily_helpers = {} + +load_dotenv(override=True) + + +def cleanup(): + # Clean up function, just to be extra safe + for entry in bot_procs.values(): + proc = entry[0] + proc.terminate() + proc.wait() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + aiohttp_session = aiohttp.ClientSession() + daily_helpers["rest"] = DailyRESTHelper( + daily_api_key=os.getenv("DAILY_API_KEY", ""), + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=aiohttp_session, + ) + yield + await aiohttp_session.close() + cleanup() + + +app = FastAPI(lifespan=lifespan) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/") +async def start_agent(request: Request): + print(f"!!! Creating room") + room = await daily_helpers["rest"].create_room(DailyRoomParams()) + print(f"!!! Room URL: {room.url}") + # Ensure the room property is present + if not room.url: + raise HTTPException( + status_code=500, + detail="Missing 'room' property in request data. Cannot start agent without a target room!", + ) + + # Check if there is already an existing process running in this room + num_bots_in_room = sum( + 1 for proc in bot_procs.values() if proc[1] == room.url and proc[0].poll() is None + ) + if num_bots_in_room >= MAX_BOTS_PER_ROOM: + raise HTTPException(status_code=500, detail=f"Max bot limited reach for room: {room.url}") + + # Get the token for the room + token = await daily_helpers["rest"].get_token(room.url) + + if not token: + raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}") + + # Spawn a new agent, and join the user session + # Note: this is mostly for demonstration purposes (refer to 'deployment' in README) + try: + proc = subprocess.Popen( + [f"python3 -m bot -u {room.url} -t {token}"], + shell=True, + bufsize=1, + cwd=os.path.dirname(os.path.abspath(__file__)), + ) + bot_procs[proc.pid] = (proc, room.url) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}") + + return RedirectResponse(room.url) + + +@app.get("/status/{pid}") +def get_status(pid: int): + # Look up the subprocess + proc = bot_procs.get(pid) + + # If the subprocess doesn't exist, return an error + if not proc: + raise HTTPException(status_code=404, detail=f"Bot with process id: {pid} not found") + + # Check the status of the subprocess + if proc[0].poll() is None: + status = "running" + else: + status = "finished" + + return JSONResponse({"bot_id": pid, "status": status}) + + +if __name__ == "__main__": + import uvicorn + + default_host = os.getenv("HOST", "0.0.0.0") + default_port = int(os.getenv("FAST_API_PORT", "7860")) + + parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server") + parser.add_argument("--host", type=str, default=default_host, help="Host address") + parser.add_argument("--port", type=int, default=default_port, help="Port number") + parser.add_argument("--reload", action="store_true", help="Reload code on change") + + config = parser.parse_args() + + uvicorn.run( + "server:app", + host=config.host, + port=config.port, + reload=config.reload, + )