services(elevenlabs): add elevenlabs package and use streaming

examples/deployment/flyio-example/bot.py

@@ -1,5 +1,4 @@
 import asyncio
-import aiohttp
 import os
 import sys
 import argparse
@@ -27,7 +26,6 @@ daily_api_url = os.getenv("DAILY_API_URL", "https://api.daily.co/v1")
 
 
 async def main(room_url: str, token: str):
-    async with aiohttp.ClientSession() as session:
     transport = DailyTransport(
         room_url,
         token,
@@ -45,7 +43,6 @@ async def main(room_url: str, token: str):
     )
 
     tts = ElevenLabsTTSService(
-            aiohttp_session=session,
         api_key=os.getenv("ELEVENLABS_API_KEY", ""),
         voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
     )

examples/dialin-chatbot/bot_daily.py

@@ -1,5 +1,4 @@
 import asyncio
-import aiohttp
 import os
 import sys
 import argparse
@@ -29,7 +28,6 @@ daily_api_url = os.getenv("DAILY_API_URL", "https://api.daily.co/v1")
 
 
 async def main(room_url: str, token: str, callId: str, callDomain: str):
-    async with aiohttp.ClientSession() as session:
     # diallin_settings are only needed if Daily's SIP URI is used
     # If you are handling this via Twilio, Telnyx, set this to None
     # and handle call-forwarding when on_dialin_ready fires.
@@ -56,14 +54,14 @@ async def main(room_url: str, token: str, callId: str, callDomain: str):
     )
 
     tts = ElevenLabsTTSService(
-            aiohttp_session=session,
         api_key=os.getenv("ELEVENLABS_API_KEY", ""),
         voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
     )
 
     llm = OpenAILLMService(
         api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+        model="gpt-4o"
+    )
 
     messages = [
         {

examples/dialin-chatbot/bot_twilio.py

@@ -1,5 +1,4 @@
 import asyncio
-import aiohttp
 import os
 import sys
 import argparse
@@ -36,8 +35,7 @@ daily_api_key = os.getenv("DAILY_API_KEY", "")
 
 
 async def main(room_url: str, token: str, callId: str, sipUri: str):
-    async with aiohttp.ClientSession() as session:
+    # dialin_settings are only needed if Daily's SIP URI is used
-        # diallin_settings are only needed if Daily's SIP URI is used
     # If you are handling this via Twilio, Telnyx, set this to None
     # and handle call-forwarding when on_dialin_ready fires.
     transport = DailyTransport(
@@ -57,14 +55,14 @@ async def main(room_url: str, token: str, callId: str, sipUri: str):
     )
 
     tts = ElevenLabsTTSService(
-            aiohttp_session=session,
         api_key=os.getenv("ELEVENLABS_API_KEY", ""),
         voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""),
     )
 
     llm = OpenAILLMService(
         api_key=os.getenv("OPENAI_API_KEY"),
-            model="gpt-4o")
+        model="gpt-4o"
+    )
 
     messages = [
         {

examples/foundational/05-sync-speech-and-image.py

@@ -89,7 +89,6 @@ async def main():
         )
 
         tts = ElevenLabsTTSService(
-            aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
         )

examples/foundational/05a-local-sync-speech-and-image.py

@@ -85,7 +85,6 @@ async def main():
                 model="gpt-4o")
 
             tts = ElevenLabsTTSService(
-                aiohttp_session=session,
                 api_key=os.getenv("ELEVENLABS_API_KEY"),
                 voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
 

examples/foundational/06a-image-sync.py

@@ -79,7 +79,6 @@ async def main():
         )
 
         tts = ElevenLabsTTSService(
-            aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
         )

examples/foundational/07b-interruptible-langchain.py

@@ -18,7 +18,6 @@ from pipecat.processors.aggregators.llm_response import (
     LLMAssistantResponseAggregator, LLMUserResponseAggregator)
 from pipecat.processors.frameworks.langchain import LangchainProcessor
 from pipecat.services.cartesia import CartesiaTTSService
-from pipecat.services.elevenlabs import ElevenLabsTTSService
 from pipecat.transports.services.daily import DailyParams, DailyTransport
 from pipecat.vad.silero import SileroVADAnalyzer
 

examples/foundational/11-sound-effects.py

@@ -104,7 +104,6 @@ async def main():
             model="gpt-4o")
 
         tts = ElevenLabsTTSService(
-            aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             voice_id="ErXwobaYiN019PkySvjV",
         )

examples/simple-chatbot/bot.py

@@ -111,7 +111,6 @@ async def main():
         )
 
         tts = ElevenLabsTTSService(
-            aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             #
             # English

examples/storytelling-chatbot/src/bot.py

@@ -60,7 +60,6 @@ async def main(room_url, token=None):
         )
 
         tts_service = ElevenLabsTTSService(
-            aiohttp_session=session,
             api_key=os.getenv("ELEVENLABS_API_KEY"),
             voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
         )

pyproject.toml

@@ -39,6 +39,7 @@ azure = [ "azure-cognitiveservices-speech~=1.40.0" ]
 cartesia = [ "websockets~=12.0" ]
 daily = [ "daily-python~=0.10.1" ]
 deepgram = [ "deepgram-sdk~=3.5.0" ]
+elevenlabs = [ "elevenlabs~=1.7.0" ]
 examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ]
 fal = [ "fal-client~=0.4.1" ]
 gladia = [ "websockets~=12.0" ]

src/pipecat/services/cartesia.py

@@ -10,7 +10,7 @@ import base64
 import asyncio
 import time
 
-from typing import AsyncGenerator, Mapping
+from typing import AsyncGenerator
 
 from pipecat.frames.frames import (
     CancelFrame,

src/pipecat/services/elevenlabs.py

@@ -4,16 +4,36 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import aiohttp
-
 from typing import AsyncGenerator, Literal
 from pydantic import BaseModel
 
-from pipecat.frames.frames import AudioRawFrame, ErrorFrame, Frame, TTSStartedFrame, TTSStoppedFrame
+from pipecat.frames.frames import AudioRawFrame, Frame, TTSStartedFrame, TTSStoppedFrame
 from pipecat.services.ai_services import TTSService
 
 from loguru import logger
 
+# See .env.example for ElevenLabs configuration needed
+try:
+    from elevenlabs.client import AsyncElevenLabs
+except ModuleNotFoundError as e:
+    logger.error(f"Exception: {e}")
+    logger.error(
+        "In order to use ElevenLabs, you need to `pip install pipecat-ai[elevenlabs]`. Also, set `ELEVENLABS_API_KEY` environment variable.")
+    raise Exception(f"Missing module: {e}")
+
+
+def sample_rate_from_output_format(output_format: str) -> int:
+    match output_format:
+        case "pcm_16000":
+            return 16000
+        case "pcm_22050":
+            return 22050
+        case "pcm_24000":
+            return 24000
+        case "pcm_44100":
+            return 44100
+    return 16000
+
 
 class ElevenLabsTTSService(TTSService):
     class InputParams(BaseModel):
@@ -24,21 +44,24 @@ class ElevenLabsTTSService(TTSService):
             *,
             api_key: str,
             voice_id: str,
-            aiohttp_session: aiohttp.ClientSession,
             model: str = "eleven_turbo_v2_5",
             params: InputParams = InputParams(),
             **kwargs):
         super().__init__(**kwargs)
 
-        self._api_key = api_key
         self._voice_id = voice_id
         self._model = model
         self._params = params
-        self._aiohttp_session = aiohttp_session
+        self._client = AsyncElevenLabs(api_key=api_key)
+        self._sample_rate = sample_rate_from_output_format(params.output_format)
 
     def can_generate_metrics(self) -> bool:
         return True
 
+    async def set_model(self, model: str):
+        logger.debug(f"Switching TTS model to: [{model}]")
+        self._model = model
+
     async def set_voice(self, voice: str):
         logger.debug(f"Switching TTS voice to: [{voice}]")
         self._voice_id = voice
@@ -46,34 +69,25 @@ class ElevenLabsTTSService(TTSService):
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         logger.debug(f"Generating TTS: [{text}]")
 
-        url = f"https://api.elevenlabs.io/v1/text-to-speech/{self._voice_id}/stream"
+        await self.start_tts_usage_metrics(text)
-
-        payload = {"text": text, "model_id": self._model}
-
-        querystring = {
-            "output_format": self._params.output_format
-        }
-
-        headers = {
-            "xi-api-key": self._api_key,
-            "Content-Type": "application/json",
-        }
-
         await self.start_ttfb_metrics()
 
-        async with self._aiohttp_session.post(url, json=payload, headers=headers, params=querystring) as r:
+        results = await self._client.generate(
-            if r.status != 200:
+            text=text,
-                text = await r.text()
+            voice=self._voice_id,
-                logger.error(f"{self} error getting audio (status: {r.status}, error: {text})")
+            model=self._model,
-                yield ErrorFrame(f"Error getting audio (status: {r.status}, error: {text})")
+            output_format=self._params.output_format
-                return
+        )
-
-            await self.start_tts_usage_metrics(text)
 
+        tts_started = False
+        async for audio in results:
+            # This is so we send TTSStartedFrame when we have the first audio
+            # bytes.
+            if not tts_started:
                 await self.push_frame(TTSStartedFrame())
-            async for chunk in r.content:
+                tts_started = True
-                if len(chunk) > 0:
             await self.stop_ttfb_metrics()
-                    frame = AudioRawFrame(chunk, 16000, 1)
+            frame = AudioRawFrame(audio, self._sample_rate, 1)
             yield frame
+
         await self.push_frame(TTSStoppedFrame())