wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup continues

Browse Source
This commit is contained in:
Moishe Lettvin
2024-01-26 07:57:41 -05:00
parent 42c142aff0
commit 646db8b9bd
4 changed files with 62 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/dailyai/services/azure_ai_services.py
View File

@@ -5,7 +5,6 @@ import json
from openai import AsyncAzureOpenAI
import os
import requests
from collections.abc import AsyncGenerator
@@ -16,7 +15,10 @@ from PIL import Image
from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
class AzureTTSService(TTSService):
def __init__(self, speech_key=None, speech_region=None):
def __init__(
self, speech_key=None, speech_region=None, voice_name="en-US-SaraNeural"
):
super().__init__()
speech_key = speech_key or os.getenv("AZURE_SPEECH_SERVICE_KEY")
@@ -25,11 +27,13 @@ class AzureTTSService(TTSService):
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None)
self.voice_name = voice_name
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
self.logger.info("Running azure tts")
ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
ssml = f"<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
"xmlns:mstts='http://www.w3.org/2001/mstts'>" \
"<voice name='en-US-SaraNeural'>" \
f"<voice name={self.voice_name}>" \
"<mstts:silence type='Sentenceboundary' value='20ms' />" \
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \
"<prosody rate='1.05'>" \
@@ -92,13 +96,25 @@ class AzureLLMService(LLMService):
class AzureImageGenServiceREST(ImageGenService):
def __init__(self, image_size:str, api_key=None, azure_endpoint=None, api_version=None, model=None, aiohttp_client_session=None):
def __init__(
self,
image_size: str,
api_key: str | None = None,
azure_endpoint: str | None = None,
api_version: str | None = None,
model: str | None = None,
aiohttp_session: aiohttp.ClientSession | None=None,
timeout_seconds=120,
):
super().__init__(image_size=image_size)
self.api_key = api_key or os.getenv("AZURE_DALLE_KEY")
self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
self.api_version = api_version or "2023-06-01-preview"
self.model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID")
self.aiohttp_client_session = aiohttp_client_session or aiohttp.ClientSession()
self.aiohttp_session: aiohttp.ClientSession = (
aiohttp_session or aiohttp.ClientSession()
)
self.timeout_seconds = timeout_seconds
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}"
@@ -108,13 +124,13 @@ class AzureImageGenServiceREST(ImageGenService):
"size": self.image_size,
"n": 1,
}
async with self.aiohttp_client_session.post(
async with self.aiohttp_session.post(
url, headers=headers, json=body
) as submission:
operation_location = submission.headers['operation-location']
status = ""
attempts_left = 120
attempts_left = self.timeout_seconds
json_response = None
while status != "succeeded":
attempts_left -= 1
@@ -122,7 +138,7 @@ class AzureImageGenServiceREST(ImageGenService):
raise Exception("Image generation timed out")
await asyncio.sleep(1)
response = await self.aiohttp_client_session.get(operation_location, headers=headers)
response = await self.aiohttp_session.get(operation_location, headers=headers)
json_response = await response.json()
status = json_response["status"]
@@ -131,7 +147,7 @@ class AzureImageGenServiceREST(ImageGenService):
raise Exception("Image generation failed")
# Load the image from the url
async with self.aiohttp_client_session.get(image_url) as response:
async with self.aiohttp_session.get(image_url) as response:
image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream)
return (image_url, image.tobytes())

38
src/dailyai/services/elevenlabs_ai_service.py
View File

@@ -9,28 +9,30 @@ from dailyai.services.ai_services import TTSService
class ElevenLabsTTSService(TTSService):
def __init__(self, api_key=None, voice_id=None):
def __init__(self, api_key=None, voice_id=None, aiohttp_session:aiohttp.ClientSession=None):
super().__init__()
self.api_key = api_key or os.getenv("ELEVENLABS_API_KEY")
self.voice_id = voice_id or os.getenv("ELEVENLABS_VOICE_ID")
self.aiohttp_session = aiohttp_session or aiohttp.ClientSession()
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
async with aiohttp.ClientSession() as session:
url = f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}/stream"
payload = {"text": sentence, "model_id": "eleven_turbo_v2"}
querystring = {"output_format": "pcm_16000", "optimize_streaming_latency": 2}
headers = {
"xi-api-key": self.api_key,
"Content-Type": "application/json",
}
async with session.post(url, json=payload, headers=headers, params=querystring) as r:
if r.status != 200:
self.logger.error(
f"audio fetch status code: {r.status}, error: {r.text}"
)
return
url = f"https://api.elevenlabs.io/v1/text-to-speech/{self.voice_id}/stream"
payload = {"text": sentence, "model_id": "eleven_turbo_v2"}
querystring = {"output_format": "pcm_16000", "optimize_streaming_latency": 2}
headers = {
"xi-api-key": self.api_key,
"Content-Type": "application/json",
}
async with self.aiohttp_session.post(
url, json=payload, headers=headers, params=querystring
) as r:
if r.status != 200:
self.logger.error(
f"audio fetch status code: {r.status}, error: {r.text}"
)
return
async for chunk in r.content:
if chunk:
yield chunk
async for chunk in r.content:
if chunk:
yield chunk

4
src/dailyai/services/fal_ai_services.py
View File

@@ -34,9 +34,9 @@ class FalImageGenService(ImageGenService):
raise Exception("Image generation failed")
return image_url
print(f"fetching image url...")
print("fetching image url...")
image_url = await asyncio.to_thread(get_image_url, sentence, self.image_size)
print(f"got image url, downloading image...")
print("got image url, downloading image...")
# Load the image from the url
async with aiohttp.ClientSession() as session:
async with session.get(image_url) as response:

25
src/dailyai/services/open_ai_services.py
View File

@@ -1,6 +1,4 @@
import requests
import aiohttp
import asyncio
from PIL import Image
import io
from openai import AsyncOpenAI
@@ -9,7 +7,7 @@ import os
import json
from collections.abc import AsyncGenerator
from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService
from dailyai.services.ai_services import LLMService, ImageGenService
class OpenAILLMService(LLMService):
@@ -50,11 +48,19 @@ class OpenAILLMService(LLMService):
return None
class OpenAIImageGenService(ImageGenService):
def __init__(self, image_size:str, api_key=None, model=None):
def __init__(
self,
image_size: str,
api_key=None,
model=None,
aiohttp_session: aiohttp.ClientSession | None = None,
):
super().__init__(image_size=image_size)
api_key = api_key or os.getenv("OPEN_AI_KEY")
self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"
self.client = AsyncOpenAI(api_key=api_key)
self.aiohttp_session=aiohttp_session or aiohttp.ClientSession()
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
self.logger.info("Generating OpenAI image", sentence)
@@ -70,10 +76,7 @@ class OpenAIImageGenService(ImageGenService):
raise Exception("No image provided in response", image)
# Load the image from the url
async with aiohttp.ClientSession() as session:
async with session.get(image_url) as response:
image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream)
return (image_url, image.tobytes())
return (image_url, dalle_im.tobytes())
async with self.aiohttp_session.get(image_url) as response:
image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream)
return (image_url, image.tobytes())