wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add InterruptibleConversationWrapper

Browse Source
This commit is contained in:
Moishe Lettvin
2024-01-23 21:07:27 -05:00
parent 31db156dfc
commit 795a339542
14 changed files with 226 additions and 1243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/dailyai/services/ai_services.py
View File

@@ -7,6 +7,7 @@ import wave
from dailyai.queue_frame import (
AudioQueueFrame,
ControlQueueFrame,
EndStreamQueueFrame,
ImageQueueFrame,
LLMMessagesQueueFrame,
@@ -28,11 +29,15 @@ class AIService:
pass
async def run_to_queue(self, queue: asyncio.Queue, frames, add_end_of_stream=False) -> None:
async for frame in self.run(frames):
await queue.put(frame)
try:
async for frame in self.run(frames):
await queue.put(frame)
if add_end_of_stream:
await queue.put(EndStreamQueueFrame())
if add_end_of_stream:
await queue.put(EndStreamQueueFrame())
except Exception as e:
print("Exception in run_to_queue", e)
raise e
async def run(
self,
@@ -67,9 +72,8 @@ class AIService:
@abstractmethod
async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]:
# This is a trick for the interpreter (and linter) to know that this is a generator.
if False:
yield QueueFrame()
if isinstance(frame, ControlQueueFrame):
yield frame
@abstractmethod
async def finalize(self) -> AsyncGenerator[QueueFrame, None]:
@@ -87,7 +91,9 @@ class LLMService(AIService):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
if isinstance(frame, ControlQueueFrame):
yield frame
elif isinstance(frame, LLMMessagesQueueFrame):
async for text_chunk in self.run_llm_async(frame.messages):
yield TextQueueFrame(text_chunk)
@@ -110,9 +116,11 @@ class TTSService(AIService):
yield bytes()
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if not isinstance(frame, TextQueueFrame):
if isinstance(frame, ControlQueueFrame):
yield frame
return
elif not isinstance(frame, TextQueueFrame):
return
text: str | None = None
if not self.aggregate_sentences:
@@ -149,6 +157,7 @@ class ImageGenService(AIService):
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if not isinstance(frame, TextQueueFrame):
yield frame
return
(url, image_data) = await self.run_image_gen(frame.text)

5
src/dailyai/services/azure_ai_services.py
View File

@@ -35,10 +35,7 @@ class AzureTTSService(TTSService):
"<prosody rate='1.05'>" \
f"{sentence}" \
"</prosody></mstts:express-as></voice></speak> "
try:
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
except Exception as e:
self.logger.error("Error in azure tts", e)
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
self.logger.info("Got azure tts result")
if result.reason == ResultReason.SynthesizingAudioCompleted:
self.logger.info("Returning result")

25
src/dailyai/services/daily_transport_service.py
View File

@@ -7,6 +7,7 @@ import types
from functools import partial
from queue import Queue, Empty
from typing import AsyncGenerator
from dailyai.queue_frame import (
AudioQueueFrame,
@@ -14,6 +15,7 @@ from dailyai.queue_frame import (
ImageQueueFrame,
QueueFrame,
StartStreamQueueFrame,
TextQueueFrame,
TranscriptionQueueFrame,
)
@@ -114,6 +116,7 @@ class DailyTransportService(EventHandler):
handler(*args, **kwargs)
except Exception as e:
self.logger.error(f"Exception in event handler {event_name}: {e}")
raise e
def add_event_handler(self, event_name: str, handler):
if not event_name.startswith("on_"):
@@ -214,7 +217,6 @@ class DailyTransportService(EventHandler):
if self.token and self.start_transcription:
self.client.start_transcription(self.transcription_settings)
def _receive_audio(self):
"""Receive audio from the Daily call and put it on the receive queue"""
seconds = 1
@@ -223,9 +225,13 @@ class DailyTransportService(EventHandler):
buffer = self.speaker.read_frames(desired_frame_count)
if len(buffer) > 0:
frame = AudioQueueFrame(buffer)
asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self.loop)
if self.loop:
asyncio.run_coroutine_threadsafe(self.receive_queue.put(frame), self.loop)
async def get_receive_frames(self):
def interrupt(self):
self.is_interrupted.set()
async def get_receive_frames(self) -> AsyncGenerator[QueueFrame, None]:
while True:
frame = await self.receive_queue.get()
yield frame
@@ -265,6 +271,7 @@ class DailyTransportService(EventHandler):
await asyncio.sleep(1)
except Exception as e:
self.logger.error(f"Exception {e}")
raise e
finally:
self.client.leave()
@@ -341,6 +348,7 @@ class DailyTransportService(EventHandler):
time.sleep(1.0 / 8) # 8 fps
except Exception as e:
self.logger.error(f"Exception {e} in camera thread.")
raise e
def frame_consumer(self):
self.logger.info("🎬 Starting frame consumer thread")
@@ -382,11 +390,11 @@ class DailyTransportService(EventHandler):
self.mic.write_frames(bytes(b))
b = bytearray()
else:
if self.interrupt_time:
self.logger.info(
f"Lag to stop stream after interruption {time.perf_counter() - self.interrupt_time}"
)
self.interrupt_time = None
# if there are leftover audio bytes, write them now; failing to do so
# can cause static in the audio stream.
if len(b):
self.mic.write_frames(bytes(b))
b = bytearray()
if isinstance(frame, StartStreamQueueFrame):
self.is_interrupted.clear()
@@ -398,5 +406,6 @@ class DailyTransportService(EventHandler):
self.mic.write_frames(bytes(b))
except Exception as e:
self.logger.error(f"Exception in frame_consumer: {e}, {len(b)}")
raise e
b = bytearray()