diff --git a/examples/foundational/tmp-khk.py b/examples/foundational/tmp-khk.py index cede7fc14..3fbae9dac 100644 --- a/examples/foundational/tmp-khk.py +++ b/examples/foundational/tmp-khk.py @@ -81,6 +81,17 @@ class GreedyLLMAggregator(FrameProcessor): logger.debug(f"error: {e}") +class ClearableDeepgramTTSService(DeepgramTTSService): + def __init___(self, **kwargs): + super().__init(**kwargs) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartInterruptionFrame): + self._current_sentence = "" + + @dataclass class BufferedSentence: audio_frames: List[AudioRawFrame] = field(default_factory=list) @@ -144,8 +155,16 @@ class VADGate(FrameProcessor): await self.push_frame(frame, direction) return - if isinstance(frame, LLMFullResponseStartFrame): + # There are two ways we can be interrupted. During greedy inference, a new + # LLM response can start. Or, during playout, we can get a traditional + # user interruption frame. + if (isinstance(frame, LLMFullResponseStartFrame) or + isinstance(frame, StartInterruptionFrame)): + logger.debug(f"{frame} - Handle interruption in VADGate") self._sentences = [] + if self._audio_pusher_task: + self._audio_pusher_task.cancel() + self._audio_pusher_task = None await self.push_frame(frame, direction) return @@ -166,9 +185,8 @@ class VADGate(FrameProcessor): try: while True: if not self._sentences: - logger.debug("Audio buffer empty") - self._audio_pusher_task = None - return + await asyncio.sleep(0.01) + continue if self.vad_analyzer._vad_state != VADState.QUIET: await asyncio.sleep(0.01) @@ -190,12 +208,13 @@ class VADGate(FrameProcessor): # assume linear16 encoding (2 bytes per sample). todo: add some more # metadata to AudioRawFrame, maybe duration += (len(frame.audio) / 2 / frame.num_channels) / sample_rate + await asyncio.sleep(duration - 20 / 1000) if self.context: + logger.debug(f"Appending assistant message to context: [{s.text_frame.text}]") self.context.messages.append( {"role": "assistant", "content": s.text_frame.text} ) await self.push_frame(s.text_frame) - await asyncio.sleep(duration - 20 / 1000) except Exception as e: logger.debug(f"Exception {e}") @@ -215,7 +234,7 @@ async def main(room_url: str, token): ) ) - tts = DeepgramTTSService( + tts = ClearableDeepgramTTSService( aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-asteria-en",