diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index 3ae4540f0..429e5a9fb 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -14,6 +14,7 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index 40029e673..a17ee3e85 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -17,6 +17,13 @@ from loguru import logger from pipecat.frames.frames import ( AudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + TranscriptionFrame, + TextFrame, + StartFrame, + EndFrame, CancelFrame, EndFrame, ErrorFrame, @@ -339,6 +346,12 @@ class UltravoxSTTService(AIService): # Concatenate audio frames - all should be int16 now audio_data = np.concatenate(audio_arrays) + audio_int16 = audio_data # Already in int16 format + # Save int16 audio + + # Convert int16 to float32 and normalize for model input + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + # Generate text using the model if self._model: try: @@ -349,11 +362,11 @@ class UltravoxSTTService(AIService): await self.start_ttfb_metrics() await self.start_processing_metrics() - async for response in self._model.generate( + async for response in self.model.generate( messages=[{"role": "user", "content": "<|audio|>\n"}], - temperature=self._temperature, - max_tokens=self._max_tokens, - audio=audio_data, + temperature=self.temperature, + max_tokens=self.max_tokens, + audio=audio_float32, ): # Stop TTFB metrics after first response await self.stop_ttfb_metrics() @@ -369,18 +382,13 @@ class UltravoxSTTService(AIService): await self.stop_processing_metrics() logger.info(f"Generated text: {full_response}") - # Create a transcription frame with the generated text - transcription = full_response.strip() - if transcription: - yield TranscriptionFrame( - user_id="", - text=transcription, - timestamp=time_now_iso8601(), - ) - else: - logger.warning("Empty transcription result") - yield ErrorFrame("Empty transcription result") + yield LLMFullResponseStartFrame() + + text_frame = LLMTextFrame(text=full_response.strip()) + yield text_frame + + yield LLMFullResponseEndFrame() except Exception as e: logger.error(f"Error generating text from model: {e}") diff --git a/tests/test_user_idle_processor.py b/tests/test_user_idle_processor.py index a2f2fd386..7ea6f8744 100644 --- a/tests/test_user_idle_processor.py +++ b/tests/test_user_idle_processor.py @@ -86,9 +86,9 @@ class TestUserIdleProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_down_frames, ) - assert not callback_called.is_set(), ( - "Idle callback was called even though bot speaking frames reset the timer" - ) + assert ( + not callback_called.is_set() + ), "Idle callback was called even though bot speaking frames reset the timer" async def test_idle_retry_callback(self): """Test that retry count increases until user activity resets it."""