Compare commits
17 Commits
v0.0.102
...
fix/speech
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9786c4f8da | ||
|
|
eb0ce5aea1 | ||
|
|
67ea485566 | ||
|
|
d99a256715 | ||
|
|
dcbcab1542 | ||
|
|
e75ccd9c2f | ||
|
|
a80919ceff | ||
|
|
1fe4538982 | ||
|
|
9a48d93bd2 | ||
|
|
0c3e59ed61 | ||
|
|
ec2b38dc29 | ||
|
|
0574167fbd | ||
|
|
972ad93e18 | ||
|
|
ac53594967 | ||
|
|
b063d9d43b | ||
|
|
48e93beadf | ||
|
|
883b24f577 |
28
.claude/skills/pr-submit/SKILL.md
Normal file
28
.claude/skills/pr-submit/SKILL.md
Normal file
@@ -0,0 +1,28 @@
|
||||
---
|
||||
name: pr-submit
|
||||
description: Create and submit a GitHub PR from the current branch
|
||||
---
|
||||
|
||||
Submit the current changes as a GitHub pull request.
|
||||
|
||||
## Instructions
|
||||
|
||||
1. Check the current state of the repository:
|
||||
- Run `git status` to see staged, unstaged, and untracked changes
|
||||
- Run `git diff` to see current changes
|
||||
- Run `git log --oneline -10` to see recent commits
|
||||
|
||||
2. If there are uncommitted changes relevant to the PR:
|
||||
- Ask the user if they want a specific prefix for the branch name (e.g., `alice/`, `fix/`, `feat/`)
|
||||
- Create a new branch based on the current branch
|
||||
- Commit the changes using multiple commits if the changes are unrelated
|
||||
|
||||
3. Push the branch and create the PR:
|
||||
- Push with `-u` flag to set upstream tracking
|
||||
- Create the PR using `gh pr create`
|
||||
|
||||
4. After the PR is created:
|
||||
- Run `/changelog <pr_number>` to generate changelog files, then commit and push them
|
||||
- Run `/pr-description <pr_number>` to update the PR description
|
||||
|
||||
5. Return the PR URL to the user.
|
||||
@@ -153,6 +153,3 @@ When adding a new service:
|
||||
|
||||
Test utilities live in `src/pipecat/tests/utils.py`. Use `run_test()` to send frames through a pipeline and assert expected output frames in each direction. Use `SleepFrame(sleep=N)` to add delays between frames.
|
||||
|
||||
## Pull Requests
|
||||
|
||||
After creating a PR, use `/changelog <pr_number>` to generate the changelog file and `/pr-description <pr_number>` to update the PR description.
|
||||
|
||||
1
changelog/3706.changed.md
Normal file
1
changelog/3706.changed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added `X-User-Agent` and `X-Request-Id` headers to `InworldTTSService` for better traceability.
|
||||
1
changelog/3713.fixed.md
Normal file
1
changelog/3713.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Fixed `SmallWebRTCTransport` input audio resampling to properly handle all sample rates, including 8kHz audio.
|
||||
5
changelog/3722.fixed.md
Normal file
5
changelog/3722.fixed.md
Normal file
@@ -0,0 +1,5 @@
|
||||
- Fixed a race condition in `SpeechTimeoutUserTurnStopStrategy` where a finalized
|
||||
transcript arriving after `user_speech_timeout` elapsed from VAD stop would
|
||||
immediately trigger a turn stop, even if the user was still speaking. STT
|
||||
processing latency was consuming the `user_speech_timeout` window, leaving no
|
||||
time for the user to resume speaking.
|
||||
@@ -27,16 +27,11 @@ from loguru import logger
|
||||
print("🚀 Starting Pipecat bot...")
|
||||
print("⏳ Loading models and imports (20 seconds, first run only)\n")
|
||||
|
||||
logger.info("Loading Local Smart Turn Analyzer V3...")
|
||||
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
||||
|
||||
logger.info("✅ Local Smart Turn Analyzer V3 loaded")
|
||||
logger.info("Loading Silero VAD model...")
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
|
||||
logger.info("✅ Silero VAD model loaded")
|
||||
|
||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||
from pipecat.frames.frames import LLMRunFrame
|
||||
|
||||
logger.info("Loading pipeline components...")
|
||||
@@ -55,10 +50,6 @@ from pipecat.services.deepgram.stt import DeepgramSTTService
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.daily.transport import DailyParams
|
||||
from pipecat.turns.user_stop.turn_analyzer_user_turn_stop_strategy import (
|
||||
TurnAnalyzerUserTurnStopStrategy,
|
||||
)
|
||||
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
||||
|
||||
logger.info("✅ All components loaded successfully!")
|
||||
|
||||
@@ -87,12 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
context = LLMContext(messages)
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())]
|
||||
),
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
),
|
||||
user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()),
|
||||
)
|
||||
|
||||
pipeline = Pipeline(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
agent_name = "quickstart-test"
|
||||
image = "markatdaily/quickstart-test:latest"
|
||||
image = "your_username/quickstart-test:latest"
|
||||
secret_set = "quickstart-test-secrets"
|
||||
agent_profile = "agent-1x"
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ version = "0.1.0"
|
||||
description = "Quickstart example for building voice AI bots with Pipecat"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"pipecat-ai[webrtc,daily,silero,deepgram,openai,cartesia,local-smart-turn-v3,runner]",
|
||||
"pipecat-ai[webrtc,daily,silero,deepgram,openai,cartesia,runner]",
|
||||
"pipecat-ai-cli"
|
||||
]
|
||||
|
||||
@@ -17,4 +17,4 @@ dev = [
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
[tool.ruff.lint]
|
||||
select = ["I"]
|
||||
select = ["I"]
|
||||
|
||||
@@ -16,11 +16,16 @@ Inworld’s text-to-speech (TTS) models offer ultra-realistic, context-aware spe
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple
|
||||
|
||||
import aiohttp
|
||||
import websockets
|
||||
from loguru import logger
|
||||
|
||||
from pipecat import version as pipecat_version
|
||||
|
||||
USER_AGENT = f"pipecat/{pipecat_version()}"
|
||||
from pydantic import BaseModel
|
||||
|
||||
try:
|
||||
@@ -236,9 +241,12 @@ class InworldHttpTTSService(WordTTSService):
|
||||
# Use WORD timestamps for simplicity and correct spacing/capitalization
|
||||
payload["timestampType"] = self._timestamp_type
|
||||
|
||||
request_id = str(uuid.uuid4())
|
||||
headers = {
|
||||
"Authorization": f"Basic {self._api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"X-User-Agent": USER_AGENT,
|
||||
"X-Request-Id": request_id,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -252,7 +260,7 @@ class InworldHttpTTSService(WordTTSService):
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
error_text = await response.text()
|
||||
logger.error(f"Inworld API error: {error_text}")
|
||||
logger.error(f"Inworld API error (request_id={request_id}): {error_text}")
|
||||
yield ErrorFrame(error=f"Inworld API error: {error_text}")
|
||||
return
|
||||
|
||||
@@ -693,8 +701,13 @@ class InworldTTSService(AudioContextWordTTSService):
|
||||
if self._websocket and self._websocket.state is State.OPEN:
|
||||
return
|
||||
|
||||
logger.debug("Connecting to Inworld WebSocket TTS")
|
||||
headers = [("Authorization", f"Basic {self._api_key}")]
|
||||
request_id = str(uuid.uuid4())
|
||||
logger.debug(f"Connecting to Inworld WebSocket TTS (request_id={request_id})")
|
||||
headers = [
|
||||
("Authorization", f"Basic {self._api_key}"),
|
||||
("X-User-Agent", USER_AGENT),
|
||||
("X-Request-Id", request_id),
|
||||
]
|
||||
self._websocket = await websocket_connect(self._url, additional_headers=headers)
|
||||
await self._call_event_handler("on_connected")
|
||||
except Exception as e:
|
||||
|
||||
@@ -233,9 +233,8 @@ class SmallWebRTCClient:
|
||||
self._out_sample_rate = None
|
||||
self._leave_counter = 0
|
||||
|
||||
# We are always resampling it for 16000 if the sample_rate that we receive is bigger than that.
|
||||
# otherwise we face issues with Silero VAD
|
||||
self._pipecat_resampler = AudioResampler("s16", "mono", 16000)
|
||||
# Audio resampler - will be configured during setup with target sample rate
|
||||
self._audio_in_resampler = None
|
||||
|
||||
@self._webrtc_connection.event_handler("connected")
|
||||
async def on_connected(connection: SmallWebRTCConnection):
|
||||
@@ -375,32 +374,22 @@ class SmallWebRTCClient:
|
||||
await asyncio.sleep(0.01)
|
||||
continue
|
||||
|
||||
if frame.sample_rate > self._in_sample_rate:
|
||||
resampled_frames = self._pipecat_resampler.resample(frame)
|
||||
for resampled_frame in resampled_frames:
|
||||
# 16-bit PCM bytes
|
||||
pcm_array = resampled_frame.to_ndarray().astype(np.int16)
|
||||
pcm_bytes = pcm_array.tobytes()
|
||||
del pcm_array # free NumPy array immediately
|
||||
# Resample if needed, otherwise use the frame as-is
|
||||
frames_to_process = (
|
||||
self._audio_in_resampler.resample(frame)
|
||||
if frame.sample_rate != self._in_sample_rate
|
||||
else [frame]
|
||||
)
|
||||
|
||||
audio_frame = InputAudioRawFrame(
|
||||
audio=pcm_bytes,
|
||||
sample_rate=resampled_frame.sample_rate,
|
||||
num_channels=self._audio_in_channels,
|
||||
)
|
||||
audio_frame.pts = frame.pts
|
||||
del pcm_bytes # reference kept in audio_frame
|
||||
|
||||
yield audio_frame
|
||||
else:
|
||||
# 16-bit PCM bytes
|
||||
pcm_array = frame.to_ndarray().astype(np.int16)
|
||||
for processed_frame in frames_to_process:
|
||||
# Convert to 16-bit PCM bytes
|
||||
pcm_array = processed_frame.to_ndarray().astype(np.int16)
|
||||
pcm_bytes = pcm_array.tobytes()
|
||||
del pcm_array # free NumPy array immediately
|
||||
|
||||
audio_frame = InputAudioRawFrame(
|
||||
audio=pcm_bytes,
|
||||
sample_rate=frame.sample_rate,
|
||||
sample_rate=self._in_sample_rate,
|
||||
num_channels=self._audio_in_channels,
|
||||
)
|
||||
audio_frame.pts = frame.pts
|
||||
@@ -450,6 +439,7 @@ class SmallWebRTCClient:
|
||||
self._out_sample_rate = _params.audio_out_sample_rate or frame.audio_out_sample_rate
|
||||
self._params = _params
|
||||
self._leave_counter += 1
|
||||
self._audio_in_resampler = AudioResampler("s16", "mono", self._in_sample_rate)
|
||||
|
||||
async def connect(self):
|
||||
"""Establish the WebRTC connection."""
|
||||
|
||||
@@ -34,8 +34,12 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy):
|
||||
after the user stops speaking, adjusted by the VAD stop_secs.
|
||||
|
||||
For services that support finalization (TranscriptionFrame.finalized=True),
|
||||
the turn can be triggered immediately once the finalized transcript is
|
||||
received and the user resume speaking timeout has elapsed.
|
||||
receiving the finalized transcript allows the strategy to shorten the
|
||||
timeout by removing the STT wait component, since only the
|
||||
`user_speech_timeout` portion is still needed. If `user_speech_timeout`
|
||||
has already elapsed when the transcript arrives, the original timeout
|
||||
continues running to provide a buffer for VAD to detect any resumed
|
||||
speech before triggering.
|
||||
"""
|
||||
|
||||
def __init__(self, *, user_speech_timeout: float = 0.6, **kwargs):
|
||||
@@ -126,8 +130,26 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy):
|
||||
self._text += frame.text
|
||||
if frame.finalized:
|
||||
self._transcript_finalized = True
|
||||
# For finalized transcripts, check if we can trigger early
|
||||
await self._maybe_trigger_user_turn_stopped()
|
||||
# With the transcript finalized, we no longer need to wait for
|
||||
# STT latency. If a timeout is running (from VAD stop), recalculate
|
||||
# to use only user_speech_timeout, potentially shortening the wait.
|
||||
if self._timeout_task and self._vad_stopped_time is not None:
|
||||
elapsed = time.time() - self._vad_stopped_time
|
||||
remaining = self._user_speech_timeout - elapsed
|
||||
if remaining > 0:
|
||||
# Shorten timeout: replace STT+speech timeout with just
|
||||
# remaining speech timeout since STT is done.
|
||||
await self.task_manager.cancel_task(self._timeout_task)
|
||||
self._timeout_task = self.task_manager.create_task(
|
||||
self._timeout_handler(remaining), f"{self}::_timeout_handler"
|
||||
)
|
||||
# If remaining <= 0: user_speech_timeout has elapsed, but the
|
||||
# original timeout (which may include extra STT wait time) is
|
||||
# still running. Let it complete naturally — this provides a
|
||||
# buffer for VAD to detect any resumed speech before triggering.
|
||||
elif self._timeout_task is None:
|
||||
# Timeout already completed, check if we should trigger now
|
||||
await self._maybe_trigger_user_turn_stopped()
|
||||
|
||||
# Fallback: handle transcripts when no VAD stop was received.
|
||||
# This handles edge cases where transcripts arrive without VAD firing.
|
||||
@@ -178,25 +200,10 @@ class SpeechTimeoutUserTurnStopStrategy(BaseUserTurnStopStrategy):
|
||||
Conditions:
|
||||
- User is not currently speaking
|
||||
- We have transcription text
|
||||
- Either the timeout has elapsed OR we have a finalized transcript
|
||||
and user_speech_timeout has elapsed
|
||||
- The timeout has fully elapsed (timeout task completed)
|
||||
"""
|
||||
if self._vad_user_speaking or not self._text:
|
||||
return
|
||||
|
||||
# For finalized transcripts, check if user_speech_timeout has elapsed.
|
||||
# If elapsed, trigger user turn stopped immediately. Else, wait for user resume
|
||||
# speaking timeout.
|
||||
if self._transcript_finalized and self._vad_stopped_time is not None:
|
||||
elapsed = time.time() - self._vad_stopped_time
|
||||
if elapsed >= self._user_speech_timeout:
|
||||
# Cancel any remaining timeout since we're triggering now
|
||||
if self._timeout_task:
|
||||
await self.task_manager.cancel_task(self._timeout_task)
|
||||
self._timeout_task = None
|
||||
await self.trigger_user_turn_stopped()
|
||||
return
|
||||
|
||||
# For non-finalized, only trigger if timeout task has completed
|
||||
if self._timeout_task is None:
|
||||
await self.trigger_user_turn_stopped()
|
||||
|
||||
@@ -452,6 +452,72 @@ class TestSpeechTimeoutUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
|
||||
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
|
||||
self.assertTrue(should_start)
|
||||
|
||||
async def test_finalized_transcript_does_not_trigger_early_with_slow_stt(self):
|
||||
"""Test that a finalized transcript arriving after user_speech_timeout
|
||||
but before the full timeout does not trigger immediately.
|
||||
|
||||
This reproduces a race condition where:
|
||||
- STT has high latency (effective_stt_wait > user_speech_timeout)
|
||||
- User pauses briefly, VAD fires stop
|
||||
- The full timeout = max(effective_stt_wait, user_speech_timeout)
|
||||
- The finalized transcript arrives after user_speech_timeout from VAD stop
|
||||
but before the full timeout
|
||||
- The user resumes speaking before the full timeout
|
||||
|
||||
Previously, the early trigger path would fire because
|
||||
time.time() - vad_stopped_time >= user_speech_timeout, even though the
|
||||
user was about to resume speaking.
|
||||
"""
|
||||
user_speech_timeout = 0.1
|
||||
strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=user_speech_timeout)
|
||||
await strategy.setup(self.task_manager)
|
||||
|
||||
# Set high STT P99 latency so effective_stt_wait > user_speech_timeout
|
||||
stt_timeout = 0.5
|
||||
stop_secs = 0.1
|
||||
await strategy.process_frame(
|
||||
STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout)
|
||||
)
|
||||
# effective_stt_wait = max(0, 0.5 - 0.1) = 0.4
|
||||
# timeout = max(0.4, 0.1) = 0.4
|
||||
|
||||
should_start = None
|
||||
|
||||
@strategy.event_handler("on_user_turn_stopped")
|
||||
async def on_user_turn_stopped(strategy, params):
|
||||
nonlocal should_start
|
||||
should_start = True
|
||||
|
||||
# S - user starts speaking
|
||||
await strategy.process_frame(VADUserStartedSpeakingFrame())
|
||||
|
||||
# E - user pauses briefly
|
||||
await strategy.process_frame(VADUserStoppedSpeakingFrame(stop_secs=stop_secs))
|
||||
|
||||
# Wait for user_speech_timeout to elapse but NOT the full timeout
|
||||
await asyncio.sleep(user_speech_timeout + 0.05) # 0.15s elapsed
|
||||
self.assertIsNone(should_start)
|
||||
|
||||
# Finalized transcript arrives (simulating slow STT).
|
||||
# At this point, elapsed from VAD stop (~0.15s) > user_speech_timeout (0.1s).
|
||||
# The old code would trigger immediately here.
|
||||
await strategy.process_frame(
|
||||
TranscriptionFrame(text="Hello!", user_id="cat", timestamp="", finalized=True)
|
||||
)
|
||||
|
||||
# Should NOT trigger — the full timeout (0.4s) hasn't elapsed yet,
|
||||
# giving the user time to resume speaking
|
||||
self.assertIsNone(should_start)
|
||||
|
||||
# User resumes speaking — this cancels the timeout
|
||||
await strategy.process_frame(VADUserStartedSpeakingFrame())
|
||||
|
||||
# Wait well past the original timeout
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Should still not have triggered — user resumed speaking
|
||||
self.assertIsNone(should_start)
|
||||
|
||||
async def test_sie_delay_it(self):
|
||||
strategy = await self._create_strategy()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user