Compare commits
322 Commits
aleix/fram
...
fix/event-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f2ffa8fed | ||
|
|
829c5f4604 | ||
|
|
dc8ea615d9 | ||
|
|
a3d206050d | ||
|
|
f48a567873 | ||
|
|
e69ccd8ea7 | ||
|
|
11924bb980 | ||
|
|
af89154e96 | ||
|
|
1485ea0831 | ||
|
|
e22bc777d8 | ||
|
|
043403fe23 | ||
|
|
1e1160906e | ||
|
|
f7d3e63063 | ||
|
|
473d39791b | ||
|
|
2114abb8c6 | ||
|
|
4fb4c26f55 | ||
|
|
2e8e574ea5 | ||
|
|
84c7e97be2 | ||
|
|
a6e7c99d55 | ||
|
|
ac3fa7f91f | ||
|
|
6eadad53b2 | ||
|
|
b11150f31f | ||
|
|
836cf60611 | ||
|
|
1c13ad95a5 | ||
|
|
1e8516e91d | ||
|
|
32c775311d | ||
|
|
28d0bb98de | ||
|
|
a9a9f3aeaa | ||
|
|
c2a0735975 | ||
|
|
41cb53f6c2 | ||
|
|
58552af8fd | ||
|
|
c7ab87b0cc | ||
|
|
11ecc5fdee | ||
|
|
19fb3eed9f | ||
|
|
b292b32374 | ||
|
|
63d1393bb0 | ||
|
|
37914cb062 | ||
|
|
ec40696854 | ||
|
|
2249f3d673 | ||
|
|
d2df324f29 | ||
|
|
67fdb0b659 | ||
|
|
e77bdf66f9 | ||
|
|
1b3b67779c | ||
|
|
6c7e386391 | ||
|
|
ba25b279d6 | ||
|
|
e7c83c19b6 | ||
|
|
7be7fb49a3 | ||
|
|
bcccb4cbb3 | ||
|
|
e9f1d951d3 | ||
|
|
e5632a9339 | ||
|
|
1510fb4fc0 | ||
|
|
64a1ad2649 | ||
|
|
4458ca1d24 | ||
|
|
21aaa48e62 | ||
|
|
e75c241030 | ||
|
|
60216048a8 | ||
|
|
f3c2e29fb4 | ||
|
|
ce99924be4 | ||
|
|
5de80a60d4 | ||
|
|
5753762350 | ||
|
|
885b318b04 | ||
|
|
7a22d58cf4 | ||
|
|
c8e4b462c9 | ||
|
|
30a3f42255 | ||
|
|
26ddb2de2f | ||
|
|
f60eeaa212 | ||
|
|
8cf72b36cb | ||
|
|
38c3bcef96 | ||
|
|
80604ba7b6 | ||
|
|
256c70c631 | ||
|
|
0e3532c529 | ||
|
|
9942fcfeb2 | ||
|
|
003c24ca6e | ||
|
|
ed120d014d | ||
|
|
e76a3d04f0 | ||
|
|
641d17007f | ||
|
|
9293b5f24a | ||
|
|
c1f3cbd1d4 | ||
|
|
78fa2ab65e | ||
|
|
56da2caeed | ||
|
|
a541d65255 | ||
|
|
a3d7e9eafe | ||
|
|
54933bea2a | ||
|
|
fcab9899cc | ||
|
|
be098e85db | ||
|
|
ed0ff46a87 | ||
|
|
7ae0d651d6 | ||
|
|
efd4432cfb | ||
|
|
24082b84f2 | ||
|
|
dcd5840341 | ||
|
|
9e705ce768 | ||
|
|
965466cc09 | ||
|
|
f3993f1775 | ||
|
|
e107902b14 | ||
|
|
e7b5ff49f4 | ||
|
|
e33172c44e | ||
|
|
3d858e8aa6 | ||
|
|
eab059c49a | ||
|
|
4aaff04fb3 | ||
|
|
cb364f3cab | ||
|
|
a9bfb090c3 | ||
|
|
c4ae4025f3 | ||
|
|
15067c678d | ||
|
|
5ae592f38e | ||
|
|
9cdbc56be3 | ||
|
|
86ed485711 | ||
|
|
7e1b4a4e90 | ||
|
|
4531d517da | ||
|
|
6fd5847f84 | ||
|
|
2015eba9b2 | ||
|
|
84f16ee895 | ||
|
|
5b2af03b16 | ||
|
|
b313395dc3 | ||
|
|
0d6bdbee10 | ||
|
|
248dac3a9d | ||
|
|
be49a54856 | ||
|
|
bd9ee0d646 | ||
|
|
442e0e582d | ||
|
|
38194c0cff | ||
|
|
0ebdaba03c | ||
|
|
ee82377d68 | ||
|
|
861588e4a3 | ||
|
|
1ab3bf2ef6 | ||
|
|
bb00d223c9 | ||
|
|
86fbfaddd1 | ||
|
|
5612bf513b | ||
|
|
87d0dc9e24 | ||
|
|
30fbcfbf71 | ||
|
|
5d90f4ea06 | ||
|
|
f6d09e1574 | ||
|
|
b8e48dee7f | ||
|
|
a6ccb9ec69 | ||
|
|
66551ebdf5 | ||
|
|
21534f7d83 | ||
|
|
d591f9e108 | ||
|
|
aa2589d3be | ||
|
|
9d6067fa78 | ||
|
|
027e54425a | ||
|
|
e268c73c41 | ||
|
|
d3c57e2da0 | ||
|
|
02eace5a16 | ||
|
|
15bc1dd999 | ||
|
|
b937956dc8 | ||
|
|
efbc0c8510 | ||
|
|
d0f227189c | ||
|
|
41eef5efc4 | ||
|
|
f00f9d9f1a | ||
|
|
ae59b3ba36 | ||
|
|
6668712f7b | ||
|
|
8812686b17 | ||
|
|
8b0f0b5bb4 | ||
|
|
f5e8a04e3b | ||
|
|
a298ce3b41 | ||
|
|
31daa889e8 | ||
|
|
76a058178e | ||
|
|
3304b18ac2 | ||
|
|
b95a6afe77 | ||
|
|
f6ed7d7582 | ||
|
|
cd3290df1c | ||
|
|
2296caf529 | ||
|
|
90ded6658d | ||
|
|
7e97fb80a5 | ||
|
|
b58471fdb1 | ||
|
|
46b4f9f29b | ||
|
|
ec20d72aba | ||
|
|
5743e2a99b | ||
|
|
2f429a2e76 | ||
|
|
3e982f7a4a | ||
|
|
89484e281d | ||
|
|
14a115f372 | ||
|
|
e96595fe59 | ||
|
|
f58d21862b | ||
|
|
aac24ad2d4 | ||
|
|
1df9575e20 | ||
|
|
64609fe80f | ||
|
|
533a54e111 | ||
|
|
b59c3eb470 | ||
|
|
0366fc35cb | ||
|
|
d86ff4b1ee | ||
|
|
f8040324e1 | ||
|
|
9c81acb159 | ||
|
|
65395b1112 | ||
|
|
d2696be03b | ||
|
|
2da4d420f9 | ||
|
|
a992f95c02 | ||
|
|
edd8e07df6 | ||
|
|
c813d43da0 | ||
|
|
c973445ab7 | ||
|
|
25f6ba76d6 | ||
|
|
8f47c569f9 | ||
|
|
c16801e524 | ||
|
|
dafcd0448f | ||
|
|
24a52375c7 | ||
|
|
5f9e95038e | ||
|
|
5cbb21afb2 | ||
|
|
119fab2996 | ||
|
|
38d354c4ed | ||
|
|
cdb1074e11 | ||
|
|
4b61fd2d7d | ||
|
|
5a0a5c120b | ||
|
|
d92926ae54 | ||
|
|
b34af5da24 | ||
|
|
5da1f86575 | ||
|
|
b0185e3539 | ||
|
|
7232da6ba1 | ||
|
|
9dff75cd44 | ||
|
|
6038860be0 | ||
|
|
4653de9f03 | ||
|
|
fef79651ef | ||
|
|
3d54ca0a7c | ||
|
|
199986815c | ||
|
|
0a3c00f68b | ||
|
|
3e2467eb71 | ||
|
|
c4cc476c3d | ||
|
|
cc6ff1ac54 | ||
|
|
b075502c4c | ||
|
|
35a99f92ab | ||
|
|
4fe0836cf9 | ||
|
|
8b7cc65ae6 | ||
|
|
4d495ba74f | ||
|
|
de5de0b162 | ||
|
|
311da30802 | ||
|
|
16819a5caa | ||
|
|
72a44c2fcd | ||
|
|
7783b20b91 | ||
|
|
962ccbc0d7 | ||
|
|
4d61c5d7b2 | ||
|
|
7ca4597ade | ||
|
|
f1a22728ab | ||
|
|
ca88fc849f | ||
|
|
ccd795445f | ||
|
|
1874269a48 | ||
|
|
8b20373a8e | ||
|
|
15dcb77a0c | ||
|
|
5d2fac9cd7 | ||
|
|
682b253760 | ||
|
|
f440de82e2 | ||
|
|
5e0e6822c7 | ||
|
|
2aadac7a4d | ||
|
|
1098394486 | ||
|
|
b90a34228f | ||
|
|
8bf8ebd34b | ||
|
|
673d88417c | ||
|
|
3a7b489208 | ||
|
|
7ae9eebc34 | ||
|
|
8f83ba5878 | ||
|
|
b8af3fa214 | ||
|
|
5ddec4f596 | ||
|
|
8f4b4f4941 | ||
|
|
953349f262 | ||
|
|
b52ae0e56b | ||
|
|
893b448534 | ||
|
|
973769b8bc | ||
|
|
c8fa9d34e1 | ||
|
|
3069deb92f | ||
|
|
68c9c01747 | ||
|
|
5e8f0baa12 | ||
|
|
8d1286cc00 | ||
|
|
bda4dd339a | ||
|
|
f2e3034d24 | ||
|
|
2626154a64 | ||
|
|
b770b2a419 | ||
|
|
158c34b0f9 | ||
|
|
d507c88d3e | ||
|
|
98f70b775f | ||
|
|
54f4b824e4 | ||
|
|
2aa5307f0a | ||
|
|
6c10d6ef8a | ||
|
|
89b36f2b25 | ||
|
|
79a6adbcf3 | ||
|
|
95f00a3c4b | ||
|
|
3f8373f76f | ||
|
|
23a9d3f4d7 | ||
|
|
333279f45a | ||
|
|
add5f51201 | ||
|
|
d1bedef5b3 | ||
|
|
54cf0116a8 | ||
|
|
6b252fb46e | ||
|
|
3e00a16f0f | ||
|
|
ecfd93544a | ||
|
|
3ec89e49bf | ||
|
|
8762506e9f | ||
|
|
7204bf9914 | ||
|
|
f62c262f23 | ||
|
|
10aa784809 | ||
|
|
904f5dc183 | ||
|
|
c61a5e7173 | ||
|
|
81b28beef5 | ||
|
|
0d34356678 | ||
|
|
5412840a93 | ||
|
|
137bbb3d2c | ||
|
|
5a40054ac2 | ||
|
|
be621fbc5c | ||
|
|
9ab4836601 | ||
|
|
4671102833 | ||
|
|
67401a275b | ||
|
|
c422588071 | ||
|
|
fb12fec899 | ||
|
|
c53c49558f | ||
|
|
1a26a2daa4 | ||
|
|
d8be1282b5 | ||
|
|
91bc5236b5 | ||
|
|
b278957111 | ||
|
|
1c80c739d6 | ||
|
|
700a94222b | ||
|
|
d5d2156689 | ||
|
|
8203ad08a8 | ||
|
|
31907b90f0 | ||
|
|
7b595f10ce | ||
|
|
4f93d331b7 | ||
|
|
32c6dccebe | ||
|
|
cbdc2b7d2d | ||
|
|
66a9dc70c7 | ||
|
|
846ca500d3 | ||
|
|
bd6afd445d | ||
|
|
0663bbc2fb | ||
|
|
8e7a951af8 | ||
|
|
ba1aeb8f7f | ||
|
|
f7c74cfa80 | ||
|
|
2e700c8576 | ||
|
|
f4626a4fc4 | ||
|
|
e0b40a330f |
2
.github/workflows/coverage.yaml
vendored
2
.github/workflows/coverage.yaml
vendored
@@ -33,7 +33,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain
|
||||
uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain --extra websocket
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: |
|
||||
|
||||
2
.github/workflows/tests.yaml
vendored
2
.github/workflows/tests.yaml
vendored
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain
|
||||
uv sync --group dev --extra anthropic --extra aws --extra google --extra langchain --extra websocket
|
||||
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@@ -51,4 +51,7 @@ docs/api/_build/
|
||||
docs/api/api
|
||||
|
||||
# uv
|
||||
.python-version
|
||||
.python-version
|
||||
|
||||
# Pipecat
|
||||
whisker_setup.py
|
||||
535
CHANGELOG.md
535
CHANGELOG.md
@@ -7,6 +7,541 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
<!-- towncrier release notes start -->
|
||||
|
||||
## [0.0.99] - 2026-01-13
|
||||
|
||||
### Added
|
||||
|
||||
- Introducing user turn strategies. User turn strategies indicate when the user
|
||||
turn starts or stops. In conversational agents, these are often referred to
|
||||
as start/stop speaking or turn-taking plans or policies.
|
||||
|
||||
User turn start strategies indicate when the user starts speaking (e.g.
|
||||
using VAD events or when a user says one or more words).
|
||||
|
||||
User turn stop strategies indicate when the user stops speaking (e.g. using
|
||||
an end-of-turn detection model or by observing incoming transcriptions).
|
||||
|
||||
A list of strategies can be specified for both strategies; strategies are
|
||||
evaluated in order until one evaluates to true.
|
||||
|
||||
Available user turn start strategies:
|
||||
|
||||
- VADUserTurnStartStrategy
|
||||
- TranscriptionUserTurnStartStrategy
|
||||
- MinWordsUserTurnStartStrategy
|
||||
- ExternalUserTurnStartStrategy
|
||||
|
||||
Available user turn stop strategies:
|
||||
|
||||
- TranscriptionUserTurnStopStrategy
|
||||
- TurnAnalyzerUserTurnStopStrategy
|
||||
- ExternalUserTurnStopStrategy
|
||||
|
||||
The default strategies are:
|
||||
|
||||
- start: [VADUserTurnStartStrategy, TranscriptionUserTurnStartStrategy]
|
||||
- stop: [TranscriptionUserTurnStopStrategy]
|
||||
|
||||
Turn strategies are configured when setting up `LLMContextAggregatorPair`.
|
||||
For example:
|
||||
|
||||
```python
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
stop=[
|
||||
TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams())
|
||||
)
|
||||
],
|
||||
)
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
In order to use the user turn strategies you must update to the new
|
||||
universal `LLMContext` and `LLMContextAggregatorPair`.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- Added `RNNoiseFilter` for real-time noise suppression using RNNoise neural
|
||||
network via pyrnnoise library.
|
||||
(PR [#3205](https://github.com/pipecat-ai/pipecat/pull/3205))
|
||||
|
||||
- Added `GrokRealtimeLLMService` for xAI's Grok Voice Agent API with real-time
|
||||
voice conversations:
|
||||
|
||||
- Support for real-time audio streaming with WebSocket connection
|
||||
- Built-in server-side VAD (Voice Activity Detection)
|
||||
- Multiple voice options: Ara, Rex, Sal, Eve, Leo
|
||||
- Built-in tools support: web_search, x_search, file_search
|
||||
- Custom function calling with standard Pipecat tools schema
|
||||
- Configurable audio formats (PCM at 8kHz-48kHz)
|
||||
(PR [#3267](https://github.com/pipecat-ai/pipecat/pull/3267))
|
||||
|
||||
- Added an approximation of TTFB for Ultravox.
|
||||
(PR [#3268](https://github.com/pipecat-ai/pipecat/pull/3268))
|
||||
|
||||
- Added a new `AudioContextTTSService` to the TTS service base classes. The
|
||||
`AudioContextWordTTSService` now inherits from `AudioContextTTSService` and
|
||||
`WebsocketWordTTSService`.
|
||||
(PR [#3289](https://github.com/pipecat-ai/pipecat/pull/3289))
|
||||
|
||||
- `LLMUserAggregator` now exposes the following events:
|
||||
|
||||
- `on_user_turn_started`: triggered when a user turn starts
|
||||
- `on_user_turn_stopped`: triggered when a user turn ends
|
||||
- `on_user_turn_stop_timeout`: triggered when a user turn does not stop
|
||||
and times out
|
||||
(PR [#3291](https://github.com/pipecat-ai/pipecat/pull/3291))
|
||||
|
||||
- Introducing user mute strategies. User mute strategies indicate when user
|
||||
input should be muted based on the current system state.
|
||||
|
||||
In conversational agents, user mute strategies are used to prevent user
|
||||
input from interrupting bot speech, tool execution, or other critical system
|
||||
operations.
|
||||
|
||||
A list of strategies can be specified; all strategies are evaluated for
|
||||
every frame so that each strategy can maintain its internal state. A user
|
||||
frame is muted if any of the configured strategies indicates it should be
|
||||
muted.
|
||||
|
||||
Available user mute strategies:
|
||||
|
||||
- `FirstSpeechUserMuteStrategy`
|
||||
- `MuteUntilFirstBotCompleteUserMuteStrategy`
|
||||
- `AlwaysUserMuteStrategy`
|
||||
- `FunctionCallUserMuteStrategy`
|
||||
|
||||
User mute strategies replace the legacy `STTMuteFilter` and provide a more
|
||||
flexible and composable approach to muting user input.
|
||||
|
||||
User mute strategies are configured when setting up the
|
||||
`LLMContextAggregatorPair`. For example:
|
||||
|
||||
```python
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_mute_strategies=[
|
||||
FirstSpeechUserMuteStrategy(),
|
||||
]
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
In order to use user mute strategies you should update to the new universal
|
||||
`LLMContext` and `LLMContextAggregatorPair`.
|
||||
(PR [#3292](https://github.com/pipecat-ai/pipecat/pull/3292))
|
||||
|
||||
- Added `use_ssl` parameter to `NvidiaSTTService`, `NvidiaSegmentedSTTService`
|
||||
and `NvidiaTTSService`.
|
||||
(PR [#3300](https://github.com/pipecat-ai/pipecat/pull/3300))
|
||||
|
||||
- Added `enable_interruptions` constructor argument to all user turn
|
||||
strategies. This tells the `LLMUserAggregator` to push or not push an
|
||||
`InterruptionFrame`.
|
||||
(PR [#3316](https://github.com/pipecat-ai/pipecat/pull/3316))
|
||||
|
||||
- Added `split_sentences` parameter to `SpeechmaticsSTTService` to control
|
||||
sentence splitting behavior for finals on sentence boundaries.
|
||||
(PR [#3328](https://github.com/pipecat-ai/pipecat/pull/3328))
|
||||
|
||||
- Added word-level timestamp support to `AzureTTSService` for accurate
|
||||
text-to-audio synchronization.
|
||||
(PR [#3334](https://github.com/pipecat-ai/pipecat/pull/3334))
|
||||
|
||||
- Added `pronunciation_dict_id` parameter to `CartesiaTTSService.InputParams`
|
||||
and `CartesiaHttpTTSService.InputParams` to support Cartesia's pronunciation
|
||||
dictionary feature for custom pronunciations.
|
||||
(PR [#3346](https://github.com/pipecat-ai/pipecat/pull/3346))
|
||||
|
||||
- Added support for using the HeyGen LiveAvatar API with the `HeyGenTransport`
|
||||
(see https://www.liveavatar.com/).
|
||||
(PR [#3357](https://github.com/pipecat-ai/pipecat/pull/3357))
|
||||
|
||||
- Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`:
|
||||
|
||||
- New `start_video_paused` parameter to control initial video input state
|
||||
- New `video_frame_detail` parameter to set image processing quality
|
||||
("auto",
|
||||
"low", or "high"). This corresponds to OpenAI Realtime's `image_detail`
|
||||
parameter.
|
||||
- `set_video_input_paused()` method to pause/resume video input at runtime
|
||||
- `set_video_frame_detail()` method to adjust video frame quality
|
||||
dynamically
|
||||
- Automatic rate limiting (1 frame per second) to prevent API overload
|
||||
(PR [#3360](https://github.com/pipecat-ai/pipecat/pull/3360))
|
||||
|
||||
- Added `UserTurnProcessor`, a frame processor built on `UserTurnController`
|
||||
that pushes `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` frames
|
||||
and interruptions based on the controller's user turn strategies.
|
||||
(PR [#3372](https://github.com/pipecat-ai/pipecat/pull/3372))
|
||||
|
||||
- Added `UserTurnController` to manage user turns. It emits
|
||||
`on_user_turn_started`, `on_user_turn_stopped`, and
|
||||
`on_user_turn_stop_timeout` events, and can be integrated into processors to
|
||||
detect and handle user turns. `LLMUserAggregator` and `UserTurnProcessor` are
|
||||
implemented using this controller.
|
||||
(PR [#3372](https://github.com/pipecat-ai/pipecat/pull/3372))
|
||||
|
||||
- Added `should_interrupt` property to `DeepgramFluxSTTService`,
|
||||
`DeepgramSTTService`, and `SpeechmaticsSTTService` to configure whether the
|
||||
bot should be interrupted when the external service detects user speech.
|
||||
(PR [#3374](https://github.com/pipecat-ai/pipecat/pull/3374))
|
||||
|
||||
- `LLMAssistantAggregator` now exposes the following events:
|
||||
|
||||
- `on_assistant_turn_started`: triggered when the assistant turn starts
|
||||
- `on_assistant_turn_stopped`: triggered when the assistant turn ends
|
||||
- `on_assistant_thought`: triggered when there's an assistant thought
|
||||
available
|
||||
(PR [#3385](https://github.com/pipecat-ai/pipecat/pull/3385))
|
||||
|
||||
- Added `KrispVivaTurn` analyzer for end of turn detection using the Krisp VIVA
|
||||
SDK (requires `krisp_audio`).
|
||||
(PR [#3391](https://github.com/pipecat-ai/pipecat/pull/3391))
|
||||
|
||||
- Added support for setting up a pipeline task from external files. You can now
|
||||
register custom pipeline task setup files by setting the
|
||||
`PIPECAT_SETUP_FILES` environment variable. This variable should contain a
|
||||
colon-separated list of Python files (e.g. `export
|
||||
PIPECAT_SETUP_FILES="setup1.py:setup.py:..."`). Each file must define a
|
||||
function with the following signature:
|
||||
|
||||
```python
|
||||
async def setup_pipeline_task(task: PipelineTask):
|
||||
...
|
||||
```
|
||||
|
||||
(PR [#3397](https://github.com/pipecat-ai/pipecat/pull/3397))
|
||||
|
||||
- Added a keepalive task for `InworldTTSService` to keep the service connected
|
||||
in the event of no generations for longer periods of time.
|
||||
(PR [#3403](https://github.com/pipecat-ai/pipecat/pull/3403))
|
||||
|
||||
- Added `enable_vad` to `Params` for use in the `GladiaSTTService`. When
|
||||
enabled, `GladiaSTTService` acts as the turn controller, emitting
|
||||
`UserStartedSpeakingFrame`, `UserStoppedSpeakingFrame`, and optionally
|
||||
`InterruptionFrame`.
|
||||
(PR [#3404](https://github.com/pipecat-ai/pipecat/pull/3404))
|
||||
|
||||
- Added `should_interrupt` property to `GladiaSTTService` to configure whether
|
||||
the bot should be interrupted when the external service detects user speech.
|
||||
(PR [#3404](https://github.com/pipecat-ai/pipecat/pull/3404))
|
||||
|
||||
- Added `VonageFrameSerializer` for the Vonage Video API Audio Connector
|
||||
WebSocket protocol.
|
||||
(PR [#3410](https://github.com/pipecat-ai/pipecat/pull/3410))
|
||||
|
||||
- Added `append_trailing_space` parameter to `TTSService` to automatically
|
||||
append a trailing space to text before sending to TTS, helping prevent some
|
||||
services from vocalizing trailing punctuation.
|
||||
(PR [#3424](https://github.com/pipecat-ai/pipecat/pull/3424))
|
||||
|
||||
### Changed
|
||||
|
||||
- Updated `ElevenLabsRealtimeSTTService` to accept the
|
||||
`include_language_detection` parameter to detect language.
|
||||
|
||||
```python
|
||||
stt = ElevenLabsRealtimeSTTService(
|
||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||
include_language_detection=True
|
||||
)
|
||||
```
|
||||
|
||||
(PR [#3216](https://github.com/pipecat-ai/pipecat/pull/3216))
|
||||
|
||||
- Updated `SpeechmaticsSTTService` to use new Python Voice SDK with improved
|
||||
VAD, Smart Turn capabilities, and brings dramatic improvements to latency
|
||||
without any impact on accuracy. Use the `turn_detection_mode` parameter to control
|
||||
the endpointing of speech, with `TurnDetectionMode.EXTERNAL` (default),
|
||||
`TurnDetectionMode.ADAPTIVE`, or `TurnDetectionMode.SMART_TURN`.
|
||||
|
||||
```python
|
||||
stt = SpeechmaticsSTTService(
|
||||
api_key=os.getenv("SPEECHMATICS_API_KEY"),
|
||||
params=SpeechmaticsSTTService.InputParams(
|
||||
language=Language.EN,
|
||||
turn_detection_mode=SpeechmaticsSTTService.TurnDetectionMode.ADAPTIVE,
|
||||
speaker_active_format="<{speaker_id}>{text}</{speaker_id}>",
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
(PR [#3225](https://github.com/pipecat-ai/pipecat/pull/3225))
|
||||
|
||||
- `daily-python` updated to 0.23.0.
|
||||
(PR [#3257](https://github.com/pipecat-ai/pipecat/pull/3257))
|
||||
|
||||
- `TranscriptionFrame` and `InterimTranscriptionFrame` produced by
|
||||
`DailyTransport` now include the transport source (i.e., the originating
|
||||
audio track).
|
||||
(PR [#3257](https://github.com/pipecat-ai/pipecat/pull/3257))
|
||||
|
||||
- Updates to Inworld TTS services:
|
||||
|
||||
- Improved `InworldTTSService`'s websocket implementation to better flush
|
||||
and close context to better handle long inputs.
|
||||
- Improved docstrings for `InworldTTSService` and `InworldHttpTTSService`.
|
||||
(PR [#3288](https://github.com/pipecat-ai/pipecat/pull/3288))
|
||||
|
||||
- Improved the error handling and reconnection logic for `WebsocketServer` by
|
||||
distinguishing between errors when disconnecting and websocket communication
|
||||
errors.
|
||||
(PR [#3392](https://github.com/pipecat-ai/pipecat/pull/3392))
|
||||
|
||||
- Updated `DeepgramSTTService` to push user started/stopped speaking and
|
||||
interruption frames when `vad_enabled` is set to true. This centralizes the
|
||||
frames into the service, removing the need to have your application code
|
||||
handle Deepgram's events and push these frames.
|
||||
(PR [#3314](https://github.com/pipecat-ai/pipecat/pull/3314))
|
||||
|
||||
- Added encoding validation to `DeepgramTTSService` to prevent unsupported
|
||||
encodings from reaching the API. The service now raises `ValueError` at
|
||||
initialization with a clear error message.
|
||||
(PR [#3329](https://github.com/pipecat-ai/pipecat/pull/3329))
|
||||
|
||||
- Updated `read_audio_frame` & `read_video_frame` methods in
|
||||
`SmallWebRTCClient` to check if the track is enabled before logging a
|
||||
warning.
|
||||
(PR [#3336](https://github.com/pipecat-ai/pipecat/pull/3336))
|
||||
|
||||
- Updated `CartesiaTTSService` to support setting `language=None`, resulting in
|
||||
Cartesia auto-detecting the language of the conversation.
|
||||
(PR [#3366](https://github.com/pipecat-ai/pipecat/pull/3366))
|
||||
|
||||
- The bundled Smart Turn weights are now updated to v3.2, which has better
|
||||
handling of short utterances, and is more robust against background noise.
|
||||
(PR [#3367](https://github.com/pipecat-ai/pipecat/pull/3367))
|
||||
|
||||
- Updated `SpeechmaticsSTTService` dependency to `speechmatics-voice[smart]>=0.2.6`
|
||||
(PR [#3371](https://github.com/pipecat-ai/pipecat/pull/3371))
|
||||
|
||||
- Smart Turn now takes into account `vad_start_seconds` when buffering audio,
|
||||
meaning that the start of the turn audio is not cut off. This improves
|
||||
accuracy for short utterances.
|
||||
|
||||
- The default value of `pre_speech_ms` is now set to 500ms for Smart Turn.
|
||||
(PR [#3377](https://github.com/pipecat-ai/pipecat/pull/3377))
|
||||
|
||||
- Improved Krisp SDK management to allow `KrispVivaTurn` and `KrispVivaFilter`
|
||||
to share a single SDK instance within the same process.
|
||||
(PR [#3391](https://github.com/pipecat-ai/pipecat/pull/3391))
|
||||
|
||||
- Updated default model for `GroqTTSService` to `canopylabs/orpheus-v1-english`
|
||||
and voice ID to `autumn`.
|
||||
(PR [#3399](https://github.com/pipecat-ai/pipecat/pull/3399))
|
||||
|
||||
- Enhanced `FastAPIWebsocketTransport` with optional protocol-level audio
|
||||
packetization via the `fixed_audio_packet_size` parameter to support media
|
||||
endpoints requiring strict framing and real-time pacing.
|
||||
(PR [#3410](https://github.com/pipecat-ai/pipecat/pull/3410))
|
||||
|
||||
- `DeepgramTTSService` and `RimeTTSService` now set `append_trailing_space` to
|
||||
`True` to prevent punctuation (e.g., “dot”) from being pronounced.
|
||||
(PR [#3424](https://github.com/pipecat-ai/pipecat/pull/3424))
|
||||
|
||||
- Updated `GeminiLiveLLMService` to push `LLMThoughtStartFrame`,
|
||||
`LLMThoughtTextFrame`, and `LLMThoughtEndFrame` when the model returns
|
||||
thought content.
|
||||
(PR [#3431](https://github.com/pipecat-ai/pipecat/pull/3431))
|
||||
|
||||
### Deprecated
|
||||
|
||||
- `pipecat.audio.interruptions.MinWordsInterruptionStrategy` is deprecated. Use
|
||||
`pipecat.turns.user_start.MinWordsUserTurnStartStrategy` with
|
||||
`LLMUserAggregator`'s new `user_turn_strategies` parameter instead.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- `FrameProcessor.interruption_strategies` is deprecated, use
|
||||
`LLMUserAggregator`'s new `user_turn_strategies` parameter instead.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- The `LLMUserAggregatorParams` and `LLMAssistantAggregatorParams` classes in
|
||||
`pipecat.processors.aggregators.llm_response` are now deprecated. Use the new
|
||||
universal `LLMContext` and `LLMContextAggregatorPair` instead.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- Deprecated the `emulated` field in the `UserStartedSpeakingFrame` and
|
||||
`UserStoppedSpeakingFrame` frames.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- `EmulateUserStartedSpeakingFrame` and `EmulateUserStoppedSpeakingFrame`
|
||||
frames are deprecated.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- ⚠️ `TransportParams.turn_analyzer` is deprecated and might result in
|
||||
unexpected behavior, use `LLMUserAggregator`'s new `user_turn_strategies`
|
||||
parameter instead.
|
||||
(PR [#3045](https://github.com/pipecat-ai/pipecat/pull/3045))
|
||||
|
||||
- For `SpeechmaticsSTTService`, the `end_of_utterance_mode` parameter is
|
||||
deprecated. Use the new `turn_detection_mode` parameter instead, with
|
||||
`TurnDetectionMode.EXTERNAL`,`TurnDetectionMode.ADAPTIVE`, or
|
||||
`TurnDetectionMode.SMART_TURN`. The `enable_vad` parameter is also
|
||||
deprecated and is inferred from the `turn_detection_mode`.
|
||||
(PR [#3225](https://github.com/pipecat-ai/pipecat/pull/3225))
|
||||
|
||||
- `OpenAILLMContext` and its associated things (context aggregators, etc.) are
|
||||
now deprecated in favor of the universal `LLMContext` and its associated
|
||||
things.
|
||||
|
||||
From the developer's point of view, switching to using `LLMContext`
|
||||
machinery will usually be a matter of going from this:
|
||||
|
||||
```python
|
||||
context = OpenAILLMContext(messages, tools)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
```
|
||||
|
||||
To this:
|
||||
|
||||
```
|
||||
context = LLMContext(messages, tools)
|
||||
context_aggregator = LLMContextAggregatorPair(context)
|
||||
```
|
||||
|
||||
(PR [#3263](https://github.com/pipecat-ai/pipecat/pull/3263))
|
||||
|
||||
- `STTMuteFilter` is deprecated and will be removed in a future version. Use
|
||||
`LLMUserAggregator`'s new `user_mute_strategies` instead.
|
||||
(PR [#3292](https://github.com/pipecat-ai/pipecat/pull/3292))
|
||||
|
||||
- `FrameProcessor.interruptions_allowed` is now deprecated, use
|
||||
`LLMUserAggregator`'s new parameter `user_mute_strategies` instead.
|
||||
(PR [#3297](https://github.com/pipecat-ai/pipecat/pull/3297))
|
||||
|
||||
- `PipelineParams.allow_interruptions` is now deprecated, use
|
||||
`LLMUserAggregator`'s new parameter `user_turn_strategies` instead. For
|
||||
example, to disable interruptions but still get user turns you can do:
|
||||
|
||||
```python
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
start=[TranscriptionUserTurnStartStrategy(enable_interruptions=False)],
|
||||
),
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
(PR [#3297](https://github.com/pipecat-ai/pipecat/pull/3297))
|
||||
|
||||
- `TranscriptProcessor` and related data classes and frames
|
||||
(`TranscriptionMessage`, `ThoughtTranscriptionMessage`,
|
||||
`TranscriptionUpdateFrame`) are deprecated. Use `LLMUserAggregator`'s and
|
||||
`LLMAssistantAggregator`'s new events (`on_user_turn_stopped` and
|
||||
`on_assistant_turn_stopped`) instead.
|
||||
(PR [#3385](https://github.com/pipecat-ai/pipecat/pull/3385))
|
||||
|
||||
- Deprecated support for the `vad_events` `LiveOptions` in
|
||||
`DeepgramSTTService`. Instead, use a local Silero VAD for VAD events.
|
||||
Additionally, deprecated `should_interrupt` which will be removed along with
|
||||
`vad_events` support in a future release.
|
||||
(PR [#3386](https://github.com/pipecat-ai/pipecat/pull/3386))
|
||||
|
||||
- Loading external observers from files is deprecated, use the new pipeline
|
||||
task setup files and `PIPECAT_SETUP_FILES` environment variable instead.
|
||||
(PR [#3397](https://github.com/pipecat-ai/pipecat/pull/3397))
|
||||
|
||||
### Fixed
|
||||
|
||||
- Improved error handling in `ElevenLabsRealtimeSTTService`
|
||||
(PR [#3233](https://github.com/pipecat-ai/pipecat/pull/3233))
|
||||
|
||||
- Fixed an issue in `ElevenLabsRealtimeSTTService` causing an infinite loop
|
||||
that blocks the process if the websocket disconnects due to an error
|
||||
(PR [#3233](https://github.com/pipecat-ai/pipecat/pull/3233))
|
||||
|
||||
- Fixed a bug in `STTMuteFilter` where the user was not always muted during
|
||||
function calls, especially when there were multiple simultaneous calls.
|
||||
(PR [#3292](https://github.com/pipecat-ai/pipecat/pull/3292))
|
||||
|
||||
- Fixed a `RNNoiseFilter` issue that would cause a "[Errno 12] Cannot allocate
|
||||
memory" error when processing silence audio frames.
|
||||
(PR [#3322](https://github.com/pipecat-ai/pipecat/pull/3322))
|
||||
|
||||
- Updated `SpeechmaticsSTTService` for version `0.0.99+`:
|
||||
|
||||
- Fixed `SpeechmaticsSTTService` to listen for `VADUserStoppedSpeakingFrame`
|
||||
in order to finalize transcription.
|
||||
- Default to `TurnDetectionMode.FIXED` for Pipecat-controlled end of turn
|
||||
detection.
|
||||
- Only emit VAD + interruption frames if VAD is enabled within the plugin
|
||||
(modes other than `TurnDetectionMode.FIXED` or `TurnDetectionMode.EXTERNAL`).
|
||||
(PR [#3328](https://github.com/pipecat-ai/pipecat/pull/3328))
|
||||
|
||||
- Fixed an issue with function calling where a handler failing to invoke its
|
||||
result callback could leave the context stuck in IN_PROGRESS, causing LLM
|
||||
inference for subsequent function call results to block while waiting on the
|
||||
unresolved call.
|
||||
(PR [#3343](https://github.com/pipecat-ai/pipecat/pull/3343))
|
||||
|
||||
- Fixed an issue with DeepgramTTSService where the model would output "Dot"
|
||||
instead of a period in some circumstances.
|
||||
(PR [#3345](https://github.com/pipecat-ai/pipecat/pull/3345))
|
||||
|
||||
- Fixed an issue in `traced_stt` where `model_name` in OpenTelemetry appears as
|
||||
`unknown`.
|
||||
(PR [#3351](https://github.com/pipecat-ai/pipecat/pull/3351))
|
||||
|
||||
- Fixed an issue in GeminiLiveLLMService where TranscriptionFrames were
|
||||
occasionally not pushed.
|
||||
(PR [#3356](https://github.com/pipecat-ai/pipecat/pull/3356))
|
||||
|
||||
- Fixed potential memory leaks and initialization issues in `KrispVivaFilter`
|
||||
by improving SDK lifecycle management.
|
||||
(PR [#3391](https://github.com/pipecat-ai/pipecat/pull/3391))
|
||||
|
||||
- Fixed timing issue in `BaseOutputTransport` where the bot speaking flag was
|
||||
set after awaiting, allowing the event loop to re-enter the method before the
|
||||
guard was set.
|
||||
(PR [#3400](https://github.com/pipecat-ai/pipecat/pull/3400))
|
||||
|
||||
- Fixed parallel function calling when using Gemini thinking.
|
||||
(PR [3420](https://github.com/pipecat-ai/pipecat/pull/3420))
|
||||
|
||||
- Fixed an issue in `traced_llm` where `model_name` in OpenTelemetry appears as
|
||||
`unknown`.
|
||||
(PR [#3422](https://github.com/pipecat-ai/pipecat/pull/3422))
|
||||
|
||||
- Fixed an issue in `traced_tts`, `traced_gemini_live`, and
|
||||
`traced_openai_realtime` where `model_name` in OpenTelemetry appears as
|
||||
`unknown`.
|
||||
(PR [#3428](https://github.com/pipecat-ai/pipecat/pull/3428))
|
||||
|
||||
- Fixed `request_image_frame` (for backwards compatibility) and restored
|
||||
function-call–related fields in `UserImageRequestFrame` and
|
||||
`UserImageRawFrame`, preventing a case where adding a non-LLM message to the
|
||||
context could trigger duplicate LLM inferences (on image arrival and on
|
||||
function-call result), potentially causing an infinite inference loop.
|
||||
(PR [#3430](https://github.com/pipecat-ai/pipecat/pull/3430))
|
||||
|
||||
- Fixed `LLMContext.create_audio_message()` by correcting an internal helper
|
||||
that was incorrectly declared async while being run in `asyncio.to_thread()`.
|
||||
(PR [#3435](https://github.com/pipecat-ai/pipecat/pull/3435))
|
||||
|
||||
### Other
|
||||
|
||||
- Added `52-live-transcription.py` foundational example demonstrating live
|
||||
transcription and translation from English to Spanish. In this example, the
|
||||
bot is not interruptible: as the user continues speaking, English
|
||||
transcriptions are queued, and the bot continuously translates and speaks
|
||||
each queued sentence in Spanish without being interrupted by new user speech.
|
||||
(PR [#3316](https://github.com/pipecat-ai/pipecat/pull/3316))
|
||||
|
||||
- Added a new foundational example `53-concurrent-llm-evaluation.py` that shows
|
||||
how to use `UserTurnProcessor`.
|
||||
(PR [#3372](https://github.com/pipecat-ai/pipecat/pull/3372))
|
||||
|
||||
- Added a new foundational example `28-user-assistant-turns.py` that shows how
|
||||
to use the new `LLMUserAggregator` and `LLMAssistantAggregator` events to
|
||||
gather a conversation transcript.
|
||||
(PR [#3385](https://github.com/pipecat-ai/pipecat/pull/3385))
|
||||
|
||||
## [0.0.98] - 2025-12-17
|
||||
|
||||
### Added
|
||||
|
||||
2
LICENSE
2
LICENSE
@@ -1,6 +1,6 @@
|
||||
BSD 2-Clause License
|
||||
|
||||
Copyright (c) 2024–2025, Daily
|
||||
Copyright (c) 2024–2026, Daily
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
@@ -73,12 +73,12 @@ Catch new features, interviews, and how-tos on our [Pipecat TV](https://www.yout
|
||||
|
||||
| Category | Services |
|
||||
| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
|
||||
| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/stt/elevenlabs), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Gradium](https://docs.pipecat.ai/server/services/stt/gradium), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [Hathora](https://docs.pipecat.ai/server/services/stt/hathora), [NVIDIA Riva](https://docs.pipecat.ai/server/services/stt/riva), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [SambaNova (Whisper)](https://docs.pipecat.ai/server/services/stt/sambanova), [Sarvam](https://docs.pipecat.ai/server/services/stt/sarvam), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Speechmatics](https://docs.pipecat.ai/server/services/stt/speechmatics), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) |
|
||||
| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [Mistral](https://docs.pipecat.ai/server/services/llm/mistral), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [SambaNova](https://docs.pipecat.ai/server/services/llm/sambanova) [Together AI](https://docs.pipecat.ai/server/services/llm/together) |
|
||||
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
|
||||
| Text-to-Speech | [Async](https://docs.pipecat.ai/server/services/tts/asyncai), [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Camb AI](https://docs.pipecat.ai/server/services/tts/camb), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [Gradium](https://docs.pipecat.ai/server/services/tts/gradium), [Groq](https://docs.pipecat.ai/server/services/tts/groq), [Hathora](https://docs.pipecat.ai/server/services/tts/hathora), [Hume](https://docs.pipecat.ai/server/services/tts/hume), [Inworld](https://docs.pipecat.ai/server/services/tts/inworld), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [NVIDIA Riva](https://docs.pipecat.ai/server/services/tts/riva), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [Speechmatics](https://docs.pipecat.ai/server/services/tts/speechmatics), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) |
|
||||
| Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [Grok Voice Agent](https://docs.pipecat.ai/server/services/s2s/grok), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai), [Ultravox](https://docs.pipecat.ai/server/services/s2s/ultravox), |
|
||||
| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [SmallWebRTCTransport](https://docs.pipecat.ai/server/services/transport/small-webrtc), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local |
|
||||
| Serializers | [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx) |
|
||||
| Serializers | [Exotel](https://docs.pipecat.ai/server/utilities/serializers/exotel), [Plivo](https://docs.pipecat.ai/server/utilities/serializers/plivo), [Twilio](https://docs.pipecat.ai/server/utilities/serializers/twilio), [Telnyx](https://docs.pipecat.ai/server/utilities/serializers/telnyx), [Vonage](https://docs.pipecat.ai/server/utilities/serializers/vonage) |
|
||||
| Video | [HeyGen](https://docs.pipecat.ai/server/services/video/heygen), [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) |
|
||||
| Memory | [mem0](https://docs.pipecat.ai/server/services/memory/mem0) |
|
||||
| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) |
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
- Introducing user turn strategies. User turn strategies indicate when the user turn starts or stops. In conversational agents, these are often referred to as start/stop speaking or turn-taking plans or policies.
|
||||
|
||||
User turn start strategies indicate when the user starts speaking (e.g. using VAD events or when a user says one or more words).
|
||||
|
||||
User turn stop strategies indicate when the user stops speaking (e.g. using an end-of-turn detection model or by observing incoming transcriptions).
|
||||
|
||||
A list of strategies can be specified for both strategies; strategies are evaluated in order until one evaluates to true.
|
||||
|
||||
Available user turn start strategies:
|
||||
- VADUserTurnStartStrategy
|
||||
- TranscriptionUserTurnStartStrategy
|
||||
- MinWordsUserTurnStartStrategy
|
||||
- ExternalUserTurnStartStrategy
|
||||
|
||||
Available user turn stop strategies:
|
||||
- TranscriptionUserTurnStopStrategy
|
||||
- TurnAnalyzerUserTurnStopStrategy
|
||||
- ExternalUserTurnStopStrategy
|
||||
|
||||
The default strategies are:
|
||||
|
||||
- start: [VADUserTurnStartStrategy, TranscriptionUserTurnStartStrategy]
|
||||
- stop: [TranscriptionUserTurnStopStrategy]
|
||||
|
||||
Turn strategies are configured when setting up `LLMContextAggregatorPair`. For example:
|
||||
|
||||
```python
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
stop=[
|
||||
TurnAnalyzerUserTurnStopStrategy(
|
||||
turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams())
|
||||
)
|
||||
],
|
||||
)
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
In order to use the user turn strategies you must update to the new universal `LLMContext` and `LLMContextAggregatorPair`.
|
||||
@@ -1 +0,0 @@
|
||||
- ⚠️ `TransportParams.turn_analyzer` is deprecated and might result in unexpected behavior, use `LLMUserAggregator`'s new `turn_start_strategies` parameter instead.
|
||||
@@ -1 +0,0 @@
|
||||
- `FrameProcessor.interruption_strategies` is deprecated, use `LLMUserAggregator`'s new `turn_start_strategies` parameter instead.
|
||||
@@ -1 +0,0 @@
|
||||
- `EmulateUserStartedSpeakingFrame` and `EmulateUserStoppedSpeakingFrame` frames are deprecated.
|
||||
@@ -1 +0,0 @@
|
||||
- Deprecated the `emulated` field in the `UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame` frames.
|
||||
@@ -1 +0,0 @@
|
||||
- The `LLMUserAggregatorParams` and `LLMAssistantAggregatorParams` classes in `pipecat.processors.aggregators.llm_response` are now deprecated. Use the new universal `LLMContext` and `LLMContextAggregatorPair` instead.
|
||||
@@ -1 +0,0 @@
|
||||
- `pipecat.audio.interruptions.MinWordsInterruptionStrategy` is deprecated. Use `pipecat.turns.user_start.MinWordsUserTurnStartStrategy` with `LLMUserAggregator`'s new `turn_start_strategies` parameter instead.
|
||||
1
changelog/3169.added.md
Normal file
1
changelog/3169.added.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added Hathora service to support Hathora-hosted TTS and STT models (only non-streaming)
|
||||
@@ -1 +0,0 @@
|
||||
- Added `RNNoiseFilter` for real-time noise suppression using RNNoise neural network via pyrnnoise library.
|
||||
@@ -1,15 +0,0 @@
|
||||
- Updated `SpeechmaticsSTTService` to use new Python Voice SDK with improved VAD,
|
||||
Smart Turn capabilities, and brings dramatic improvements to latency without
|
||||
any impact on accuracy. Use the `turn_detection_mode` parameter to control the
|
||||
endpointing of speech, with `TurnDetectionMode.EXTERNAL` (default),
|
||||
`TurnDetectionMode.ADAPTIVE`, or `TurnDetectionMode.SMART_TURN`.
|
||||
```python
|
||||
stt = SpeechmaticsSTTService(
|
||||
api_key=os.getenv("SPEECHMATICS_API_KEY"),
|
||||
params=SpeechmaticsSTTService.InputParams(
|
||||
language=Language.EN,
|
||||
turn_detection_mode=SpeechmaticsSTTService.TurnDetectionMode.ADAPTIVE,
|
||||
speaker_active_format="<{speaker_id}>{text}</{speaker_id}>",
|
||||
),
|
||||
)
|
||||
```
|
||||
@@ -1,4 +0,0 @@
|
||||
- For `SpeechmaticsSTTService`, the `end_of_utterance_mode` parameter is deprecated.
|
||||
Use the new `turn_detection_mode` parameter instead, with `TurnDetectionMode.EXTERNAL`,
|
||||
`TurnDetectionMode.ADAPTIVE`, or `TurnDetectionMode.SMART_TURN`. The `enable_vad`
|
||||
parameter is also deprecated and is inferred from the `turn_detection_mode`.
|
||||
@@ -1,2 +0,0 @@
|
||||
- Improved error handling in `ElevenLabsRealtimeSTTService`
|
||||
- Fixed an issue in `ElevenLabsRealtimeSTTService` causing an infinite loop that blocks the process if the websocket disconnects due to an error
|
||||
@@ -1 +0,0 @@
|
||||
- `TranscriptionFrame` and `InterimTranscriptionFrame` produced by `DailyTransport` now include the transport source (i.e., the originating audio track).
|
||||
@@ -1 +0,0 @@
|
||||
- `daily-python` updated to 0.23.0.
|
||||
@@ -1,15 +0,0 @@
|
||||
- `OpenAILLMContext` and its associated things (context aggregators, etc.) are now deprecated in favor of the universal `LLMContext` and its associated things.
|
||||
|
||||
From the developer's point of view, switching to using `LLMContext` machinery will usually be a matter of going from this:
|
||||
|
||||
```python
|
||||
context = OpenAILLMContext(messages, tools)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
```
|
||||
|
||||
To this:
|
||||
|
||||
```
|
||||
context = LLMContext(messages, tools)
|
||||
context_aggregator = LLMContextAggregatorPair(context)
|
||||
```
|
||||
@@ -1,8 +0,0 @@
|
||||
- Added `GrokRealtimeLLMService` for xAI's Grok Voice Agent API with real-time voice conversations:
|
||||
|
||||
- Support for real-time audio streaming with WebSocket connection
|
||||
- Built-in server-side VAD (Voice Activity Detection)
|
||||
- Multiple voice options: Ara, Rex, Sal, Eve, Leo
|
||||
- Built-in tools support: web_search, x_search, file_search
|
||||
- Custom function calling with standard Pipecat tools schema
|
||||
- Configurable audio formats (PCM at 8kHz-48kHz)
|
||||
@@ -1 +0,0 @@
|
||||
- Added an approximation of TTFB for Ultravox.
|
||||
1
changelog/3287.changed.md
Normal file
1
changelog/3287.changed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Enhanced interruption handling in `AsyncAITTSService` by supporting multi-context WebSocket sessions for more robust context management.
|
||||
1
changelog/3287.fixed.md
Normal file
1
changelog/3287.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Corrected TTFB metric calculation in `AsyncAIHttpTTSService`.
|
||||
@@ -1,5 +0,0 @@
|
||||
- Updates to Inworld TTS services:
|
||||
|
||||
- Improved `InworldTTSService`'s websocket implementation to better flush and
|
||||
close context to better handle long inputs.
|
||||
- Improved docstrings for `InworldTTSService` and `InworldHttpTTSService`.
|
||||
@@ -1 +0,0 @@
|
||||
- Added a new `AudioContextTTSService` to the TTS service base classes. The `AudioContextWordTTSService` now inherits from `AudioContextTTSService` and `WebsocketWordTTSService`.
|
||||
@@ -1,4 +0,0 @@
|
||||
- `LLMUserAggregator` now exposes the following events:
|
||||
- `on_user_turn_started`: triggered when a user turn starts
|
||||
- `on_user_turn_stopped`: triggered when a user turn ends
|
||||
- `on_user_turn_stop_timeout`: triggered when a user turn does not stop and times out
|
||||
@@ -1,29 +0,0 @@
|
||||
- Introducing user mute strategies. User mute strategies indicate when user input should be muted based on the current system state.
|
||||
|
||||
In conversational agents, user mute strategies are used to prevent user input from interrupting bot speech, tool execution, or other critical system operations.
|
||||
|
||||
A list of strategies can be specified; all strategies are evaluated for every frame so that each strategy can maintain its internal state. A user frame is muted if any of the configured strategies indicates it should be muted.
|
||||
|
||||
Available user mute strategies:
|
||||
|
||||
* `FirstSpeechUserMuteStrategy`
|
||||
* `MuteUntilFirstBotCompleteUserMuteStrategy`
|
||||
* `AlwaysUserMuteStrategy`
|
||||
* `FunctionCallUserMuteStrategy`
|
||||
|
||||
User mute strategies replace the legacy `STTMuteFilter` and provide a more flexible and composable approach to muting user input.
|
||||
|
||||
User mute strategies are configured when setting up the `LLMContextAggregatorPair`. For example:
|
||||
|
||||
```python
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_mute_strategies=[
|
||||
FirstSpeechUserMuteStrategy(),
|
||||
]
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
In order to use user mute strategies you should update to the new universal `LLMContext` and `LLMContextAggregatorPair`.
|
||||
@@ -1 +0,0 @@
|
||||
- `STTMuteFilter` is deprecated and will be removed in a future version. Use `LLMUserAggregator`'s new `user_mute_strategies` instead.
|
||||
@@ -1 +0,0 @@
|
||||
- Fixed a bug in `STTMuteFilter` where the user was not always muted during function calls, especially when there were multiple simultaneous calls.
|
||||
@@ -1 +0,0 @@
|
||||
- `FrameProcessor.interruptions_allowed` is now deprecated, use `LLMUserAggregator`'s new parameter `user_mute_strategies` instead.
|
||||
@@ -1,12 +0,0 @@
|
||||
- `PipelineParams.allow_interruptions` is now deprecated, use `LLMUserAggregator`'s new parameter `turn_start_strategies` instead. For example, to disable interruptions but still get user turns you can do:
|
||||
|
||||
```python
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
start=[TranscriptionUserTurnStartStrategy(enable_interruptions=False)],
|
||||
),
|
||||
),
|
||||
)
|
||||
```
|
||||
@@ -1 +0,0 @@
|
||||
- Added `use_ssl` parameter to `NvidiaSTTService`, `NvidiaSegmentedSTTService` and `NvidiaTTSService`.
|
||||
@@ -1 +0,0 @@
|
||||
- Updated `DeepgramSTTService` to push user started/stopped speaking and interruption frames when `vad_enabled` is set to true. This centralizes the frames into the service, removing the need to have your application code handle Deepgram's events and push these frames.
|
||||
@@ -1 +0,0 @@
|
||||
- Added `enable_interruptions` constructor argument to all user turn strategies. This tells the `LLMUserAggregator` to push or not push an `InterruptionFrame`.
|
||||
@@ -1 +0,0 @@
|
||||
- Added `52-live-transcription.py` foundational example demonstrating live transcription and translation from English to Spanish. In this example, the bot is not interruptible: as the user continues speaking, English transcriptions are queued, and the bot continuously translates and speaks each queued sentence in Spanish without being interrupted by new user speech.
|
||||
1
changelog/3349.added.md
Normal file
1
changelog/3349.added.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added `CambTTSService`, using Camb.ai's TTS integration with MARS models (mars-flash, mars-pro, mars-instruct) for high-quality text-to-speech synthesis.
|
||||
8
changelog/3446.fixed.md
Normal file
8
changelog/3446.fixed.md
Normal file
@@ -0,0 +1,8 @@
|
||||
- Fixed an issue where the "bot-llm-text" RTVI event would not fire for realtime (speech-to-speech) services:
|
||||
|
||||
- `AWSNovaSonicLLMService`
|
||||
- `GeminiLiveLLMService`
|
||||
- `OpenAIRealtimeLLMService`
|
||||
- `GrokRealtimeLLMService`
|
||||
|
||||
The issue was that these services weren't pushing `LLMTextFrame`s. Now they do.
|
||||
1
changelog/3454.fixed.md
Normal file
1
changelog/3454.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Fixed an issue where `on_user_turn_stop_timeout` could fire while a user is talking when using `ExternalUserTurnStrategies`.
|
||||
1
changelog/3455.fixed.md
Normal file
1
changelog/3455.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Fixed an issue where user turn start strategies were not being reset after a user turn started, causing incorrect strategy behavior.
|
||||
1
changelog/3461.added.md
Normal file
1
changelog/3461.added.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added the `additional_headers` param to `WebsocketClientParams`, allowing `WebsocketClientTransport` to send custom headers on connect, for cases such as authentication.
|
||||
1
changelog/3462.fixed.md
Normal file
1
changelog/3462.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Fixed `MinWordsUserTurnStartStrategy` to not aggregate transcriptions, preventing incorrect turn starts when words are spoken with pauses between them.
|
||||
1
changelog/3479.deprecated.md
Normal file
1
changelog/3479.deprecated.md
Normal file
@@ -0,0 +1 @@
|
||||
- For consistency with other package names, we just deprecated `pipecat.turns.mute` (introduced in Pipecat 0.0.99) in favor of `pipecat.turns.user_mute`.
|
||||
1
changelog/3480.fixed.md
Normal file
1
changelog/3480.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Fixed an issue where Grok Realtime would error out when running with SmallWebRTC transport.
|
||||
1
changelog/3482.added.md
Normal file
1
changelog/3482.added.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added `UserIdleController` for detecting user idle state, integrated into `LLMUserAggregator` and `UserTurnProcessor` via optional `user_idle_timeout` parameter. Emits `on_user_turn_idle` event for application-level handling. Deprecated `UserIdleProcessor` in favor of the new compositional approach.
|
||||
1
changelog/3483.changed.md
Normal file
1
changelog/3483.changed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Throttle `UserSpeakingFrame` to broadcast at most every 200ms instead of on every audio chunk, reducing frame processing overhead during user speech.
|
||||
1
changelog/3484.fixed.md
Normal file
1
changelog/3484.fixed.md
Normal file
@@ -0,0 +1 @@
|
||||
- Fixed a `Mem0MemoryService` issue where passing `async_mode: true` was causing an error. See https://docs.mem0.ai/platform/features/async-mode-default-change.
|
||||
3
changelog/3489.fixed.md
Normal file
3
changelog/3489.fixed.md
Normal file
@@ -0,0 +1,3 @@
|
||||
- Fixed `AzureTTSService` transcript formatting issues:
|
||||
- Punctuation now appears without extra spaces (e.g., "Hello!" instead of "Hello !")
|
||||
- CJK languages (Chinese, Japanese, Korean) no longer have unwanted spaces between characters
|
||||
1
changelog/3490.added.md
Normal file
1
changelog/3490.added.md
Normal file
@@ -0,0 +1 @@
|
||||
- Added `on_user_mute_started` and `on_user_mute_stopped` event handlers to `LLMUserAggregator` for tracking user mute state changes.
|
||||
@@ -91,6 +91,25 @@ autodoc_mock_imports = [
|
||||
# MLX dependencies (Apple Silicon specific)
|
||||
"mlx",
|
||||
"mlx_whisper", # Note: might need underscore format too
|
||||
# Pydantic v2 compatibility issues in third-party SDKs
|
||||
"hume",
|
||||
"hume.tts",
|
||||
"hume.tts.types",
|
||||
"cartesia",
|
||||
"camb",
|
||||
"sarvamai",
|
||||
"openpipe",
|
||||
"openai.types.beta.realtime",
|
||||
"langchain_core",
|
||||
"langchain_core.messages",
|
||||
# FastAPI - Pydantic v2 compatibility issues during Sphinx autodoc
|
||||
"fastapi",
|
||||
"fastapi.applications",
|
||||
"fastapi.routing",
|
||||
"fastapi.params",
|
||||
"fastapi.middleware",
|
||||
"fastapi.responses",
|
||||
"uvicorn",
|
||||
]
|
||||
|
||||
# HTML output settings
|
||||
|
||||
@@ -31,6 +31,9 @@ AZURE_DALLE_API_KEY=...
|
||||
AZURE_DALLE_ENDPOINT=https://...
|
||||
AZURE_DALLE_MODEL=...
|
||||
|
||||
# Camb.ai
|
||||
CAMB_API_KEY=...
|
||||
|
||||
# Cartesia
|
||||
CARTESIA_API_KEY=...
|
||||
CARTESIA_VOICE_ID=...
|
||||
@@ -82,6 +85,9 @@ GROK_API_KEY=...
|
||||
# Groq
|
||||
GROQ_API_KEY=...
|
||||
|
||||
# Hathora
|
||||
HATHORA_API_KEY=...
|
||||
|
||||
# Heygen
|
||||
HEYGEN_API_KEY=...
|
||||
HEYGEN_LIVE_AVATAR_API_KEY=...
|
||||
@@ -97,7 +103,8 @@ INWORLD_API_KEY=...
|
||||
KRISP_MODEL_PATH=...
|
||||
|
||||
# Krisp Viva
|
||||
KRISP_VIVA_MODEL_PATH=...
|
||||
KRISP_VIVA_FILTER_MODEL_PATH=...
|
||||
KRISP_VIVA_TURN_MODEL_PATH=...
|
||||
|
||||
# LiveKit
|
||||
LIVEKIT_API_KEY=...
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -85,7 +85,7 @@ async def run_example(webrtc_connection: SmallWebRTCConnection):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -98,11 +98,11 @@ async def run_example(webrtc_connection: SmallWebRTCConnection):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -68,7 +68,7 @@ async def main():
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -82,11 +82,11 @@ async def main():
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -78,7 +78,7 @@ async def main():
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -91,11 +91,11 @@ async def main():
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -106,7 +106,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -119,12 +119,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(),
|
||||
stt,
|
||||
context_aggregator.user(),
|
||||
user_aggregator,
|
||||
llm,
|
||||
tts,
|
||||
ml,
|
||||
transport.output(),
|
||||
context_aggregator.assistant(),
|
||||
assistant_aggregator,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -138,12 +138,12 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(),
|
||||
stt,
|
||||
context_aggregator.user(),
|
||||
user_aggregator,
|
||||
llm,
|
||||
tts,
|
||||
image_sync_aggregator,
|
||||
transport.output(),
|
||||
context_aggregator.assistant(),
|
||||
assistant_aggregator,
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -77,7 +77,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -90,11 +90,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -89,11 +89,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -131,7 +131,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()),
|
||||
)
|
||||
@@ -140,11 +140,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -117,7 +117,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -132,11 +132,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -103,7 +103,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
lc = LangchainProcessor(history_chain)
|
||||
|
||||
context = LLMContext()
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -116,11 +116,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
lc, # Langchain
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -71,7 +71,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()),
|
||||
)
|
||||
@@ -80,11 +80,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -81,7 +81,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -96,11 +96,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -86,7 +86,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -99,11 +99,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -72,7 +72,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()),
|
||||
)
|
||||
@@ -81,11 +81,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -75,7 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -88,11 +88,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -85,7 +85,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -100,11 +100,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -91,11 +91,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -91,11 +91,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -93,11 +93,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -84,7 +84,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -97,11 +97,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -84,7 +84,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -97,11 +97,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -78,7 +78,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -91,11 +91,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -83,7 +83,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -96,11 +96,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -81,7 +81,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -96,11 +96,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
140
examples/foundational/07j-interruptible-gladia-vad.py
Normal file
140
examples/foundational/07j-interruptible-gladia-vad.py
Normal file
@@ -0,0 +1,140 @@
|
||||
#
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
|
||||
import os
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||
from pipecat.frames.frames import LLMRunFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.aggregators.llm_response_universal import (
|
||||
LLMContextAggregatorPair,
|
||||
LLMUserAggregatorParams,
|
||||
)
|
||||
from pipecat.runner.types import RunnerArguments
|
||||
from pipecat.runner.utils import create_transport
|
||||
from pipecat.services.cartesia.tts import CartesiaTTSService
|
||||
from pipecat.services.gladia.config import GladiaInputParams, LanguageConfig
|
||||
from pipecat.services.gladia.stt import GladiaSTTService
|
||||
from pipecat.services.openai.llm import OpenAILLMService
|
||||
from pipecat.transcriptions.language import Language
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from pipecat.transports.daily.transport import DailyParams
|
||||
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
|
||||
from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
# We store functions so objects (e.g. SileroVADAnalyzer) don't get
|
||||
# instantiated. The function will be called when the desired transport gets
|
||||
# selected.
|
||||
transport_params = {
|
||||
"daily": lambda: DailyParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
),
|
||||
"twilio": lambda: FastAPIWebsocketParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
),
|
||||
"webrtc": lambda: TransportParams(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
logger.info(f"Starting bot")
|
||||
|
||||
stt = GladiaSTTService(
|
||||
api_key=os.getenv("GLADIA_API_KEY", ""),
|
||||
region=os.getenv("GLADIA_REGION"),
|
||||
params=GladiaInputParams(
|
||||
language_config=LanguageConfig(
|
||||
languages=[Language.EN],
|
||||
),
|
||||
enable_vad=True,
|
||||
),
|
||||
)
|
||||
|
||||
tts = CartesiaTTSService(
|
||||
api_key=os.getenv("CARTESIA_API_KEY", ""),
|
||||
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
|
||||
)
|
||||
|
||||
llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY", ""))
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": f"You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way.",
|
||||
},
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(user_turn_strategies=ExternalUserTurnStrategies()),
|
||||
)
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
enable_metrics=True,
|
||||
enable_usage_metrics=True,
|
||||
),
|
||||
idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
|
||||
)
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(transport, client):
|
||||
logger.info(f"Client connected")
|
||||
# Kick off the conversation.
|
||||
messages.append({"role": "system", "content": "Please introduce yourself to the user."})
|
||||
await task.queue_frames([LLMRunFrame()])
|
||||
|
||||
@transport.event_handler("on_client_disconnected")
|
||||
async def on_client_disconnected(transport, client):
|
||||
logger.info(f"Client disconnected")
|
||||
await task.cancel()
|
||||
|
||||
runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
|
||||
await runner.run(task)
|
||||
|
||||
|
||||
async def bot(runner_args: RunnerArguments):
|
||||
"""Main bot entry point compatible with Pipecat Cloud."""
|
||||
transport = await create_transport(runner_args, transport_params)
|
||||
await run_bot(transport, runner_args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from pipecat.runner.run import main
|
||||
|
||||
main()
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -87,7 +87,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -100,11 +100,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -75,7 +75,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -88,11 +88,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User respones
|
||||
user_aggregator, # User respones
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -76,7 +76,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -89,11 +89,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt,
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -120,7 +120,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
|
||||
# Setup context aggregators for message handling
|
||||
context = LLMContext()
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -133,11 +133,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # Speech-to-text
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # Strands Agents processor
|
||||
tts, # Text-to-speech
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -93,11 +93,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -102,7 +102,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -115,11 +115,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # Gemini TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -107,7 +107,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -120,11 +120,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # Gemini TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -103,11 +103,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User respones
|
||||
user_aggregator, # User respones
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -90,7 +90,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -103,11 +103,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User respones
|
||||
user_aggregator, # User respones
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
@@ -80,7 +80,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
@@ -93,11 +93,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,9 +1,26 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
# Copyright (c) 2024-2026, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
"""Interruptible bot with Krisp VIVA noise filtering and turn detection.
|
||||
|
||||
This example demonstrates a conversational bot with:
|
||||
- Krisp VIVA noise reduction on incoming audio
|
||||
- Krisp VIVA Turn detection for natural interruptions
|
||||
- Voice activity detection (VAD)
|
||||
|
||||
Required environment variables:
|
||||
- KRISP_VIVA_FILTER_MODEL_PATH: Path to the Krisp noise filter model file (.kef)
|
||||
- KRISP_VIVA_TURN_MODEL_PATH: Path to the Krisp turn detection model file (.kef)
|
||||
- DEEPGRAM_API_KEY: Deepgram API key for STT/TTS
|
||||
- OPENAI_API_KEY: OpenAI API key for LLM
|
||||
|
||||
Optional environment variables:
|
||||
- KRISP_NOISE_SUPPRESSION_LEVEL: Noise suppression level 0-100 (default: 100)
|
||||
Higher values = more aggressive noise reduction
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
@@ -11,7 +28,7 @@ from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.audio.filters.krisp_viva_filter import KrispVivaFilter
|
||||
from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
|
||||
from pipecat.audio.turn.krisp_viva_turn import KrispTurnParams, KrispVivaTurn
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.audio.vad.vad_analyzer import VADParams
|
||||
from pipecat.frames.frames import LLMRunFrame
|
||||
@@ -78,11 +95,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
]
|
||||
|
||||
context = LLMContext(messages)
|
||||
context_aggregator = LLMContextAggregatorPair(
|
||||
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||
context,
|
||||
user_params=LLMUserAggregatorParams(
|
||||
user_turn_strategies=UserTurnStrategies(
|
||||
stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=LocalSmartTurnAnalyzerV3())]
|
||||
stop=[TurnAnalyzerUserTurnStopStrategy(turn_analyzer=KrispVivaTurn())]
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -91,11 +108,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
stt, # STT
|
||||
context_aggregator.user(), # User responses
|
||||
user_aggregator, # User responses
|
||||
llm, # LLM
|
||||
tts, # TTS
|
||||
transport.output(), # Transport bot output
|
||||
context_aggregator.assistant(), # Assistant spoken responses
|
||||
assistant_aggregator, # Assistant spoken responses
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user