Vonage Video Connector Transport
This commit is contained in:
@@ -211,6 +211,11 @@ TWILIO_AUTH_TOKEN=...
|
|||||||
# Ultravox Realtime
|
# Ultravox Realtime
|
||||||
ULTRAVOX_API_KEY=...
|
ULTRAVOX_API_KEY=...
|
||||||
|
|
||||||
|
# Vonage
|
||||||
|
VONAGE_APPLICATION_ID=...
|
||||||
|
VONAGE_SESSION_ID=...
|
||||||
|
VONAGE_TOKEN=...
|
||||||
|
|
||||||
# WhatsApp
|
# WhatsApp
|
||||||
WHATSAPP_TOKEN=...
|
WHATSAPP_TOKEN=...
|
||||||
WHATSAPP_WEBHOOK_VERIFICATION_TOKEN=...
|
WHATSAPP_WEBHOOK_VERIFICATION_TOKEN=...
|
||||||
|
|||||||
@@ -55,6 +55,20 @@ Then, run the example with:
|
|||||||
uv run getting-started/06-voice-agent.py -t twilio -x NGROK_HOST_NAME
|
uv run getting-started/06-voice-agent.py -t twilio -x NGROK_HOST_NAME
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Vonage
|
||||||
|
|
||||||
|
It is also possible to run the example through a Vonage session. Just provide the values for the following variables in
|
||||||
|
the `.env` file:
|
||||||
|
* VONAGE_APPLICATION_ID
|
||||||
|
* VONAGE_SESSION_ID
|
||||||
|
* VONAGE_TOKEN
|
||||||
|
|
||||||
|
Then, run the example with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
uv run getting-started/06-voice-agent.py -t vonage
|
||||||
|
```
|
||||||
|
|
||||||
## Directory Structure
|
## Directory Structure
|
||||||
|
|
||||||
### [`getting-started/`](./getting-started/)
|
### [`getting-started/`](./getting-started/)
|
||||||
|
|||||||
108
examples/transports/transports-vonage.py
Normal file
108
examples/transports/transports-vonage.py
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024-2026, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Example of using AWS Nova Sonic LLM service with Vonage Video Connector transport."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||||
|
from pipecat.frames.frames import LLMRunFrame
|
||||||
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
|
from pipecat.pipeline.task import PipelineTask
|
||||||
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||||
|
from pipecat.processors.aggregators.llm_response_universal import (
|
||||||
|
LLMContextAggregatorPair,
|
||||||
|
LLMUserAggregatorParams,
|
||||||
|
)
|
||||||
|
from pipecat.runner.vonage import configure
|
||||||
|
from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService
|
||||||
|
from pipecat.transports.vonage.video_connector import (
|
||||||
|
VonageVideoConnectorTransport,
|
||||||
|
VonageVideoConnectorTransportParams,
|
||||||
|
)
|
||||||
|
|
||||||
|
load_dotenv(override=True)
|
||||||
|
|
||||||
|
logger.remove(0)
|
||||||
|
logger.add(sys.stderr, level="DEBUG")
|
||||||
|
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
"""Main entry point for the nova sonic vonage video connector example."""
|
||||||
|
(application_id, session_id, token) = await configure()
|
||||||
|
|
||||||
|
system_instruction = (
|
||||||
|
"You are a friendly assistant. The user and you will engage in a spoken dialog exchanging "
|
||||||
|
"the transcripts of a natural real-time conversation. Keep your responses short, generally "
|
||||||
|
"two or three sentences for chatty scenarios. "
|
||||||
|
f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}"
|
||||||
|
)
|
||||||
|
transport = VonageVideoConnectorTransport(
|
||||||
|
application_id,
|
||||||
|
session_id,
|
||||||
|
token,
|
||||||
|
VonageVideoConnectorTransportParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
publisher_name="Bot",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
llm = AWSNovaSonicLLMService(
|
||||||
|
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""),
|
||||||
|
access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""),
|
||||||
|
region=os.getenv("AWS_REGION", ""),
|
||||||
|
session_token=os.getenv("AWS_SESSION_TOKEN", ""),
|
||||||
|
voice_id="tiffany",
|
||||||
|
)
|
||||||
|
context = LLMContext(
|
||||||
|
messages=[
|
||||||
|
{"role": "system", "content": f"{system_instruction}"},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "Tell me a fun fact!",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
||||||
|
context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer())
|
||||||
|
)
|
||||||
|
|
||||||
|
pipeline = Pipeline(
|
||||||
|
[
|
||||||
|
transport.input(),
|
||||||
|
user_aggregator,
|
||||||
|
llm,
|
||||||
|
transport.output(),
|
||||||
|
assistant_aggregator,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
task = PipelineTask(pipeline)
|
||||||
|
|
||||||
|
# Handle client connection event
|
||||||
|
event_handler: Callable[[str], Callable[[Any], Any]] = transport.event_handler
|
||||||
|
|
||||||
|
@event_handler("on_client_connected")
|
||||||
|
async def on_client_connected(transport: VonageVideoConnectorTransport, client: object) -> None:
|
||||||
|
logger.info(f"Client connected")
|
||||||
|
await task.queue_frames([LLMRunFrame()])
|
||||||
|
|
||||||
|
runner = PipelineRunner()
|
||||||
|
|
||||||
|
await asyncio.gather(runner.run(task))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
@@ -119,6 +119,7 @@ tavus = [ "pipecat-ai[daily]" ]
|
|||||||
together = []
|
together = []
|
||||||
tracing = [ "opentelemetry-sdk>=1.33.0,<2", "opentelemetry-api>=1.33.0,<2", "opentelemetry-instrumentation>=0.54b0,<1" ]
|
tracing = [ "opentelemetry-sdk>=1.33.0,<2", "opentelemetry-api>=1.33.0,<2", "opentelemetry-instrumentation>=0.54b0,<1" ]
|
||||||
ultravox = [ "pipecat-ai[websockets-base]" ]
|
ultravox = [ "pipecat-ai[websockets-base]" ]
|
||||||
|
vonage-video-connector = [ "vonage-video-connector~=0.2.3b0; python_full_version>='3.13' and python_full_version<'3.14' and platform_system=='Linux'" ]
|
||||||
webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ]
|
webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ]
|
||||||
websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<1" ]
|
websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<1" ]
|
||||||
websockets-base = [ "websockets>=13.1,<16.0" ]
|
websockets-base = [ "websockets>=13.1,<16.0" ]
|
||||||
|
|||||||
@@ -108,8 +108,10 @@ from pipecat.runner.types import (
|
|||||||
DailyRunnerArguments,
|
DailyRunnerArguments,
|
||||||
RunnerArguments,
|
RunnerArguments,
|
||||||
SmallWebRTCRunnerArguments,
|
SmallWebRTCRunnerArguments,
|
||||||
|
VonageRunnerArguments,
|
||||||
WebSocketRunnerArguments,
|
WebSocketRunnerArguments,
|
||||||
)
|
)
|
||||||
|
from pipecat.runner.vonage import configure as configure_vonage
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import uvicorn
|
import uvicorn
|
||||||
@@ -983,6 +985,25 @@ async def _run_daily_direct(args: argparse.Namespace):
|
|||||||
await bot_module.bot(runner_args)
|
await bot_module.bot(runner_args)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_vonage():
|
||||||
|
"""Run Vonage bot (no FastAPI server)."""
|
||||||
|
logger.info("Running Vonage transport...")
|
||||||
|
|
||||||
|
application_id, session_id, token = await configure_vonage()
|
||||||
|
runner_args = VonageRunnerArguments(
|
||||||
|
application_id=application_id, session_id=session_id, token=token
|
||||||
|
)
|
||||||
|
runner_args.handle_sigint = True
|
||||||
|
|
||||||
|
# Get the bot module and run it directly
|
||||||
|
bot_module = _get_bot_module()
|
||||||
|
|
||||||
|
print(f"Joining Vonage session: {runner_args.session_id}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
await bot_module.bot(runner_args)
|
||||||
|
|
||||||
|
|
||||||
def _validate_and_clean_proxy(proxy: str) -> str:
|
def _validate_and_clean_proxy(proxy: str) -> str:
|
||||||
"""Validate and clean proxy hostname, removing protocol if present."""
|
"""Validate and clean proxy hostname, removing protocol if present."""
|
||||||
if not proxy:
|
if not proxy:
|
||||||
@@ -1062,7 +1083,7 @@ def main(parser: argparse.ArgumentParser | None = None):
|
|||||||
"-t",
|
"-t",
|
||||||
"--transport",
|
"--transport",
|
||||||
type=str,
|
type=str,
|
||||||
choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS],
|
choices=["daily", "vonage", "webrtc", *TELEPHONY_TRANSPORTS],
|
||||||
default=None,
|
default=None,
|
||||||
help=(
|
help=(
|
||||||
"Restrict the server to a single transport and set it as the default for /start. "
|
"Restrict the server to a single transport and set it as the default for /start. "
|
||||||
@@ -1169,6 +1190,12 @@ def main(parser: argparse.ArgumentParser | None = None):
|
|||||||
if args.proxy:
|
if args.proxy:
|
||||||
print(f" → XML webhook: http://{args.host}:{args.port}/")
|
print(f" → XML webhook: http://{args.host}:{args.port}/")
|
||||||
print(f" → WebSocket: ws://{args.host}:{args.port}/ws")
|
print(f" → WebSocket: ws://{args.host}:{args.port}/ws")
|
||||||
|
elif args.transport == "vonage":
|
||||||
|
print()
|
||||||
|
print(f"🚀 Bot ready!")
|
||||||
|
asyncio.run(_run_vonage())
|
||||||
|
print()
|
||||||
|
return
|
||||||
print()
|
print()
|
||||||
|
|
||||||
RUNNER_DOWNLOADS_FOLDER = args.folder
|
RUNNER_DOWNLOADS_FOLDER = args.folder
|
||||||
|
|||||||
@@ -99,6 +99,21 @@ class DailyRunnerArguments(RunnerArguments):
|
|||||||
token: str | None = None
|
token: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class VonageRunnerArguments(RunnerArguments):
|
||||||
|
"""Daily transport session arguments for the runner.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
application_id: Vonage application ID
|
||||||
|
session_id: Vonage session ID
|
||||||
|
token: Vonage Session Token
|
||||||
|
"""
|
||||||
|
|
||||||
|
application_id: str
|
||||||
|
session_id: str
|
||||||
|
token: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class WebSocketRunnerArguments(RunnerArguments):
|
class WebSocketRunnerArguments(RunnerArguments):
|
||||||
"""WebSocket transport session arguments for the runner.
|
"""WebSocket transport session arguments for the runner.
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
from fastapi import WebSocket
|
from fastapi import WebSocket
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
@@ -42,9 +42,10 @@ from pipecat.runner.types import (
|
|||||||
DailyRunnerArguments,
|
DailyRunnerArguments,
|
||||||
LiveKitRunnerArguments,
|
LiveKitRunnerArguments,
|
||||||
SmallWebRTCRunnerArguments,
|
SmallWebRTCRunnerArguments,
|
||||||
|
VonageRunnerArguments,
|
||||||
WebSocketRunnerArguments,
|
WebSocketRunnerArguments,
|
||||||
)
|
)
|
||||||
from pipecat.transports.base_transport import BaseTransport
|
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||||
|
|
||||||
|
|
||||||
def _detect_transport_type_from_message(message_data: dict) -> str:
|
def _detect_transport_type_from_message(message_data: dict) -> str:
|
||||||
@@ -271,6 +272,14 @@ def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
from pipecat.transports.vonage.video_connector import VonageVideoConnectorTransport
|
||||||
|
|
||||||
|
if isinstance(transport, VonageVideoConnectorTransport):
|
||||||
|
return client["streamId"]
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
logger.warning(f"Unable to get client id from unsupported transport {type(transport)}")
|
logger.warning(f"Unable to get client id from unsupported transport {type(transport)}")
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@@ -303,6 +312,24 @@ async def maybe_capture_participant_camera(
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
from pipecat.transports.vonage.video_connector import (
|
||||||
|
SubscribeSettings,
|
||||||
|
VonageVideoConnectorTransport,
|
||||||
|
)
|
||||||
|
|
||||||
|
if isinstance(transport, VonageVideoConnectorTransport):
|
||||||
|
await transport.subscribe_to_stream(
|
||||||
|
client["streamId"],
|
||||||
|
SubscribeSettings(
|
||||||
|
subscribe_to_audio=True,
|
||||||
|
subscribe_to_video=True,
|
||||||
|
preferred_framerate=framerate if framerate != 0 else None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
async def maybe_capture_participant_screen(
|
async def maybe_capture_participant_screen(
|
||||||
transport: BaseTransport, client: Any, framerate: int = 0
|
transport: BaseTransport, client: Any, framerate: int = 0
|
||||||
@@ -534,6 +561,11 @@ async def create_transport(
|
|||||||
audio_out_enabled=True,
|
audio_out_enabled=True,
|
||||||
# add_wav_header and serializer will be set automatically
|
# add_wav_header and serializer will be set automatically
|
||||||
),
|
),
|
||||||
|
"vonage": lambda: VonageVideoConnectorParams(
|
||||||
|
audio_in_enabled=True,
|
||||||
|
audio_out_enabled=True,
|
||||||
|
vad_analyzer=SileroVADAnalyzer(),
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
transport = await create_transport(runner_args, transport_params)
|
transport = await create_transport(runner_args, transport_params)
|
||||||
@@ -587,6 +619,31 @@ async def create_transport(
|
|||||||
runner_args.room_name,
|
runner_args.room_name,
|
||||||
params=params,
|
params=params,
|
||||||
)
|
)
|
||||||
|
elif isinstance(runner_args, VonageRunnerArguments):
|
||||||
|
from pipecat.transports.vonage.video_connector import (
|
||||||
|
VonageVideoConnectorTransport,
|
||||||
|
VonageVideoConnectorTransportParams,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
params = cast(
|
||||||
|
VonageVideoConnectorTransportParams,
|
||||||
|
_get_transport_params("vonage", transport_params),
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
webrtc_params: TransportParams = cast(
|
||||||
|
TransportParams, _get_transport_params("webrtc", transport_params)
|
||||||
|
)
|
||||||
|
params = VonageVideoConnectorTransportParams(
|
||||||
|
**webrtc_params.model_dump(),
|
||||||
|
video_in_auto_subscribe=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return VonageVideoConnectorTransport(
|
||||||
|
runner_args.application_id,
|
||||||
|
runner_args.session_id,
|
||||||
|
runner_args.token,
|
||||||
|
params=params,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Unsupported runner arguments type: {type(runner_args)}")
|
raise ValueError(f"Unsupported runner arguments type: {type(runner_args)}")
|
||||||
|
|||||||
52
src/pipecat/runner/vonage.py
Normal file
52
src/pipecat/runner/vonage.py
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024-2026, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Vonage session configuration utilities.
|
||||||
|
|
||||||
|
This module extracts the necessary parameters to connect to a Vonage Video session.
|
||||||
|
|
||||||
|
Required environment variables:
|
||||||
|
|
||||||
|
- VONAGE_APPLICATION_ID - Vonage application ID
|
||||||
|
- VONAGE_SESSION_ID - Vonage session ID
|
||||||
|
- VONAGE_TOKEN - Vonage token
|
||||||
|
|
||||||
|
Example:
|
||||||
|
from pipecat.runner.vonage import configure
|
||||||
|
|
||||||
|
application_id, session_id, token = await configure()
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
async def configure() -> tuple[str, str, str]:
|
||||||
|
"""Configure Vonage application ID, session ID and token from environment.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple containing the server application_id, session_id and token.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If required Vonage configuration is not provided.
|
||||||
|
"""
|
||||||
|
application_id = os.getenv("VONAGE_APPLICATION_ID")
|
||||||
|
session_id = os.getenv("VONAGE_SESSION_ID")
|
||||||
|
token = os.getenv("VONAGE_TOKEN")
|
||||||
|
|
||||||
|
if not application_id:
|
||||||
|
raise Exception(
|
||||||
|
"No Vonage application ID specified. Use set VONAGE_APPLICATION_ID in your environment."
|
||||||
|
)
|
||||||
|
|
||||||
|
if not session_id:
|
||||||
|
raise Exception(
|
||||||
|
"No Vonage Session ID specified. Use set VONAGE_SESSION_ID in your environment."
|
||||||
|
)
|
||||||
|
|
||||||
|
if not token:
|
||||||
|
raise Exception("No Vonage token specified. Use set VONAGE_TOKEN in your environment.")
|
||||||
|
|
||||||
|
return (application_id, session_id, token)
|
||||||
0
src/pipecat/transports/vonage/__init__.py
Normal file
0
src/pipecat/transports/vonage/__init__.py
Normal file
1092
src/pipecat/transports/vonage/client.py
Normal file
1092
src/pipecat/transports/vonage/client.py
Normal file
File diff suppressed because it is too large
Load Diff
150
src/pipecat/transports/vonage/utils.py
Normal file
150
src/pipecat/transports/vonage/utils.py
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024-2026, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
"""Vonage Video Connector utils."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass, replace
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import numpy.typing as npt
|
||||||
|
|
||||||
|
from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AudioProps:
|
||||||
|
"""Audio properties for normalization.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
sample_rate: The sample rate of the audio.
|
||||||
|
is_stereo: Whether the audio is stereo (True) or mono (False).
|
||||||
|
"""
|
||||||
|
|
||||||
|
sample_rate: int
|
||||||
|
is_stereo: bool
|
||||||
|
|
||||||
|
|
||||||
|
class ImageFormat(StrEnum):
|
||||||
|
"""Enum for image formats."""
|
||||||
|
|
||||||
|
PLANAR_YUV420 = "PLANAR_YUV420"
|
||||||
|
PACKED_YUV444 = "PACKED_YUV444"
|
||||||
|
RGB = "RGB"
|
||||||
|
RGBA = "RGBA"
|
||||||
|
BGR = "BGR"
|
||||||
|
BGRA = "BGRA"
|
||||||
|
|
||||||
|
|
||||||
|
def check_audio_data(
|
||||||
|
buffer: bytes | memoryview, number_of_frames: int, number_of_channels: int
|
||||||
|
) -> None:
|
||||||
|
"""Check the audio sample width based on buffer size, number of frames and channels."""
|
||||||
|
if number_of_channels not in (1, 2):
|
||||||
|
raise ValueError(f"We only accept mono or stereo audio, got {number_of_channels}")
|
||||||
|
|
||||||
|
if isinstance(buffer, memoryview):
|
||||||
|
bytes_per_sample = buffer.itemsize
|
||||||
|
else:
|
||||||
|
bytes_per_sample = len(buffer) // (number_of_frames * number_of_channels)
|
||||||
|
|
||||||
|
if bytes_per_sample != 2:
|
||||||
|
raise ValueError(f"We only accept 16 bit PCM audio, got {bytes_per_sample * 8} bit")
|
||||||
|
|
||||||
|
|
||||||
|
def process_audio_channels(
|
||||||
|
audio: npt.NDArray[np.int16], current: AudioProps, target: AudioProps
|
||||||
|
) -> npt.NDArray[np.int16]:
|
||||||
|
"""Normalize audio channels to the target properties."""
|
||||||
|
if current.is_stereo != target.is_stereo:
|
||||||
|
if target.is_stereo:
|
||||||
|
audio = np.repeat(audio, 2)
|
||||||
|
else:
|
||||||
|
audio = audio.reshape(-1, 2).mean(axis=1).astype(np.int16)
|
||||||
|
|
||||||
|
return audio
|
||||||
|
|
||||||
|
|
||||||
|
async def process_audio(
|
||||||
|
resampler: BaseAudioResampler,
|
||||||
|
audio: npt.NDArray[np.int16],
|
||||||
|
current: AudioProps,
|
||||||
|
target: AudioProps,
|
||||||
|
) -> npt.NDArray[np.int16]:
|
||||||
|
"""Normalize audio to the target properties."""
|
||||||
|
res_audio = audio
|
||||||
|
if current.sample_rate != target.sample_rate:
|
||||||
|
# first normalize channels to mono if needed, then resample, then normalize channels to target
|
||||||
|
res_audio = process_audio_channels(res_audio, current, replace(current, is_stereo=False))
|
||||||
|
current = replace(current, is_stereo=False)
|
||||||
|
res_audio_bytes: bytes = await resampler.resample(
|
||||||
|
res_audio.tobytes(), current.sample_rate, target.sample_rate
|
||||||
|
)
|
||||||
|
res_audio = np.frombuffer(res_audio_bytes, dtype=np.int16)
|
||||||
|
|
||||||
|
res_audio = process_audio_channels(res_audio, current, target)
|
||||||
|
|
||||||
|
return res_audio
|
||||||
|
|
||||||
|
|
||||||
|
def image_colorspace_conversion(
|
||||||
|
image: bytes, size: tuple[int, int], from_format: ImageFormat, to_format: ImageFormat
|
||||||
|
) -> bytes | None:
|
||||||
|
"""Convert image colorspace from one format to another."""
|
||||||
|
match (from_format, to_format):
|
||||||
|
case (fmt1, fmt2) if fmt1 == fmt2:
|
||||||
|
return image
|
||||||
|
case (ImageFormat.RGB, ImageFormat.BGR) | (ImageFormat.BGR, ImageFormat.RGB):
|
||||||
|
np_input = np.frombuffer(image, dtype=np.uint8)
|
||||||
|
np_output = np_input.reshape(size[1], size[0], 3)[:, :, ::-1]
|
||||||
|
return np_output.tobytes()
|
||||||
|
case (ImageFormat.RGBA, ImageFormat.BGRA) | (ImageFormat.BGRA, ImageFormat.RGBA):
|
||||||
|
np_input = np.frombuffer(image, dtype=np.uint8)
|
||||||
|
np_output = np_input.reshape(size[1], size[0], 4)[:, :, [2, 1, 0, 3]]
|
||||||
|
return np_output.tobytes()
|
||||||
|
case (ImageFormat.PLANAR_YUV420, ImageFormat.PACKED_YUV444):
|
||||||
|
# YUV420 (I420) has Y plane of size width*height, U and V planes of size (width/2)*(height/2)
|
||||||
|
# Packed YUV444 interleaves Y, U, V values for each pixel (YUVYUVYUV...)
|
||||||
|
width, height = size
|
||||||
|
y_plane_size = width * height
|
||||||
|
uv_plane_size_420 = (width // 2) * (height // 2)
|
||||||
|
|
||||||
|
np_input = np.frombuffer(image, dtype=np.uint8)
|
||||||
|
y_plane = np_input[:y_plane_size].reshape(height, width)
|
||||||
|
u_plane_420 = np_input[y_plane_size : y_plane_size + uv_plane_size_420].reshape(
|
||||||
|
height // 2, width // 2
|
||||||
|
)
|
||||||
|
v_plane_420 = np_input[
|
||||||
|
y_plane_size + uv_plane_size_420 : y_plane_size + 2 * uv_plane_size_420
|
||||||
|
].reshape(height // 2, width // 2)
|
||||||
|
|
||||||
|
# Upsample U and V planes by repeating each pixel in 2x2 blocks
|
||||||
|
u_plane_444 = np.repeat(np.repeat(u_plane_420, 2, axis=0), 2, axis=1)
|
||||||
|
v_plane_444 = np.repeat(np.repeat(v_plane_420, 2, axis=0), 2, axis=1)
|
||||||
|
|
||||||
|
# Interleave Y, U, V values for packed format (YUVYUVYUV...)
|
||||||
|
np_output = np.stack([y_plane, u_plane_444, v_plane_444], axis=-1)
|
||||||
|
return np_output.tobytes()
|
||||||
|
case (ImageFormat.PACKED_YUV444, ImageFormat.PLANAR_YUV420):
|
||||||
|
# Packed YUV444 has Y, U, V interleaved (YUVYUVYUV...)
|
||||||
|
# YUV420 (I420) has Y plane of size width*height, U and V planes of size (width/2)*(height/2)
|
||||||
|
width, height = size
|
||||||
|
|
||||||
|
np_input = np.frombuffer(image, dtype=np.uint8).reshape(height, width, 3)
|
||||||
|
y_plane = np_input[:, :, 0].reshape(height, width)
|
||||||
|
u_plane_444 = np_input[:, :, 1]
|
||||||
|
v_plane_444 = np_input[:, :, 2]
|
||||||
|
|
||||||
|
# Downsample U and V planes by taking every other pixel (2x2 -> 1 averaging)
|
||||||
|
u_plane_420 = u_plane_444[::2, ::2].reshape(height // 2, width // 2)
|
||||||
|
v_plane_420 = v_plane_444[::2, ::2].reshape(height // 2, width // 2)
|
||||||
|
|
||||||
|
# Concatenate Y, U, V planes
|
||||||
|
np_output = np.concatenate(
|
||||||
|
[y_plane.flatten(), u_plane_420.flatten(), v_plane_420.flatten()]
|
||||||
|
)
|
||||||
|
return np_output.tobytes()
|
||||||
|
case _:
|
||||||
|
return None
|
||||||
483
src/pipecat/transports/vonage/video_connector.py
Normal file
483
src/pipecat/transports/vonage/video_connector.py
Normal file
@@ -0,0 +1,483 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024-2026, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
"""Vonage Video Connector transport."""
|
||||||
|
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.frames.frames import (
|
||||||
|
CancelFrame,
|
||||||
|
EndFrame,
|
||||||
|
Frame,
|
||||||
|
InputAudioRawFrame,
|
||||||
|
InterruptionFrame,
|
||||||
|
OutputAudioRawFrame,
|
||||||
|
OutputImageRawFrame,
|
||||||
|
StartFrame,
|
||||||
|
UserImageRawFrame,
|
||||||
|
)
|
||||||
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
|
||||||
|
from pipecat.transports.base_input import BaseInputTransport
|
||||||
|
from pipecat.transports.base_output import BaseOutputTransport
|
||||||
|
from pipecat.transports.base_transport import BaseTransport
|
||||||
|
from pipecat.transports.vonage.client import (
|
||||||
|
Session,
|
||||||
|
Stream,
|
||||||
|
Subscriber,
|
||||||
|
VonageClient,
|
||||||
|
VonageClientListener,
|
||||||
|
)
|
||||||
|
|
||||||
|
# the following "as" imports help to re-export these types and avoid type checking warnings
|
||||||
|
# when importing these types from the main transport module
|
||||||
|
from pipecat.transports.vonage.client import (
|
||||||
|
SubscribeSettings as SubscribeSettings,
|
||||||
|
)
|
||||||
|
from pipecat.transports.vonage.client import (
|
||||||
|
VonageException as VonageException,
|
||||||
|
)
|
||||||
|
from pipecat.transports.vonage.client import (
|
||||||
|
VonageVideoConnectorTransportParams as VonageVideoConnectorTransportParams,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class VonageVideoConnectorInputTransport(BaseInputTransport):
|
||||||
|
"""Input transport for Vonage, handling audio input from the Vonage session.
|
||||||
|
|
||||||
|
Receives audio from a Vonage Video session and pushes it as input frames.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_params: VonageVideoConnectorTransportParams
|
||||||
|
|
||||||
|
def __init__(self, client: VonageClient, params: VonageVideoConnectorTransportParams):
|
||||||
|
"""Initialize the Vonage input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: The VonageClient instance to use.
|
||||||
|
params: Transport parameters for input configuration.
|
||||||
|
"""
|
||||||
|
super().__init__(params)
|
||||||
|
self._initialized: bool = False
|
||||||
|
self._client: VonageClient = client
|
||||||
|
self._listener_id: int = -1
|
||||||
|
self._connected: bool = False
|
||||||
|
|
||||||
|
async def start(self, frame: StartFrame) -> None:
|
||||||
|
"""Start the Vonage input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The StartFrame to initiate the transport.
|
||||||
|
"""
|
||||||
|
await super().start(frame)
|
||||||
|
|
||||||
|
if self._initialized:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._initialized = True
|
||||||
|
|
||||||
|
if self._params.audio_in_enabled or self._params.video_in_enabled:
|
||||||
|
self._listener_id = self._client.add_listener(
|
||||||
|
VonageClientListener(
|
||||||
|
on_audio_in=self._audio_in_cb,
|
||||||
|
on_video_in=self._video_in_cb,
|
||||||
|
on_error=self._on_error_cb,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await self._client.connect(frame)
|
||||||
|
self._connected = True
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(f"Error connecting to Vonage session: {exc}")
|
||||||
|
await self.push_error("Vonage video connector connection error", fatal=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
|
async def setup(self, setup: FrameProcessorSetup) -> None:
|
||||||
|
"""Set up the processor with required components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: Configuration object containing setup parameters.
|
||||||
|
"""
|
||||||
|
await super().setup(setup)
|
||||||
|
await self._client.setup(setup)
|
||||||
|
|
||||||
|
async def cleanup(self) -> None:
|
||||||
|
"""Cleanup input transport."""
|
||||||
|
await super().cleanup() # type: ignore
|
||||||
|
await self._client.cleanup()
|
||||||
|
|
||||||
|
async def _audio_in_cb(self, _session: Session, audio: InputAudioRawFrame) -> None:
|
||||||
|
if self._connected and self._params.audio_in_enabled:
|
||||||
|
await self.push_audio_frame(audio)
|
||||||
|
|
||||||
|
async def _video_in_cb(self, _subscriber: Subscriber, video: UserImageRawFrame) -> None:
|
||||||
|
if self._connected and self._params.video_in_enabled:
|
||||||
|
await self.push_video_frame(video)
|
||||||
|
|
||||||
|
async def _on_error_cb(self, session: Session, description: str, code: int) -> None:
|
||||||
|
logger.error(
|
||||||
|
f"Vonage input transport error session={session.id} code={code} description={description}"
|
||||||
|
)
|
||||||
|
if self._connected:
|
||||||
|
await self.push_error("Vonage video connector error", fatal=True)
|
||||||
|
|
||||||
|
async def stop(self, frame: EndFrame) -> None:
|
||||||
|
"""Stop the Vonage input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The EndFrame to stop the transport.
|
||||||
|
"""
|
||||||
|
await super().stop(frame)
|
||||||
|
await self._stop_client()
|
||||||
|
|
||||||
|
async def cancel(self, frame: CancelFrame) -> None:
|
||||||
|
"""Cancel the Vonage input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The CancelFrame to cancel the transport.
|
||||||
|
"""
|
||||||
|
await super().cancel(frame)
|
||||||
|
await self._stop_client()
|
||||||
|
|
||||||
|
async def _stop_client(self) -> None:
|
||||||
|
if self._connected:
|
||||||
|
self._client.remove_listener(self._listener_id)
|
||||||
|
self._connected = False
|
||||||
|
try:
|
||||||
|
await self._client.disconnect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def subscribe_to_stream(self, stream_id: str, params: SubscribeSettings) -> None:
|
||||||
|
"""Subscribe to a participant's stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
stream_id: The ID of the participant to subscribe to.
|
||||||
|
params: Subscription parameters for the subscription.
|
||||||
|
"""
|
||||||
|
await self._client.subscribe_to_stream(stream_id, params)
|
||||||
|
|
||||||
|
|
||||||
|
class VonageVideoConnectorOutputTransport(BaseOutputTransport):
|
||||||
|
"""Output transport for Vonage, handling audio output to the Vonage session.
|
||||||
|
|
||||||
|
Sends audio frames to a Vonage Video session as output.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_params: VonageVideoConnectorTransportParams
|
||||||
|
|
||||||
|
def __init__(self, client: VonageClient, params: VonageVideoConnectorTransportParams):
|
||||||
|
"""Initialize the Vonage output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: The VonageClient instance to use.
|
||||||
|
params: Transport parameters for output configuration.
|
||||||
|
"""
|
||||||
|
super().__init__(params)
|
||||||
|
self._initialized: bool = False
|
||||||
|
self._client = client
|
||||||
|
self._connected: bool = False
|
||||||
|
self._listener_id: int = -1
|
||||||
|
|
||||||
|
async def start(self, frame: StartFrame) -> None:
|
||||||
|
"""Start the Vonage output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The StartFrame to initiate the transport.
|
||||||
|
"""
|
||||||
|
await super().start(frame)
|
||||||
|
|
||||||
|
if self._initialized:
|
||||||
|
return
|
||||||
|
|
||||||
|
self._initialized = True
|
||||||
|
|
||||||
|
if self._params.audio_out_enabled or self._params.video_out_enabled:
|
||||||
|
self._listener_id = self._client.add_listener(
|
||||||
|
VonageClientListener(on_error=self._on_error_cb)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await self._client.connect(frame)
|
||||||
|
self._connected = True
|
||||||
|
except Exception as exc:
|
||||||
|
logger.error(f"Error connecting to Vonage session: {exc}")
|
||||||
|
await self.push_error("Vonage video connector connection error", fatal=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
|
async def setup(self, setup: FrameProcessorSetup) -> None:
|
||||||
|
"""Set up the processor with required components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: Configuration object containing setup parameters.
|
||||||
|
"""
|
||||||
|
await super().setup(setup)
|
||||||
|
await self._client.setup(setup)
|
||||||
|
|
||||||
|
async def cleanup(self) -> None:
|
||||||
|
"""Cleanup output transport."""
|
||||||
|
await super().cleanup() # type: ignore
|
||||||
|
await self._client.cleanup()
|
||||||
|
|
||||||
|
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
|
||||||
|
"""Process a frame for the Vonage output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
|
# if we get an interruption frame, we need to ensure the buffers inside Vonage Video Connector are cleared
|
||||||
|
if (
|
||||||
|
self._connected
|
||||||
|
and isinstance(frame, InterruptionFrame)
|
||||||
|
and self._params.clear_buffers_on_interruption
|
||||||
|
):
|
||||||
|
logger.info("Clearing Vonage media buffers due to interruption frame")
|
||||||
|
self._client.clear_media_buffers()
|
||||||
|
|
||||||
|
async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
|
||||||
|
"""Write an audio frame to the Vonage session.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The OutputAudioRawFrame to send.
|
||||||
|
"""
|
||||||
|
result = False
|
||||||
|
if self._connected and self._params.audio_out_enabled:
|
||||||
|
result = await self._client.write_audio(frame)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
|
||||||
|
"""Write a video frame to the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output video frame to write.
|
||||||
|
"""
|
||||||
|
result = False
|
||||||
|
if self._connected and self._params.video_out_enabled:
|
||||||
|
result = await self._client.write_video(frame)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def stop(self, frame: EndFrame) -> None:
|
||||||
|
"""Stop the Vonage output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The EndFrame to stop the transport.
|
||||||
|
"""
|
||||||
|
await super().stop(frame)
|
||||||
|
await self._stop_client()
|
||||||
|
|
||||||
|
async def cancel(self, frame: CancelFrame) -> None:
|
||||||
|
"""Cancel the Vonage output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The CancelFrame to cancel the transport.
|
||||||
|
"""
|
||||||
|
await super().cancel(frame)
|
||||||
|
await self._stop_client()
|
||||||
|
|
||||||
|
async def _stop_client(self) -> None:
|
||||||
|
if self._connected:
|
||||||
|
self._client.remove_listener(self._listener_id)
|
||||||
|
self._connected = False
|
||||||
|
try:
|
||||||
|
await self._client.disconnect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def _on_error_cb(self, session: Session, description: str, code: int) -> None:
|
||||||
|
logger.error(
|
||||||
|
f"Vonage output transport error session={session.id} code={code} description={description}"
|
||||||
|
)
|
||||||
|
if self._connected:
|
||||||
|
await self.push_error("Vonage video connector error", fatal=True)
|
||||||
|
|
||||||
|
|
||||||
|
class VonageVideoConnectorTransport(BaseTransport):
|
||||||
|
"""Vonage Video Connector transport implementation for Pipecat.
|
||||||
|
|
||||||
|
Provides input and output audio transport for Vonage Video sessions, supporting event handling
|
||||||
|
for session and participant lifecycle.
|
||||||
|
|
||||||
|
Supported features:
|
||||||
|
|
||||||
|
- Audio input and output transport for Vonage Video sessions
|
||||||
|
- Event handler registration for session and participant events
|
||||||
|
- Publisher and subscriber management
|
||||||
|
- Configurable audio and migration parameters
|
||||||
|
"""
|
||||||
|
|
||||||
|
_params: VonageVideoConnectorTransportParams
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
application_id: str,
|
||||||
|
session_id: str,
|
||||||
|
token: str,
|
||||||
|
params: VonageVideoConnectorTransportParams,
|
||||||
|
):
|
||||||
|
"""Initialize the Vonage Video Connector transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
application_id: The Vonage Video application ID.
|
||||||
|
session_id: The session ID to connect to.
|
||||||
|
token: The authentication token for the session.
|
||||||
|
params: Transport parameters for input/output configuration.
|
||||||
|
"""
|
||||||
|
super().__init__()
|
||||||
|
self._params = params
|
||||||
|
|
||||||
|
self._client = VonageClient(application_id, session_id, token, params)
|
||||||
|
|
||||||
|
# Register supported handlers.
|
||||||
|
self._register_event_handler("on_joined")
|
||||||
|
self._register_event_handler("on_left")
|
||||||
|
self._register_event_handler("on_error")
|
||||||
|
self._register_event_handler("on_client_connected", sync=True)
|
||||||
|
self._register_event_handler("on_client_disconnected")
|
||||||
|
self._register_event_handler("on_first_participant_joined", sync=True)
|
||||||
|
self._register_event_handler("on_participant_joined", sync=True)
|
||||||
|
self._register_event_handler("on_participant_left")
|
||||||
|
|
||||||
|
self._client.add_listener(
|
||||||
|
VonageClientListener(
|
||||||
|
on_connected=self._on_connected,
|
||||||
|
on_disconnected=self._on_disconnected,
|
||||||
|
on_error=self._on_error,
|
||||||
|
on_stream_received=self._on_stream_received,
|
||||||
|
on_stream_dropped=self._on_stream_dropped,
|
||||||
|
on_subscriber_connected=self._on_subscriber_connected,
|
||||||
|
on_subscriber_disconnected=self._on_subscriber_disconnected,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
self._input: Optional[VonageVideoConnectorInputTransport] = None
|
||||||
|
self._output: Optional[VonageVideoConnectorOutputTransport] = None
|
||||||
|
self._one_stream_received: bool = False
|
||||||
|
|
||||||
|
def input(self) -> FrameProcessor:
|
||||||
|
"""Get the input transport for Vonage.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The VonageVideoConnectorInputTransport instance.
|
||||||
|
"""
|
||||||
|
if not self._input:
|
||||||
|
self._input = VonageVideoConnectorInputTransport(self._client, self._params)
|
||||||
|
return self._input
|
||||||
|
|
||||||
|
def output(self) -> FrameProcessor:
|
||||||
|
"""Get the output transport for Vonage.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The VonageVideoConnectorOutputTransport instance.
|
||||||
|
"""
|
||||||
|
if not self._output:
|
||||||
|
self._output = VonageVideoConnectorOutputTransport(self._client, self._params)
|
||||||
|
return self._output
|
||||||
|
|
||||||
|
async def subscribe_to_stream(self, stream_id: str, params: SubscribeSettings) -> None:
|
||||||
|
"""Subscribe to a participant's stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
stream_id: The ID of the participant to subscribe to.
|
||||||
|
params: Subscription parameters for the subscription.
|
||||||
|
"""
|
||||||
|
if self._input:
|
||||||
|
await self._input.subscribe_to_stream(stream_id, params)
|
||||||
|
|
||||||
|
async def _on_connected(self, session: Session) -> None:
|
||||||
|
"""Handle session connected event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
session: The connected Session object.
|
||||||
|
"""
|
||||||
|
await self._call_event_handler("on_joined", {"sessionId": session.id})
|
||||||
|
|
||||||
|
async def _on_disconnected(self, session: Session) -> None:
|
||||||
|
"""Handle session disconnected event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
session: The disconnected Session object.
|
||||||
|
"""
|
||||||
|
await self._call_event_handler("on_left", {"sessionId": session.id})
|
||||||
|
|
||||||
|
async def _on_error(self, _session: Session, description: str, _code: int) -> None:
|
||||||
|
"""Handle session error event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
_session: The Session object.
|
||||||
|
description: Error description.
|
||||||
|
_code: Error code.
|
||||||
|
"""
|
||||||
|
await self._call_event_handler("on_error", description)
|
||||||
|
|
||||||
|
async def _on_stream_received(self, session: Session, stream: Stream) -> None:
|
||||||
|
"""Handle stream received event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
session: The Session object.
|
||||||
|
stream: The received Stream object.
|
||||||
|
"""
|
||||||
|
client = {
|
||||||
|
"sessionId": session.id,
|
||||||
|
"streamId": stream.id,
|
||||||
|
"connectionData": stream.connection.data,
|
||||||
|
}
|
||||||
|
if not self._one_stream_received:
|
||||||
|
self._one_stream_received = True
|
||||||
|
await self._call_event_handler("on_first_participant_joined", client)
|
||||||
|
|
||||||
|
await self._call_event_handler("on_participant_joined", client)
|
||||||
|
|
||||||
|
async def _on_stream_dropped(self, session: Session, stream: Stream) -> None:
|
||||||
|
"""Handle stream dropped event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
session: The Session object.
|
||||||
|
stream: The dropped Stream object.
|
||||||
|
"""
|
||||||
|
client = {
|
||||||
|
"sessionId": session.id,
|
||||||
|
"streamId": stream.id,
|
||||||
|
"connectionData": stream.connection.data,
|
||||||
|
}
|
||||||
|
await self._call_event_handler("on_participant_left", client)
|
||||||
|
|
||||||
|
async def _on_subscriber_connected(self, subscriber: Subscriber) -> None:
|
||||||
|
"""Handle subscriber connected event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
subscriber: The connected Subscriber object.
|
||||||
|
"""
|
||||||
|
await self._call_event_handler(
|
||||||
|
"on_client_connected",
|
||||||
|
{
|
||||||
|
"subscriberId": subscriber.stream.id,
|
||||||
|
"streamId": subscriber.stream.id,
|
||||||
|
"connectionData": subscriber.stream.connection.data,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _on_subscriber_disconnected(self, subscriber: Subscriber) -> None:
|
||||||
|
"""Handle subscriber disconnected event.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
subscriber: The disconnected Subscriber object.
|
||||||
|
"""
|
||||||
|
await self._call_event_handler(
|
||||||
|
"on_client_disconnected",
|
||||||
|
{
|
||||||
|
"subscriberId": subscriber.stream.id,
|
||||||
|
"streamId": subscriber.stream.id,
|
||||||
|
"connectionData": subscriber.stream.connection.data,
|
||||||
|
},
|
||||||
|
)
|
||||||
3101
tests/test_vonage_video_connector.py
Normal file
3101
tests/test_vonage_video_connector.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user