diff --git a/env.example b/env.example index 6d69cc0e9..11a7e606e 100644 --- a/env.example +++ b/env.example @@ -211,6 +211,11 @@ TWILIO_AUTH_TOKEN=... # Ultravox Realtime ULTRAVOX_API_KEY=... +# Vonage +VONAGE_APPLICATION_ID=... +VONAGE_SESSION_ID=... +VONAGE_TOKEN=... + # WhatsApp WHATSAPP_TOKEN=... WHATSAPP_WEBHOOK_VERIFICATION_TOKEN=... diff --git a/examples/README.md b/examples/README.md index 5ec86002e..8d3ff3f1f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -55,6 +55,20 @@ Then, run the example with: uv run getting-started/06-voice-agent.py -t twilio -x NGROK_HOST_NAME ``` +### Vonage + +It is also possible to run the example through a Vonage session. Just provide the values for the following variables in +the `.env` file: +* VONAGE_APPLICATION_ID +* VONAGE_SESSION_ID +* VONAGE_TOKEN + +Then, run the example with: + +```bash +uv run getting-started/06-voice-agent.py -t vonage +``` + ## Directory Structure ### [`getting-started/`](./getting-started/) diff --git a/examples/transports/transports-vonage.py b/examples/transports/transports-vonage.py new file mode 100644 index 000000000..774ca1696 --- /dev/null +++ b/examples/transports/transports-vonage.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Example of using AWS Nova Sonic LLM service with Vonage Video Connector transport.""" + +import asyncio +import os +import sys +from collections.abc import Callable +from typing import Any + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.runner.vonage import configure +from pipecat.services.aws.nova_sonic.llm import AWSNovaSonicLLMService +from pipecat.transports.vonage.video_connector import ( + VonageVideoConnectorTransport, + VonageVideoConnectorTransportParams, +) + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main() -> None: + """Main entry point for the nova sonic vonage video connector example.""" + (application_id, session_id, token) = await configure() + + system_instruction = ( + "You are a friendly assistant. The user and you will engage in a spoken dialog exchanging " + "the transcripts of a natural real-time conversation. Keep your responses short, generally " + "two or three sentences for chatty scenarios. " + f"{AWSNovaSonicLLMService.AWAIT_TRIGGER_ASSISTANT_RESPONSE_INSTRUCTION}" + ) + transport = VonageVideoConnectorTransport( + application_id, + session_id, + token, + VonageVideoConnectorTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + publisher_name="Bot", + ), + ) + + llm = AWSNovaSonicLLMService( + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""), + access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""), + region=os.getenv("AWS_REGION", ""), + session_token=os.getenv("AWS_SESSION_TOKEN", ""), + voice_id="tiffany", + ) + context = LLMContext( + messages=[ + {"role": "system", "content": f"{system_instruction}"}, + { + "role": "user", + "content": "Tell me a fun fact!", + }, + ], + ) + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, user_params=LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer()) + ) + + pipeline = Pipeline( + [ + transport.input(), + user_aggregator, + llm, + transport.output(), + assistant_aggregator, + ] + ) + + task = PipelineTask(pipeline) + + # Handle client connection event + event_handler: Callable[[str], Callable[[Any], Any]] = transport.event_handler + + @event_handler("on_client_connected") + async def on_client_connected(transport: VonageVideoConnectorTransport, client: object) -> None: + logger.info(f"Client connected") + await task.queue_frames([LLMRunFrame()]) + + runner = PipelineRunner() + + await asyncio.gather(runner.run(task)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 7206dae00..6edfb9cb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,6 +119,7 @@ tavus = [ "pipecat-ai[daily]" ] together = [] tracing = [ "opentelemetry-sdk>=1.33.0,<2", "opentelemetry-api>=1.33.0,<2", "opentelemetry-instrumentation>=0.54b0,<1" ] ultravox = [ "pipecat-ai[websockets-base]" ] +vonage-video-connector = [ "vonage-video-connector~=0.2.3b0; python_full_version>='3.13' and python_full_version<'3.14' and platform_system=='Linux'" ] webrtc = [ "aiortc>=1.14.0,<2", "opencv-python>=4.11.0.86,<5" ] websocket = [ "pipecat-ai[websockets-base]", "fastapi>=0.115.6,<1" ] websockets-base = [ "websockets>=13.1,<16.0" ] diff --git a/src/pipecat/runner/run.py b/src/pipecat/runner/run.py index 62f3b2d51..872dca5af 100644 --- a/src/pipecat/runner/run.py +++ b/src/pipecat/runner/run.py @@ -108,8 +108,10 @@ from pipecat.runner.types import ( DailyRunnerArguments, RunnerArguments, SmallWebRTCRunnerArguments, + VonageRunnerArguments, WebSocketRunnerArguments, ) +from pipecat.runner.vonage import configure as configure_vonage try: import uvicorn @@ -983,6 +985,25 @@ async def _run_daily_direct(args: argparse.Namespace): await bot_module.bot(runner_args) +async def _run_vonage(): + """Run Vonage bot (no FastAPI server).""" + logger.info("Running Vonage transport...") + + application_id, session_id, token = await configure_vonage() + runner_args = VonageRunnerArguments( + application_id=application_id, session_id=session_id, token=token + ) + runner_args.handle_sigint = True + + # Get the bot module and run it directly + bot_module = _get_bot_module() + + print(f"Joining Vonage session: {runner_args.session_id}") + print() + + await bot_module.bot(runner_args) + + def _validate_and_clean_proxy(proxy: str) -> str: """Validate and clean proxy hostname, removing protocol if present.""" if not proxy: @@ -1062,7 +1083,7 @@ def main(parser: argparse.ArgumentParser | None = None): "-t", "--transport", type=str, - choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS], + choices=["daily", "vonage", "webrtc", *TELEPHONY_TRANSPORTS], default=None, help=( "Restrict the server to a single transport and set it as the default for /start. " @@ -1169,6 +1190,12 @@ def main(parser: argparse.ArgumentParser | None = None): if args.proxy: print(f" → XML webhook: http://{args.host}:{args.port}/") print(f" → WebSocket: ws://{args.host}:{args.port}/ws") + elif args.transport == "vonage": + print() + print(f"🚀 Bot ready!") + asyncio.run(_run_vonage()) + print() + return print() RUNNER_DOWNLOADS_FOLDER = args.folder diff --git a/src/pipecat/runner/types.py b/src/pipecat/runner/types.py index b6bdd0014..6428ee507 100644 --- a/src/pipecat/runner/types.py +++ b/src/pipecat/runner/types.py @@ -99,6 +99,21 @@ class DailyRunnerArguments(RunnerArguments): token: str | None = None +@dataclass +class VonageRunnerArguments(RunnerArguments): + """Daily transport session arguments for the runner. + + Parameters: + application_id: Vonage application ID + session_id: Vonage session ID + token: Vonage Session Token + """ + + application_id: str + session_id: str + token: str + + @dataclass class WebSocketRunnerArguments(RunnerArguments): """WebSocket transport session arguments for the runner. diff --git a/src/pipecat/runner/utils.py b/src/pipecat/runner/utils.py index 19f2e6a50..2334f6da6 100644 --- a/src/pipecat/runner/utils.py +++ b/src/pipecat/runner/utils.py @@ -33,7 +33,7 @@ import json import os import re from collections.abc import Callable -from typing import Any +from typing import Any, cast from fastapi import WebSocket from loguru import logger @@ -42,9 +42,10 @@ from pipecat.runner.types import ( DailyRunnerArguments, LiveKitRunnerArguments, SmallWebRTCRunnerArguments, + VonageRunnerArguments, WebSocketRunnerArguments, ) -from pipecat.transports.base_transport import BaseTransport +from pipecat.transports.base_transport import BaseTransport, TransportParams def _detect_transport_type_from_message(message_data: dict) -> str: @@ -271,6 +272,14 @@ def get_transport_client_id(transport: BaseTransport, client: Any) -> str: except ImportError: pass + try: + from pipecat.transports.vonage.video_connector import VonageVideoConnectorTransport + + if isinstance(transport, VonageVideoConnectorTransport): + return client["streamId"] + except ImportError: + pass + logger.warning(f"Unable to get client id from unsupported transport {type(transport)}") return "" @@ -303,6 +312,24 @@ async def maybe_capture_participant_camera( except ImportError: pass + try: + from pipecat.transports.vonage.video_connector import ( + SubscribeSettings, + VonageVideoConnectorTransport, + ) + + if isinstance(transport, VonageVideoConnectorTransport): + await transport.subscribe_to_stream( + client["streamId"], + SubscribeSettings( + subscribe_to_audio=True, + subscribe_to_video=True, + preferred_framerate=framerate if framerate != 0 else None, + ), + ) + except ImportError: + pass + async def maybe_capture_participant_screen( transport: BaseTransport, client: Any, framerate: int = 0 @@ -534,6 +561,11 @@ async def create_transport( audio_out_enabled=True, # add_wav_header and serializer will be set automatically ), + "vonage": lambda: VonageVideoConnectorParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), } transport = await create_transport(runner_args, transport_params) @@ -587,6 +619,31 @@ async def create_transport( runner_args.room_name, params=params, ) + elif isinstance(runner_args, VonageRunnerArguments): + from pipecat.transports.vonage.video_connector import ( + VonageVideoConnectorTransport, + VonageVideoConnectorTransportParams, + ) + try: + params = cast( + VonageVideoConnectorTransportParams, + _get_transport_params("vonage", transport_params), + ) + except ValueError: + webrtc_params: TransportParams = cast( + TransportParams, _get_transport_params("webrtc", transport_params) + ) + params = VonageVideoConnectorTransportParams( + **webrtc_params.model_dump(), + video_in_auto_subscribe=True, + ) + + return VonageVideoConnectorTransport( + runner_args.application_id, + runner_args.session_id, + runner_args.token, + params=params, + ) else: raise ValueError(f"Unsupported runner arguments type: {type(runner_args)}") diff --git a/src/pipecat/runner/vonage.py b/src/pipecat/runner/vonage.py new file mode 100644 index 000000000..e722c63f1 --- /dev/null +++ b/src/pipecat/runner/vonage.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Vonage session configuration utilities. + +This module extracts the necessary parameters to connect to a Vonage Video session. + +Required environment variables: + +- VONAGE_APPLICATION_ID - Vonage application ID +- VONAGE_SESSION_ID - Vonage session ID +- VONAGE_TOKEN - Vonage token + +Example: + from pipecat.runner.vonage import configure + + application_id, session_id, token = await configure() +""" + +import os + + +async def configure() -> tuple[str, str, str]: + """Configure Vonage application ID, session ID and token from environment. + + Returns: + Tuple containing the server application_id, session_id and token. + + Raises: + Exception: If required Vonage configuration is not provided. + """ + application_id = os.getenv("VONAGE_APPLICATION_ID") + session_id = os.getenv("VONAGE_SESSION_ID") + token = os.getenv("VONAGE_TOKEN") + + if not application_id: + raise Exception( + "No Vonage application ID specified. Use set VONAGE_APPLICATION_ID in your environment." + ) + + if not session_id: + raise Exception( + "No Vonage Session ID specified. Use set VONAGE_SESSION_ID in your environment." + ) + + if not token: + raise Exception("No Vonage token specified. Use set VONAGE_TOKEN in your environment.") + + return (application_id, session_id, token) diff --git a/src/pipecat/transports/vonage/__init__.py b/src/pipecat/transports/vonage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/transports/vonage/client.py b/src/pipecat/transports/vonage/client.py new file mode 100644 index 000000000..c6b703793 --- /dev/null +++ b/src/pipecat/transports/vonage/client.py @@ -0,0 +1,1092 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +"""Vonage Video Connector client.""" + +import asyncio +import itertools +import threading +from collections.abc import Coroutine +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, replace +from datetime import datetime, timedelta +from enum import StrEnum +from typing import Any, Awaitable, Callable, Optional, TypeVar + +import numpy as np +from loguru import logger + +from pipecat.audio.utils import create_stream_resampler +from pipecat.frames.frames import ( + InputAudioRawFrame, + OutputAudioRawFrame, + OutputImageRawFrame, + StartFrame, + UserImageRawFrame, +) +from pipecat.processors.frame_processor import FrameProcessorSetup +from pipecat.transports.base_transport import TransportParams +from pipecat.transports.vonage.utils import ( + AudioProps, + ImageFormat, + check_audio_data, + image_colorspace_conversion, + process_audio, +) +from pipecat.utils.asyncio.task_manager import BaseTaskManager + +try: + import vonage_video_connector as vonage_video + from vonage_video_connector.models import ( + AudioData, + Connection, + LoggingSettings, + Publisher, + PublisherAudioSettings, + PublisherSettings, + SessionAudioSettings, + SessionAVSettings, + SessionSettings, + SessionVideoPublisherSettings, + SubscriberSettings, + SubscriberVideoSettings, + VideoFrame, + VideoResolution, + ) + + # the following "as" imports help to make explicit the re-exporting of these types and avoid type checking warnings + # when re-importing these types from this module + from vonage_video_connector.models import ( + Session as Session, + ) + from vonage_video_connector.models import ( + Stream as Stream, + ) + from vonage_video_connector.models import ( + Subscriber as Subscriber, + ) +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + f"In order to use Vonage Video Connector, you need to have the Vonage Video Connector python library installed." + ) + raise Exception(f"Missing module: {e}") + + +class VonageVideoConnectorTransportParams(TransportParams): + """Parameters for the Vonage Video Connector transport. + + Parameters: + publisher_name: Name of the publisher stream. + publisher_enable_opus_dtx: Whether to enable OPUS DTX for publisher audio. + session_enable_migration: Whether to enable session migration. + audio_in_auto_subscribe: Whether to automatically subscribe to audio streams. + video_in_auto_subscribe: Whether to automatically subscribe to video streams. + video_in_preferred_width: Preferred width for video input capture. + video_in_preferred_height: Preferred height for video input capture. + video_in_preferred_framerate: Preferred framerate for video input capture. + clear_buffers_on_interruption: Whether to clear media buffers when an interruption frame is received. + """ + + publisher_name: str = "Bot" + publisher_enable_opus_dtx: bool = False + session_enable_migration: bool = False + audio_in_auto_subscribe: bool = True + video_in_auto_subscribe: bool = False + video_connector_log_level: str = "INFO" + video_in_preferred_resolution: Optional[tuple[int, int]] = None + video_in_preferred_framerate: Optional[int] = None + clear_buffers_on_interruption: bool = True + + +@dataclass +class SubscribeSettings: + """Parameters for stream input subscription. + + Parameters: + capture_audio: Whether to subscribe to audio. + capture_video: Whether to subscribe to video. + preferred_resolution: Preferred resolution for video subscription. + preferred_framerate: Preferred framerate for video subscription. + """ + + subscribe_to_audio: bool = True + subscribe_to_video: bool = False + preferred_resolution: Optional[tuple[int, int]] = None + preferred_framerate: Optional[int] = None + + +class VonageException(Exception): + """Exception raised when a Vonage transport operation fails or encounters an error.""" + + pass + + +async def async_noop(*args: Any, **kwargs: Any) -> None: + """No operation async function.""" + pass + + +@dataclass +class VonageClientListener: + """Listener for Vonage client events. + + Parameters: + on_connected: Async callback when session is connected. + on_disconnected: Async callback when session is disconnected. + on_error: Async callback for session errors. + on_audio_in: Async callback for incoming audio data. + on_stream_received: Async callback when a stream is received. + on_stream_dropped: Async callback when a stream is dropped. + on_subscriber_connected: Async callback when a subscriber connects. + on_subscriber_disconnected: Async callback when a subscriber disconnects. + """ + + on_connected: Callable[[Session], Awaitable[None]] = async_noop + on_disconnected: Callable[[Session], Awaitable[None]] = async_noop + on_error: Callable[[Session, str, int], Awaitable[None]] = async_noop + on_audio_in: Callable[[Session, InputAudioRawFrame], Awaitable[None]] = async_noop + on_stream_received: Callable[[Session, Stream], Awaitable[None]] = async_noop + on_stream_dropped: Callable[[Session, Stream], Awaitable[None]] = async_noop + on_subscriber_connected: Callable[[Subscriber], Awaitable[None]] = async_noop + on_subscriber_disconnected: Callable[[Subscriber], Awaitable[None]] = async_noop + on_subscriber_video_in: Callable[[Subscriber, VideoFrame], Awaitable[None]] = async_noop + on_video_in: Callable[[Subscriber, UserImageRawFrame], Awaitable[None]] = async_noop + + +# the following StrEnum's don't use auto() to use the right capitalization + + +class VonageImageFormat(StrEnum): + """Enum for Vonage image formats.""" + + YUV420P = "YUV420P" + RGB24 = "RGB24" + ARGB32 = "ARGB32" + + +class PipecatImageFormat(StrEnum): + """Enum for Pipecat image formats.""" + + RGB = "RGB" + RGBA = "RGBA" + YCbCr = "YCbCr" + + +PIPECAT_TO_STANDARD_FORMAT_MAP: dict[PipecatImageFormat, ImageFormat] = { + PipecatImageFormat.YCbCr: ImageFormat.PACKED_YUV444, + PipecatImageFormat.RGB: ImageFormat.RGB, + PipecatImageFormat.RGBA: ImageFormat.RGBA, +} + +VONAGE_TO_STANDARD_FORMAT_MAP: dict[VonageImageFormat, ImageFormat] = { + VonageImageFormat.YUV420P: ImageFormat.PLANAR_YUV420, + VonageImageFormat.RGB24: ImageFormat.BGR, + VonageImageFormat.ARGB32: ImageFormat.BGRA, +} + +VONAGE_TO_PIPECAT_ANALOG_FORMAT_MAP: dict[VonageImageFormat, PipecatImageFormat] = { + VonageImageFormat.YUV420P: PipecatImageFormat.YCbCr, + VonageImageFormat.RGB24: PipecatImageFormat.RGB, + VonageImageFormat.ARGB32: PipecatImageFormat.RGBA, +} + +PIPECAT_TO_VONAGE_ANALOG_FORMAT_MAP: dict[PipecatImageFormat, VonageImageFormat] = { + v: k for k, v in VONAGE_TO_PIPECAT_ANALOG_FORMAT_MAP.items() +} + + +VIDEO_CONNECTOR_TIMEOUT: timedelta = timedelta(seconds=30) +DEFAULT_SAMPLE_RATE: int = 48000 + +AUDIO_QUEUE_MAXSIZE: int = 500 +VIDEO_QUEUE_MAXSIZE: int = 50 + +TA = TypeVar("TA", InputAudioRawFrame, OutputAudioRawFrame) +TE = TypeVar("TE", bound=StrEnum) +SimpleCoroutine = Coroutine[Any, Any, None] + +DUMMY_CONNECTION = Connection(id="", creation_time=datetime.min) + + +def _to_enum(value: Optional[str], enum_cls: type[TE]) -> Optional[TE]: + """Convert a string value to the specified StrEnum type, returning None if invalid.""" + try: + return enum_cls(value or "") + except ValueError: + return None + + +class VonageClient: + """Client for managing a Vonage Video session. + + Handles connection, publishing, subscribing, and event callbacks for a Vonage Video session. + """ + + def __init__( + self, + application_id: str, + session_id: str, + token: str, + params: VonageVideoConnectorTransportParams, + ): + """Initialize the Vonage client. + + Args: + application_id: The Vonage Video application ID. + session_id: The session ID to connect to. + token: The authentication token for the session. + params: Parameters to configure the Vonage client. + """ + self._client = vonage_video.VonageVideoClient() + self._application_id: str = application_id + self._session_id: str = session_id + self._token: str = token + self._params = params.model_copy( + # make sure we have auto-subscribe only if the respective media is enabled + update={ + "audio_in_auto_subscribe": params.audio_in_auto_subscribe + and params.audio_in_enabled, + "video_in_auto_subscribe": params.video_in_auto_subscribe + and params.video_in_enabled, + } + ) + # having these two settings separately to make them non-optional + self._audio_in_sample_rate = params.audio_in_sample_rate or DEFAULT_SAMPLE_RATE + self._audio_out_sample_rate = params.audio_out_sample_rate or DEFAULT_SAMPLE_RATE + + self._connected: bool = False + self._connection_counter: int = 0 + self._connecting_future: Optional[asyncio.Future[None]] = None + self._disconnecting_future: Optional[asyncio.Future[None]] = None + + self._listener_id_gen: itertools.count[int] = itertools.count() + self._listeners: dict[int, VonageClientListener] = {} + + self._publisher: Optional[Publisher] = None + self._session = Session(id=session_id) + + self._resampler = create_stream_resampler() + + self._task_manager: Optional[BaseTaskManager] = None + self._loop_thread_id = threading.get_ident() + self._event_queue: Optional[asyncio.Queue[SimpleCoroutine]] = None + self._event_task: Optional[asyncio.Task[None]] = None + self._audio_queue: Optional[asyncio.Queue[SimpleCoroutine]] = None + self._audio_task: Optional[asyncio.Task[None]] = None + self._video_queue: Optional[asyncio.Queue[SimpleCoroutine]] = None + self._video_task: Optional[asyncio.Task[None]] = None + + # used for blocking calls to connect and disconnect + self._executor = ThreadPoolExecutor(max_workers=1) + + self._session_streams: dict[str, Stream] = {} + self._session_subscriptions: dict[str, SubscribeSettings] = {} + out_pipecat_format = _to_enum(params.video_out_color_format or "RGB", PipecatImageFormat) + if out_pipecat_format is None: + raise VonageException( + f"Unsupported Pipecat output color format: {params.video_out_color_format}" + ) + self._out_pipecat_format: PipecatImageFormat = out_pipecat_format + self._video_out_color_format_vonage: VonageImageFormat = ( + PIPECAT_TO_VONAGE_ANALOG_FORMAT_MAP[self._out_pipecat_format] + ) + self._video_out_color_format: ImageFormat = VONAGE_TO_STANDARD_FORMAT_MAP[ + self._video_out_color_format_vonage + ] + + async def setup(self, setup: FrameProcessorSetup) -> None: + """Setup the client with task manager and event queues. + + Args: + setup: The frame processor setup configuration. + """ + if self._task_manager: + return + + self._task_manager = setup.task_manager + + # tasks from the generic event queue should allow concurrent processing as they + # may await on new events posted to the same queue + self._event_queue = asyncio.Queue() + self._event_task = self._task_manager.create_task( + self._sdk_cb_to_loop_task_handler(self._event_queue, allow_concurrent=True), + f"event_callback_task", + ) + # audio and video tasks should be processed one at a time + self._audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE) + self._audio_task = self._task_manager.create_task( + self._sdk_cb_to_loop_task_handler(self._audio_queue, allow_concurrent=False), + f"audio_callback_task", + ) + self._video_queue = asyncio.Queue(maxsize=VIDEO_QUEUE_MAXSIZE) + self._video_task = self._task_manager.create_task( + self._sdk_cb_to_loop_task_handler(self._video_queue, allow_concurrent=False), + f"video_callback_task", + ) + + async def cleanup(self) -> None: + """Cleanup the client, disconnecting if necessary.""" + if self._connected: + await self.disconnect() + + if self._event_task and self._task_manager: + await self._task_manager.cancel_task(self._event_task) + await self._event_task + self._event_task = None + if self._audio_task and self._task_manager: + await self._task_manager.cancel_task(self._audio_task) + await self._audio_task + self._audio_task = None + if self._video_task and self._task_manager: + await self._task_manager.cancel_task(self._video_task) + await self._video_task + self._video_task = None + + def add_listener(self, listener: VonageClientListener) -> int: + """Add a listener to the Vonage client. + + Args: + listener: The VonageClientListener to add. + + Returns: + The unique ID assigned to the listener. + """ + listener_id = next(self._listener_id_gen) + self._listeners[listener_id] = listener + return listener_id + + def remove_listener(self, listener_id: int) -> None: + """Remove a listener from the Vonage client. + + Args: + listener_id: The ID of the listener to remove. + """ + self._listeners.pop(listener_id, None) + + async def connect(self, frame: Optional[StartFrame] = None) -> None: + """Connect to the Vonage session. + + Args: + frame: Optional StartFrame to configure audio sample rates if not already set. + """ + logger.info(f"Connecting with session string {self._session_id}") + + if self._disconnecting_future is not None: + logger.info( + f"Waiting for disconnection to complete before connecting to {self._session_id}" + ) + await self._disconnecting_future + + if self._connected: + logger.info(f"Already connected to {self._session_id}") + self._connection_counter += 1 + return + + if self._connecting_future is not None: + logger.info(f"Already connecting to {self._session_id}") + + # if we already connecting, await for the publish ready event + await self._connecting_future + self._connection_counter += 1 + return + + # Set audio sample rates from StartFrame if params are not set + if frame: + if self._params.audio_in_sample_rate is None: + self._audio_in_sample_rate = frame.audio_in_sample_rate + if self._params.audio_out_sample_rate is None: + self._audio_out_sample_rate = frame.audio_out_sample_rate + + # this future will allow concurrent calls to connect to wait until the first connect call is done + self._connecting_future = self._get_event_loop().create_future() + + try: + await self._sdk_connect() + except Exception as exc: + logger.error(f"Error connecting to Vonage session: {exc}") + future = self._connecting_future + self._connecting_future = None + future.cancel() + raise exc + + logger.info(f"Connected to {self._session_id}") + self._connected = True + self._connection_counter += 1 + + # all concurrent calls to connect can now proceed + future = self._connecting_future + self._connecting_future = None + future.set_result(None) + + await self._notify_listeners(lambda listener: listener.on_connected(self._session)) + + async def disconnect(self) -> None: + """Disconnect from the Vonage session.""" + if self._connecting_future is not None: + logger.info( + f"Waiting for connection to complete before disconnecting from {self._session_id}" + ) + await self._connecting_future + + if not self._connected: + logger.info(f"Already disconnected from {self._session_id}") + return + + self._connection_counter -= 1 + if self._connection_counter != 0: + logger.info( + f"{self._connection_counter} connections still active for {self._session_id}" + ) + return + + self._disconnecting_future = self._get_event_loop().create_future() + + logger.info(f"Disconnecting from {self._session_id}") + + # ensure we clear up any pending SDK callback events and media buffers + if self._event_queue: + self._clear_queue(self._event_queue) + self.clear_media_buffers() + self._session_streams.clear() + self._session_subscriptions.clear() + + try: + await self._sdk_disconnect() + except Exception as exc: + logger.error(f"Error disconnecting from {self._session_id}: {exc}") + future = self._disconnecting_future + self._disconnecting_future = None + self._connection_counter += 1 + future.cancel() + raise + + logger.info(f"Disconnected from {self._session_id}") + + self._connected = False + + future = self._disconnecting_future + self._disconnecting_future = None + future.set_result(None) + + await self._notify_listeners(lambda listener: listener.on_disconnected(self._session)) + + def clear_media_buffers(self) -> None: + """Clear output media buffers in the Vonage session.""" + logger.debug(f"Clearing media buffers {self._session_id}") + if self._audio_queue: + self._clear_queue(self._audio_queue) + if self._video_queue: + self._clear_queue(self._video_queue) + self._client.clear_media_buffers() + + async def write_audio(self, audio_frame: OutputAudioRawFrame) -> bool: + """Write audio data to the Vonage session. + + Args: + audio_frame: Audio frame to write + """ + target_audio_props = AudioProps( + sample_rate=self._audio_out_sample_rate, + is_stereo=self._params.audio_out_channels == 2, + ) + proc_audio_frame = await self._process_audio_if_needed(audio_frame, target_audio_props) + + return self._client.add_audio( + AudioData( + sample_buffer=memoryview(proc_audio_frame.audio).cast("h"), + number_of_frames=proc_audio_frame.num_frames, + number_of_channels=self._params.audio_out_channels, + sample_rate=self._audio_out_sample_rate, + ) + ) + + async def subscribe_to_stream(self, stream_id: str, params: SubscribeSettings) -> None: + """Subscribe to a participant's stream. + + Args: + stream_id: The ID of the participant to subscribe to. + params: Subscription parameters for the subscription. + """ + previous_subscription: None | SubscribeSettings = self._session_subscriptions.get(stream_id) + if previous_subscription and previous_subscription == params: + logger.info(f"Already subscribed to stream {stream_id} with the same parameters") + return + + stream = self._session_streams.get(stream_id, None) or Stream( + id=stream_id, connection=DUMMY_CONNECTION + ) + if previous_subscription and previous_subscription != params: + logger.warning( + f"Already subscribed to stream {stream_id} with different parameters, " + f"re-subscribing with new parameters {params} " + f"(previous parameters were {previous_subscription})" + ) + self._client.unsubscribe(stream) + self._session_subscriptions.pop(stream_id, None) + + await self._sdk_subscribe(stream, params) + + async def write_video(self, frame: OutputImageRawFrame) -> bool: + """Write a video frame to the transport. + + Args: + frame: The output video frame to write. + """ + if not self._check_image_data(frame): + return False + + parsed_from_pipecat_format = _to_enum(frame.format, PipecatImageFormat) + if frame.format and not parsed_from_pipecat_format: + logger.error(f"Unsupported Pipecat image format: {frame.format}") + return False + from_pipecat_format: PipecatImageFormat = ( + parsed_from_pipecat_format or self._out_pipecat_format + ) + from_std_format = PIPECAT_TO_STANDARD_FORMAT_MAP[from_pipecat_format] + + processed_image = image_colorspace_conversion( + frame.image, + size=frame.size, + from_format=from_std_format, + to_format=self._video_out_color_format, + ) + if not processed_image: + logger.error( + f"Could not convert image from {from_std_format} to {self._video_out_color_format}" + ) + return False + + return self._client.add_video( + VideoFrame( + frame_buffer=memoryview(processed_image).cast("B"), + resolution=VideoResolution(width=frame.size[0], height=frame.size[1]), + format=str(self._video_out_color_format_vonage), + ), + ) + + async def _notify_listeners( + self, coroutine_func: Callable[[VonageClientListener], Awaitable[None]] + ) -> None: + """Notify all listeners with the given coroutine function. + + Args: + coroutine_func: The coroutine function to call for each listener. + """ + await asyncio.gather(*(coroutine_func(listener) for listener in self._listeners.values())) + + def _check_image_data(self, frame: OutputImageRawFrame) -> bool: + """Check the image data for validity. + + Args: + frame: The OutputImageRawFrame to check. + """ + res = True + frame_format = _to_enum(frame.format, PipecatImageFormat) + + if frame_format and frame_format != self._out_pipecat_format: + logger.error(f"Expected color format {self._out_pipecat_format}, got {frame_format}") + res = False + if ( + frame.size[0] != self._params.video_out_width + or frame.size[1] != self._params.video_out_height + ): + logger.error( + f"Expected resolution {self._params.video_out_width}x{self._params.video_out_height}, " + f"got {frame.size[0]}x{frame.size[1]}" + ) + res = False + return res + + async def _sdk_connect(self) -> None: + # this future will be set when the session is ready to publish, audio needs a special callback before + # publishing + ready_to_publish_future = self._get_event_loop().create_future() + + def on_session_error_cb(session: Session, description: str, code: int) -> None: + async def async_cb() -> None: + logger.warning(f"Session error {session.id} code={code} description={description}") + if not ready_to_publish_future.done(): + ready_to_publish_future.set_exception( + VonageException(f"Session error: {description} (code {code})") + ) + + await self._notify_listeners( + lambda listener: listener.on_error(session, description, code) + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def on_session_disconnected_cb(session: Session) -> None: + async def async_cb() -> None: + unexpected_disconnection = self._disconnecting_future is None and self._connected + logger.info( + f"Session disconnected {session.id} unexpected={unexpected_disconnection}" + ) + + if not ready_to_publish_future.done(): + ready_to_publish_future.set_exception( + VonageException(f"Got disconnected while waiting for connection") + ) + + if unexpected_disconnection: + await self._notify_listeners( + lambda listener: listener.on_error(session, "unexpected disconnection", -1) + ) + + self._sdk_event_cb_to_loop(async_cb()) + + # this callback will be called when the session is ready to publish audio, video-only doesn't need it + def audio_ready_cb(session: Session) -> None: + async def async_cb() -> None: + logger.info(f"Session {session.id} ready to publish") + if not ready_to_publish_future.done(): + ready_to_publish_future.set_result(None) + + self._sdk_event_cb_to_loop(async_cb()) + + def connect_proc() -> None: + if not self._client.connect( + application_id=self._application_id, + session_id=self._session_id, + token=self._token, + session_settings=SessionSettings( + av=SessionAVSettings( + audio_publisher=SessionAudioSettings( + sample_rate=self._audio_out_sample_rate, + number_of_channels=self._params.audio_out_channels, + ), + audio_subscribers_mix=SessionAudioSettings( + sample_rate=self._audio_in_sample_rate, + number_of_channels=self._params.audio_in_channels, + ), + video_publisher=SessionVideoPublisherSettings( + resolution=VideoResolution( + width=self._params.video_out_width, + height=self._params.video_out_height, + ), + fps=self._params.video_out_framerate, + format=self._video_out_color_format_vonage, + ), + ), + enable_migration=self._params.session_enable_migration, + logging=LoggingSettings(level=self._params.video_connector_log_level), + ), + on_error_cb=on_session_error_cb, + on_connected_cb=self._on_session_connected_cb, + on_disconnected_cb=on_session_disconnected_cb, + on_stream_received_cb=self._on_stream_received_cb, + on_stream_dropped_cb=self._on_stream_dropped_cb, + on_audio_data_cb=self._on_session_audio_data_cb, + on_ready_for_audio_cb=audio_ready_cb, + ): + logger.error(f"Could not connect to {self._session_id}") + raise VonageException("Could not connect to session") + + async def async_proc() -> None: + await self._get_event_loop().run_in_executor(self._executor, connect_proc) + + # when audio publishing is enabled we need to wait for the session to be ready for audio + # however, if only video is being published at this point we don't need to wait anymore + if self._params.audio_out_enabled: + logger.info(f"Waiting for {self._session_id} to be ready to publish audio") + await ready_to_publish_future + else: + ready_to_publish_future.cancel() + + try: + await asyncio.wait_for(async_proc(), timeout=VIDEO_CONNECTOR_TIMEOUT.total_seconds()) + except asyncio.TimeoutError as exc: + logger.error(f"Timeout connecting to Vonage session {self._session_id}") + + raise exc + + async def _sdk_disconnect(self) -> None: + def disconnect_proc() -> None: + if self._publisher: + self._client.unpublish() + self._publisher = None + self._client.disconnect() + + try: + await asyncio.wait_for( + self._get_event_loop().run_in_executor(self._executor, disconnect_proc), + timeout=VIDEO_CONNECTOR_TIMEOUT.total_seconds(), + ) + except asyncio.TimeoutError: + logger.error(f"Timeout disconnecting from Vonage session {self._session_id}") + raise + + async def _sdk_subscribe(self, stream: Stream, params: SubscribeSettings) -> None: + subscribed_future = self._get_event_loop().create_future() + self._session_subscriptions[stream.id] = params + + logger.info(f"Subscribing to stream {stream.id} with params {params}") + + def on_error_cb(subscriber: Subscriber, error: str, code: int) -> None: + async def async_cb() -> None: + logger.error(f"Subscriber {subscriber.stream.id} error: {error} (code {code})") + self._session_subscriptions.pop(subscriber.stream.id, None) + if not subscribed_future.done(): + subscribed_future.set_exception( + VonageException(f"Subscriber error: {error} (code {code})") + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def on_connected_cb(subscriber: Subscriber) -> None: + async def async_cb() -> None: + logger.info(f"Subscriber {subscriber.stream.id} connected") + + if not subscribed_future.done(): + subscribed_future.set_result(None) + await self._notify_listeners( + lambda listener: listener.on_subscriber_connected(subscriber) + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def on_subscriber_disconnected_cb(subscriber: Subscriber) -> None: + async def async_cb() -> None: + logger.info( + f"Subscriber disconnected session={self._session_id} subscriber={subscriber.stream.id} " + ) + self._session_subscriptions.pop(subscriber.stream.id, None) + if not subscribed_future.done(): + subscribed_future.set_exception( + VonageException( + f"Subscriber {subscriber.stream.id} disconnected before connecting" + ) + ) + await self._notify_listeners( + lambda listener: listener.on_subscriber_disconnected(subscriber) + ) + + self._sdk_event_cb_to_loop(async_cb()) + + async def process() -> None: + logger.info( + f"Subscribing to stream {stream.id} audio={params.subscribe_to_audio} " + f"video={params.subscribe_to_video}" + ) + if not self._client.subscribe( + stream, + settings=SubscriberSettings( + subscribe_to_audio=params.subscribe_to_audio, + subscribe_to_video=params.subscribe_to_video, + video_settings=SubscriberVideoSettings( + preferred_resolution=( + VideoResolution( + width=params.preferred_resolution[0], + height=params.preferred_resolution[1], + ) + if params.preferred_resolution + else None + ), + preferred_framerate=params.preferred_framerate, + ), + ), + on_error_cb=on_error_cb, + on_connected_cb=on_connected_cb, + on_disconnected_cb=on_subscriber_disconnected_cb, + on_render_frame_cb=self._on_subscriber_video_data_cb, + ): + subscribed_future.cancel() + raise VonageException(f"Could not subscribe to stream {stream.id}") + + await subscribed_future + + try: + await asyncio.wait_for(process(), timeout=VIDEO_CONNECTOR_TIMEOUT.total_seconds()) + except asyncio.TimeoutError: + logger.error(f"Timeout subscribing to Vonage stream {stream.id}") + self._session_subscriptions.pop(stream.id, None) + raise + + def _sdk_publish(self) -> None: + """Publish the audio and video streams to the Vonage session.""" + if self._params.audio_out_enabled or self._params.video_out_enabled: + logger.info( + f"Publishing audio={self._params.audio_out_enabled} video={self._params.video_out_enabled} " + f"for session {self._session_id}" + ) + # TODO this could be run in the executor pool as it blocks + self._client.publish( + settings=PublisherSettings( + name=self._params.publisher_name, + audio_settings=PublisherAudioSettings( + enable_stereo_mode=self._params.audio_out_channels == 2, + enable_opus_dtx=self._params.publisher_enable_opus_dtx, + ), + has_audio=self._params.audio_out_enabled, + has_video=self._params.video_out_enabled, + ), + on_error_cb=self._on_publisher_error_cb, + on_stream_created_cb=self._on_publisher_stream_created_cb, + on_stream_destroyed_cb=self._on_publisher_stream_destroyed_cb, + ) + else: + logger.info(f"No audio or video to publish for session {self._session_id}") + + @staticmethod + def _clear_queue(queue: asyncio.Queue[SimpleCoroutine]) -> None: + """Clear all items from the given asyncio queue.""" + try: + while True: + item = queue.get_nowait() + # Close coroutines to avoid "never awaited" warnings + item.close() + queue.task_done() + except asyncio.QueueEmpty: + pass + + def _get_event_loop(self) -> asyncio.AbstractEventLoop: + """Get the event loop from the task manager.""" + if not self._task_manager: + raise Exception(f"{self}: missing task manager (pipeline not started?)") + return self._task_manager.get_event_loop() + + async def _sdk_cb_to_loop_task_handler( + self, queue: asyncio.Queue[SimpleCoroutine], allow_concurrent: bool + ) -> None: + """Read coroutines generated from SDK callbacks in the given queue executing them in the event loop.""" + # ensure we know the thread id of the event loop + self._loop_thread_id = threading.get_ident() + # if we allow concurrent tasks, process them as they come in + if allow_concurrent: + active_tasks = set() + + async def wrapped_task(coroutine: SimpleCoroutine) -> None: + try: + await coroutine + except Exception as exc: + logger.error(f"Exception in SDK callback task: {exc}") + finally: + active_tasks.discard(task) + queue.task_done() + + try: + while True: + async_task = await queue.get() + task = asyncio.create_task(wrapped_task(async_task)) + active_tasks.add(task) + except asyncio.CancelledError: + # Cancel all active tasks + for task in active_tasks: + task.cancel() + + # Wait for them to finish cancelling + if active_tasks: + await asyncio.gather(*active_tasks, return_exceptions=True) + # if we only allow one task at a time, process them sequentially + else: + while True: + try: + async_task = await queue.get() + await async_task + queue.task_done() + except asyncio.CancelledError: + break + except Exception as exc: + logger.error(f"Exception in SDK callback task: {exc}") + + def _sdk_event_cb_to_loop(self, callback: SimpleCoroutine) -> None: + """From an SDK thread queue an event coroutine to be asynchronously executed in the task manager event loop.""" + self._sdk_cb_to_loop("event", self._event_queue, callback) + + def _sdk_audio_cb_to_loop(self, callback: SimpleCoroutine) -> None: + """From an SDK thread queue an audio coroutine to be asynchronously executed in the task manager event loop.""" + self._sdk_cb_to_loop("audio", self._audio_queue, callback) + + def _sdk_video_cb_to_loop(self, callback: SimpleCoroutine) -> None: + """From an SDK thread queue a video coroutine to be asynchronously executed in the task manager event loop.""" + self._sdk_cb_to_loop("video", self._video_queue, callback) + + def _sdk_cb_to_loop( + self, + queue_type_name: str, + queue: Optional[asyncio.Queue[SimpleCoroutine]], + async_task: SimpleCoroutine, + ) -> None: + """From an SDK thread queue a coroutine to be asynchronously executed in the task manager event loop. + + If the coroutine queue is full the event will be dropped and a warning logged. + """ + if not queue: + raise Exception(f"missing {queue_type_name} queue (pipeline not started?)") + + def put_coroutine() -> None: + try: + queue.put_nowait(async_task) + except asyncio.QueueFull: + logger.warning( + f"{queue_type_name} queue is full, dropping SDK {queue_type_name} callback." + ) + async_task.close() + + if threading.get_ident() == self._loop_thread_id: + put_coroutine() + else: + self._get_event_loop().call_soon_threadsafe(put_coroutine) + + def _on_session_connected_cb(self, session: Session) -> None: + async def async_cb() -> None: + logger.info(f"Session connected {session.id}") + self._session = session + self._sdk_publish() + + self._sdk_event_cb_to_loop(async_cb()) + + def _on_publisher_error_cb(self, publisher: Publisher, description: str, code: int) -> None: + async def async_cb() -> None: + logger.warning( + f"Publisher error session={self._session_id} publisher={publisher.stream.id} " + f"code={code} description={description}" + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def _on_publisher_stream_created_cb(self, publisher: Publisher) -> None: + async def async_cb() -> None: + logger.info( + f"Publisher stream created session={self._session_id} publisher={publisher.stream.id}" + ) + self._publisher = publisher + + self._sdk_event_cb_to_loop(async_cb()) + + def _on_publisher_stream_destroyed_cb(self, publisher: Publisher) -> None: + async def async_cb() -> None: + logger.info( + f"Publisher stream destroyed session={self._session_id} publisher={publisher.stream.id}" + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def _on_session_audio_data_cb(self, session: Session, audio_data: AudioData) -> None: + """Callback for incoming mixed audio data for all the subscribers in the session.""" + # we need to keep a copy of the audio data as it is a memory view and it will be lost when processed async later + audio_frame = InputAudioRawFrame( + audio=audio_data.sample_buffer.tobytes(), + sample_rate=audio_data.sample_rate, + num_channels=audio_data.number_of_channels, + ) + + async def async_cb() -> None: + target_audio_props = AudioProps( + sample_rate=self._audio_in_sample_rate, + is_stereo=self._params.audio_in_channels == 2, + ) + proc_audio_frame = await self._process_audio_if_needed(audio_frame, target_audio_props) + await self._notify_listeners( + lambda listener: listener.on_audio_in(session, proc_audio_frame) + ) + + self._sdk_audio_cb_to_loop(async_cb()) + + def _on_stream_received_cb(self, session: Session, stream: Stream) -> None: + async def async_cb() -> None: + logger.info(f"Stream received session={session.id} stream={stream.id}") + self._session_streams[stream.id] = stream + + # raise the event before auto subscribing so listeners can decide what to do + await self._notify_listeners( + lambda listener: listener.on_stream_received(session, stream) + ) + + # if we have auto-subscribe enabled, subscribe to the stream if it hasn't been subscribed yet + auto_subscribe = ( + self._params.audio_in_auto_subscribe or self._params.video_in_auto_subscribe + ) + if auto_subscribe and not stream.id in self._session_subscriptions: + await self._sdk_subscribe( + stream, + SubscribeSettings( + subscribe_to_audio=self._params.audio_in_auto_subscribe, + subscribe_to_video=self._params.video_in_auto_subscribe, + preferred_resolution=(self._params.video_in_preferred_resolution), + preferred_framerate=self._params.video_in_preferred_framerate, + ), + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def _on_stream_dropped_cb(self, session: Session, stream: Stream) -> None: + async def async_cb() -> None: + logger.info(f"Stream dropped session={session.id} stream={stream.id}") + if stream.id in self._session_subscriptions: + self._client.unsubscribe(stream) + self._session_subscriptions.pop(stream.id, None) + self._session_streams.pop(stream.id, None) + + await self._notify_listeners( + lambda listener: listener.on_stream_dropped(session, stream) + ) + + self._sdk_event_cb_to_loop(async_cb()) + + def _on_subscriber_video_data_cb(self, subscriber: Subscriber, frame: VideoFrame) -> None: + """Callback for incoming per stream data for all the subscribers in the session.""" + # we need to keep a copy of the audio data as it is a memory view and it will be lost when processed async later + image = frame.frame_buffer.tobytes() + + async def async_cb() -> None: + from_vonage_format = _to_enum(frame.format, VonageImageFormat) + if not from_vonage_format: + logger.error(f"Unsupported Vonage image format: {frame.format}") + return + + from_std_format = VONAGE_TO_STANDARD_FORMAT_MAP[from_vonage_format] + to_pipecat_format = VONAGE_TO_PIPECAT_ANALOG_FORMAT_MAP[from_vonage_format] + to_std_format = PIPECAT_TO_STANDARD_FORMAT_MAP[to_pipecat_format] + + processed_image = image_colorspace_conversion( + image, + size=(frame.resolution.width, frame.resolution.height), + from_format=from_std_format, + to_format=to_std_format, + ) + if not processed_image: + logger.error(f"Could not convert image from {from_std_format} to {to_std_format}") + return + + pipecat_frame = UserImageRawFrame( + user_id=subscriber.stream.id, + image=processed_image, + size=(frame.resolution.width, frame.resolution.height), + format=str(to_pipecat_format), + ) + + await self._notify_listeners( + lambda listener: listener.on_video_in(subscriber, pipecat_frame) + ) + + self._sdk_video_cb_to_loop(async_cb()) + + async def _process_audio_if_needed(self, audio_frame: TA, target_props: AudioProps) -> TA: + check_audio_data(audio_frame.audio, audio_frame.num_frames, audio_frame.num_channels) + + current_audio_props = AudioProps( + sample_rate=audio_frame.sample_rate, + is_stereo=audio_frame.num_channels == 2, + ) + if current_audio_props != target_props: + audio_np = np.frombuffer(audio_frame.audio, dtype=np.int16) + processed_audio_np = await process_audio( + self._resampler, + audio_np, + current_audio_props, + target_props, + ) + + processed_audio_frame = replace( + audio_frame, + audio=processed_audio_np.tobytes(), + sample_rate=target_props.sample_rate, + num_channels=2 if target_props.is_stereo else 1, + ) + return processed_audio_frame + else: + return audio_frame diff --git a/src/pipecat/transports/vonage/utils.py b/src/pipecat/transports/vonage/utils.py new file mode 100644 index 000000000..14d47edd3 --- /dev/null +++ b/src/pipecat/transports/vonage/utils.py @@ -0,0 +1,150 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +"""Vonage Video Connector utils.""" + +from dataclasses import dataclass, replace +from enum import StrEnum + +import numpy as np +import numpy.typing as npt + +from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler + + +@dataclass +class AudioProps: + """Audio properties for normalization. + + Parameters: + sample_rate: The sample rate of the audio. + is_stereo: Whether the audio is stereo (True) or mono (False). + """ + + sample_rate: int + is_stereo: bool + + +class ImageFormat(StrEnum): + """Enum for image formats.""" + + PLANAR_YUV420 = "PLANAR_YUV420" + PACKED_YUV444 = "PACKED_YUV444" + RGB = "RGB" + RGBA = "RGBA" + BGR = "BGR" + BGRA = "BGRA" + + +def check_audio_data( + buffer: bytes | memoryview, number_of_frames: int, number_of_channels: int +) -> None: + """Check the audio sample width based on buffer size, number of frames and channels.""" + if number_of_channels not in (1, 2): + raise ValueError(f"We only accept mono or stereo audio, got {number_of_channels}") + + if isinstance(buffer, memoryview): + bytes_per_sample = buffer.itemsize + else: + bytes_per_sample = len(buffer) // (number_of_frames * number_of_channels) + + if bytes_per_sample != 2: + raise ValueError(f"We only accept 16 bit PCM audio, got {bytes_per_sample * 8} bit") + + +def process_audio_channels( + audio: npt.NDArray[np.int16], current: AudioProps, target: AudioProps +) -> npt.NDArray[np.int16]: + """Normalize audio channels to the target properties.""" + if current.is_stereo != target.is_stereo: + if target.is_stereo: + audio = np.repeat(audio, 2) + else: + audio = audio.reshape(-1, 2).mean(axis=1).astype(np.int16) + + return audio + + +async def process_audio( + resampler: BaseAudioResampler, + audio: npt.NDArray[np.int16], + current: AudioProps, + target: AudioProps, +) -> npt.NDArray[np.int16]: + """Normalize audio to the target properties.""" + res_audio = audio + if current.sample_rate != target.sample_rate: + # first normalize channels to mono if needed, then resample, then normalize channels to target + res_audio = process_audio_channels(res_audio, current, replace(current, is_stereo=False)) + current = replace(current, is_stereo=False) + res_audio_bytes: bytes = await resampler.resample( + res_audio.tobytes(), current.sample_rate, target.sample_rate + ) + res_audio = np.frombuffer(res_audio_bytes, dtype=np.int16) + + res_audio = process_audio_channels(res_audio, current, target) + + return res_audio + + +def image_colorspace_conversion( + image: bytes, size: tuple[int, int], from_format: ImageFormat, to_format: ImageFormat +) -> bytes | None: + """Convert image colorspace from one format to another.""" + match (from_format, to_format): + case (fmt1, fmt2) if fmt1 == fmt2: + return image + case (ImageFormat.RGB, ImageFormat.BGR) | (ImageFormat.BGR, ImageFormat.RGB): + np_input = np.frombuffer(image, dtype=np.uint8) + np_output = np_input.reshape(size[1], size[0], 3)[:, :, ::-1] + return np_output.tobytes() + case (ImageFormat.RGBA, ImageFormat.BGRA) | (ImageFormat.BGRA, ImageFormat.RGBA): + np_input = np.frombuffer(image, dtype=np.uint8) + np_output = np_input.reshape(size[1], size[0], 4)[:, :, [2, 1, 0, 3]] + return np_output.tobytes() + case (ImageFormat.PLANAR_YUV420, ImageFormat.PACKED_YUV444): + # YUV420 (I420) has Y plane of size width*height, U and V planes of size (width/2)*(height/2) + # Packed YUV444 interleaves Y, U, V values for each pixel (YUVYUVYUV...) + width, height = size + y_plane_size = width * height + uv_plane_size_420 = (width // 2) * (height // 2) + + np_input = np.frombuffer(image, dtype=np.uint8) + y_plane = np_input[:y_plane_size].reshape(height, width) + u_plane_420 = np_input[y_plane_size : y_plane_size + uv_plane_size_420].reshape( + height // 2, width // 2 + ) + v_plane_420 = np_input[ + y_plane_size + uv_plane_size_420 : y_plane_size + 2 * uv_plane_size_420 + ].reshape(height // 2, width // 2) + + # Upsample U and V planes by repeating each pixel in 2x2 blocks + u_plane_444 = np.repeat(np.repeat(u_plane_420, 2, axis=0), 2, axis=1) + v_plane_444 = np.repeat(np.repeat(v_plane_420, 2, axis=0), 2, axis=1) + + # Interleave Y, U, V values for packed format (YUVYUVYUV...) + np_output = np.stack([y_plane, u_plane_444, v_plane_444], axis=-1) + return np_output.tobytes() + case (ImageFormat.PACKED_YUV444, ImageFormat.PLANAR_YUV420): + # Packed YUV444 has Y, U, V interleaved (YUVYUVYUV...) + # YUV420 (I420) has Y plane of size width*height, U and V planes of size (width/2)*(height/2) + width, height = size + + np_input = np.frombuffer(image, dtype=np.uint8).reshape(height, width, 3) + y_plane = np_input[:, :, 0].reshape(height, width) + u_plane_444 = np_input[:, :, 1] + v_plane_444 = np_input[:, :, 2] + + # Downsample U and V planes by taking every other pixel (2x2 -> 1 averaging) + u_plane_420 = u_plane_444[::2, ::2].reshape(height // 2, width // 2) + v_plane_420 = v_plane_444[::2, ::2].reshape(height // 2, width // 2) + + # Concatenate Y, U, V planes + np_output = np.concatenate( + [y_plane.flatten(), u_plane_420.flatten(), v_plane_420.flatten()] + ) + return np_output.tobytes() + case _: + return None diff --git a/src/pipecat/transports/vonage/video_connector.py b/src/pipecat/transports/vonage/video_connector.py new file mode 100644 index 000000000..f84fc3406 --- /dev/null +++ b/src/pipecat/transports/vonage/video_connector.py @@ -0,0 +1,483 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +"""Vonage Video Connector transport.""" + +from typing import Optional + +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InputAudioRawFrame, + InterruptionFrame, + OutputAudioRawFrame, + OutputImageRawFrame, + StartFrame, + UserImageRawFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup +from pipecat.transports.base_input import BaseInputTransport +from pipecat.transports.base_output import BaseOutputTransport +from pipecat.transports.base_transport import BaseTransport +from pipecat.transports.vonage.client import ( + Session, + Stream, + Subscriber, + VonageClient, + VonageClientListener, +) + +# the following "as" imports help to re-export these types and avoid type checking warnings +# when importing these types from the main transport module +from pipecat.transports.vonage.client import ( + SubscribeSettings as SubscribeSettings, +) +from pipecat.transports.vonage.client import ( + VonageException as VonageException, +) +from pipecat.transports.vonage.client import ( + VonageVideoConnectorTransportParams as VonageVideoConnectorTransportParams, +) + + +class VonageVideoConnectorInputTransport(BaseInputTransport): + """Input transport for Vonage, handling audio input from the Vonage session. + + Receives audio from a Vonage Video session and pushes it as input frames. + """ + + _params: VonageVideoConnectorTransportParams + + def __init__(self, client: VonageClient, params: VonageVideoConnectorTransportParams): + """Initialize the Vonage input transport. + + Args: + client: The VonageClient instance to use. + params: Transport parameters for input configuration. + """ + super().__init__(params) + self._initialized: bool = False + self._client: VonageClient = client + self._listener_id: int = -1 + self._connected: bool = False + + async def start(self, frame: StartFrame) -> None: + """Start the Vonage input transport. + + Args: + frame: The StartFrame to initiate the transport. + """ + await super().start(frame) + + if self._initialized: + return + + self._initialized = True + + if self._params.audio_in_enabled or self._params.video_in_enabled: + self._listener_id = self._client.add_listener( + VonageClientListener( + on_audio_in=self._audio_in_cb, + on_video_in=self._video_in_cb, + on_error=self._on_error_cb, + ) + ) + try: + await self._client.connect(frame) + self._connected = True + except Exception as exc: + logger.error(f"Error connecting to Vonage session: {exc}") + await self.push_error("Vonage video connector connection error", fatal=True) + return + + await self.set_transport_ready(frame) + + async def setup(self, setup: FrameProcessorSetup) -> None: + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ + await super().setup(setup) + await self._client.setup(setup) + + async def cleanup(self) -> None: + """Cleanup input transport.""" + await super().cleanup() # type: ignore + await self._client.cleanup() + + async def _audio_in_cb(self, _session: Session, audio: InputAudioRawFrame) -> None: + if self._connected and self._params.audio_in_enabled: + await self.push_audio_frame(audio) + + async def _video_in_cb(self, _subscriber: Subscriber, video: UserImageRawFrame) -> None: + if self._connected and self._params.video_in_enabled: + await self.push_video_frame(video) + + async def _on_error_cb(self, session: Session, description: str, code: int) -> None: + logger.error( + f"Vonage input transport error session={session.id} code={code} description={description}" + ) + if self._connected: + await self.push_error("Vonage video connector error", fatal=True) + + async def stop(self, frame: EndFrame) -> None: + """Stop the Vonage input transport. + + Args: + frame: The EndFrame to stop the transport. + """ + await super().stop(frame) + await self._stop_client() + + async def cancel(self, frame: CancelFrame) -> None: + """Cancel the Vonage input transport. + + Args: + frame: The CancelFrame to cancel the transport. + """ + await super().cancel(frame) + await self._stop_client() + + async def _stop_client(self) -> None: + if self._connected: + self._client.remove_listener(self._listener_id) + self._connected = False + try: + await self._client.disconnect() + except Exception: + pass + + async def subscribe_to_stream(self, stream_id: str, params: SubscribeSettings) -> None: + """Subscribe to a participant's stream. + + Args: + stream_id: The ID of the participant to subscribe to. + params: Subscription parameters for the subscription. + """ + await self._client.subscribe_to_stream(stream_id, params) + + +class VonageVideoConnectorOutputTransport(BaseOutputTransport): + """Output transport for Vonage, handling audio output to the Vonage session. + + Sends audio frames to a Vonage Video session as output. + """ + + _params: VonageVideoConnectorTransportParams + + def __init__(self, client: VonageClient, params: VonageVideoConnectorTransportParams): + """Initialize the Vonage output transport. + + Args: + client: The VonageClient instance to use. + params: Transport parameters for output configuration. + """ + super().__init__(params) + self._initialized: bool = False + self._client = client + self._connected: bool = False + self._listener_id: int = -1 + + async def start(self, frame: StartFrame) -> None: + """Start the Vonage output transport. + + Args: + frame: The StartFrame to initiate the transport. + """ + await super().start(frame) + + if self._initialized: + return + + self._initialized = True + + if self._params.audio_out_enabled or self._params.video_out_enabled: + self._listener_id = self._client.add_listener( + VonageClientListener(on_error=self._on_error_cb) + ) + try: + await self._client.connect(frame) + self._connected = True + except Exception as exc: + logger.error(f"Error connecting to Vonage session: {exc}") + await self.push_error("Vonage video connector connection error", fatal=True) + return + + await self.set_transport_ready(frame) + + async def setup(self, setup: FrameProcessorSetup) -> None: + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ + await super().setup(setup) + await self._client.setup(setup) + + async def cleanup(self) -> None: + """Cleanup output transport.""" + await super().cleanup() # type: ignore + await self._client.cleanup() + + async def process_frame(self, frame: Frame, direction: FrameDirection) -> None: + """Process a frame for the Vonage output transport. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + + # if we get an interruption frame, we need to ensure the buffers inside Vonage Video Connector are cleared + if ( + self._connected + and isinstance(frame, InterruptionFrame) + and self._params.clear_buffers_on_interruption + ): + logger.info("Clearing Vonage media buffers due to interruption frame") + self._client.clear_media_buffers() + + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: + """Write an audio frame to the Vonage session. + + Args: + frame: The OutputAudioRawFrame to send. + """ + result = False + if self._connected and self._params.audio_out_enabled: + result = await self._client.write_audio(frame) + + return result + + async def write_video_frame(self, frame: OutputImageRawFrame) -> bool: + """Write a video frame to the transport. + + Args: + frame: The output video frame to write. + """ + result = False + if self._connected and self._params.video_out_enabled: + result = await self._client.write_video(frame) + + return result + + async def stop(self, frame: EndFrame) -> None: + """Stop the Vonage output transport. + + Args: + frame: The EndFrame to stop the transport. + """ + await super().stop(frame) + await self._stop_client() + + async def cancel(self, frame: CancelFrame) -> None: + """Cancel the Vonage output transport. + + Args: + frame: The CancelFrame to cancel the transport. + """ + await super().cancel(frame) + await self._stop_client() + + async def _stop_client(self) -> None: + if self._connected: + self._client.remove_listener(self._listener_id) + self._connected = False + try: + await self._client.disconnect() + except Exception: + pass + + async def _on_error_cb(self, session: Session, description: str, code: int) -> None: + logger.error( + f"Vonage output transport error session={session.id} code={code} description={description}" + ) + if self._connected: + await self.push_error("Vonage video connector error", fatal=True) + + +class VonageVideoConnectorTransport(BaseTransport): + """Vonage Video Connector transport implementation for Pipecat. + + Provides input and output audio transport for Vonage Video sessions, supporting event handling + for session and participant lifecycle. + + Supported features: + + - Audio input and output transport for Vonage Video sessions + - Event handler registration for session and participant events + - Publisher and subscriber management + - Configurable audio and migration parameters + """ + + _params: VonageVideoConnectorTransportParams + + def __init__( + self, + application_id: str, + session_id: str, + token: str, + params: VonageVideoConnectorTransportParams, + ): + """Initialize the Vonage Video Connector transport. + + Args: + application_id: The Vonage Video application ID. + session_id: The session ID to connect to. + token: The authentication token for the session. + params: Transport parameters for input/output configuration. + """ + super().__init__() + self._params = params + + self._client = VonageClient(application_id, session_id, token, params) + + # Register supported handlers. + self._register_event_handler("on_joined") + self._register_event_handler("on_left") + self._register_event_handler("on_error") + self._register_event_handler("on_client_connected", sync=True) + self._register_event_handler("on_client_disconnected") + self._register_event_handler("on_first_participant_joined", sync=True) + self._register_event_handler("on_participant_joined", sync=True) + self._register_event_handler("on_participant_left") + + self._client.add_listener( + VonageClientListener( + on_connected=self._on_connected, + on_disconnected=self._on_disconnected, + on_error=self._on_error, + on_stream_received=self._on_stream_received, + on_stream_dropped=self._on_stream_dropped, + on_subscriber_connected=self._on_subscriber_connected, + on_subscriber_disconnected=self._on_subscriber_disconnected, + ) + ) + + self._input: Optional[VonageVideoConnectorInputTransport] = None + self._output: Optional[VonageVideoConnectorOutputTransport] = None + self._one_stream_received: bool = False + + def input(self) -> FrameProcessor: + """Get the input transport for Vonage. + + Returns: + The VonageVideoConnectorInputTransport instance. + """ + if not self._input: + self._input = VonageVideoConnectorInputTransport(self._client, self._params) + return self._input + + def output(self) -> FrameProcessor: + """Get the output transport for Vonage. + + Returns: + The VonageVideoConnectorOutputTransport instance. + """ + if not self._output: + self._output = VonageVideoConnectorOutputTransport(self._client, self._params) + return self._output + + async def subscribe_to_stream(self, stream_id: str, params: SubscribeSettings) -> None: + """Subscribe to a participant's stream. + + Args: + stream_id: The ID of the participant to subscribe to. + params: Subscription parameters for the subscription. + """ + if self._input: + await self._input.subscribe_to_stream(stream_id, params) + + async def _on_connected(self, session: Session) -> None: + """Handle session connected event. + + Args: + session: The connected Session object. + """ + await self._call_event_handler("on_joined", {"sessionId": session.id}) + + async def _on_disconnected(self, session: Session) -> None: + """Handle session disconnected event. + + Args: + session: The disconnected Session object. + """ + await self._call_event_handler("on_left", {"sessionId": session.id}) + + async def _on_error(self, _session: Session, description: str, _code: int) -> None: + """Handle session error event. + + Args: + _session: The Session object. + description: Error description. + _code: Error code. + """ + await self._call_event_handler("on_error", description) + + async def _on_stream_received(self, session: Session, stream: Stream) -> None: + """Handle stream received event. + + Args: + session: The Session object. + stream: The received Stream object. + """ + client = { + "sessionId": session.id, + "streamId": stream.id, + "connectionData": stream.connection.data, + } + if not self._one_stream_received: + self._one_stream_received = True + await self._call_event_handler("on_first_participant_joined", client) + + await self._call_event_handler("on_participant_joined", client) + + async def _on_stream_dropped(self, session: Session, stream: Stream) -> None: + """Handle stream dropped event. + + Args: + session: The Session object. + stream: The dropped Stream object. + """ + client = { + "sessionId": session.id, + "streamId": stream.id, + "connectionData": stream.connection.data, + } + await self._call_event_handler("on_participant_left", client) + + async def _on_subscriber_connected(self, subscriber: Subscriber) -> None: + """Handle subscriber connected event. + + Args: + subscriber: The connected Subscriber object. + """ + await self._call_event_handler( + "on_client_connected", + { + "subscriberId": subscriber.stream.id, + "streamId": subscriber.stream.id, + "connectionData": subscriber.stream.connection.data, + }, + ) + + async def _on_subscriber_disconnected(self, subscriber: Subscriber) -> None: + """Handle subscriber disconnected event. + + Args: + subscriber: The disconnected Subscriber object. + """ + await self._call_event_handler( + "on_client_disconnected", + { + "subscriberId": subscriber.stream.id, + "streamId": subscriber.stream.id, + "connectionData": subscriber.stream.connection.data, + }, + ) diff --git a/tests/test_vonage_video_connector.py b/tests/test_vonage_video_connector.py new file mode 100644 index 000000000..39ca716ed --- /dev/null +++ b/tests/test_vonage_video_connector.py @@ -0,0 +1,3101 @@ +# +# Copyright (c) 2024-2026, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import inspect +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Awaitable, Callable, Optional +from unittest.mock import ANY, AsyncMock, MagicMock, Mock, call, patch + +import numpy as np +import pytest + +from pipecat.clocks.system_clock import SystemClock +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + InputAudioRawFrame, + InterruptionFrame, + OutputAudioRawFrame, + OutputImageRawFrame, + StartFrame, + UserImageRawFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup +from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams + +# Mock the vonage_video module since it's not available in test environment +vonage_video_mock = MagicMock() +vonage_video_mock.VonageVideoClient = MagicMock() +vonage_video_mock.models = MagicMock() + + +# Create mock classes that match the expected interface + + +@dataclass(eq=True, frozen=True) +class MockAudioData: + sample_buffer: memoryview + sample_rate: int + number_of_channels: int + number_of_frames: int + + +@dataclass(eq=True, frozen=True) +class MockSession: + id: str + + +@dataclass(eq=True, frozen=True) +class MockConnection: + id: str + creation_time: datetime + data: str = "" + + +DUMMY_CONNECTION = MockConnection(id="dummy", creation_time=datetime.min) + + +@dataclass(eq=True, frozen=True) +class MockStream: + id: str + connection: MockConnection + + +@dataclass(eq=True, frozen=True) +class MockPublisher: + stream: MockStream + + +@dataclass(eq=True, frozen=True) +class MockSubscriber: + stream: Optional[MockStream] = None + + +@dataclass(eq=True, frozen=True) +class MockSessionAudioSettings: + sample_rate: int = 48000 + number_of_channels: int = 2 + + +@dataclass(eq=True, frozen=True) +class MockVideoResolution: + width: int = 640 + height: int = 480 + + +@dataclass(eq=True, frozen=True) +class MockSessionVideoPublisherSettings: + resolution: MockVideoResolution + fps: int = 30 + format: str = "YUV420P" + + +@dataclass(eq=True, frozen=True) +class MockSessionAVSettings: + audio_publisher: Optional[MockSessionAudioSettings] = None + audio_subscribers_mix: Optional[MockSessionAudioSettings] = None + video_publisher: Optional[MockSessionVideoPublisherSettings] = None + + +@dataclass(eq=True, frozen=True) +class MockLoggingSettings: + level: str = "WARN" + + +@dataclass(eq=True, frozen=True) +class MockSessionSettings: + enable_migration: bool = False + av: Optional[MockSessionAVSettings] = None + logging: Optional[MockLoggingSettings] = None + + +@dataclass(eq=True, frozen=True) +class MockPublisherAudioSettings: + enable_stereo_mode: bool = True + enable_opus_dtx: bool = False + + +@dataclass(eq=True, frozen=True) +class MockPublisherSettings: + name: str + has_audio: bool + has_video: bool + audio_settings: Optional[MockPublisherAudioSettings] = None + + +@dataclass(eq=True, frozen=True) +class MockVideoFrame: + frame_buffer: memoryview + resolution: MockVideoResolution + format: str = "YUV420P" + + +@dataclass(eq=True, frozen=True) +class MockSubscriberVideoSettings: + preferred_resolution: Optional[MockVideoResolution] = None + preferred_framerate: Optional[int] = None + + +@dataclass(eq=True, frozen=True) +class MockSubscriberSettings: + subscribe_to_audio: bool = True + subscribe_to_video: bool = True + video_settings: Optional[MockSubscriberVideoSettings] = None + + +# Set up the mock module structure +vonage_video_mock.models.AudioData = MockAudioData +vonage_video_mock.models.Session = MockSession +vonage_video_mock.models.Connection = MockConnection +vonage_video_mock.models.Stream = MockStream +vonage_video_mock.models.Publisher = MockPublisher +vonage_video_mock.models.Subscriber = MockSubscriber +vonage_video_mock.models.LoggingSettings = MockLoggingSettings +vonage_video_mock.models.SessionAVSettings = MockSessionAVSettings +vonage_video_mock.models.SessionSettings = MockSessionSettings +vonage_video_mock.models.SessionAudioSettings = MockSessionAudioSettings +vonage_video_mock.models.PublisherAudioSettings = MockPublisherAudioSettings +vonage_video_mock.models.PublisherSettings = MockPublisherSettings +vonage_video_mock.models.SessionVideoPublisherSettings = MockSessionVideoPublisherSettings +vonage_video_mock.models.SubscriberSettings = MockSubscriberSettings +vonage_video_mock.models.SubscriberVideoSettings = MockSubscriberVideoSettings +vonage_video_mock.models.VideoResolution = MockVideoResolution +vonage_video_mock.models.VideoFrame = MockVideoFrame + +# Mock the module in sys.modules so imports work +sys.modules["vonage_video_connector"] = vonage_video_mock +sys.modules["vonage_video_connector.models"] = vonage_video_mock.models + +# Now we can import the transport classes since the vonage_video module is mocked +from pipecat.transports.vonage.client import ( + VonageClient, + VonageClientListener, +) +from pipecat.transports.vonage.utils import ( + AudioProps, + ImageFormat, + check_audio_data, + image_colorspace_conversion, + process_audio, + process_audio_channels, +) +from pipecat.transports.vonage.video_connector import ( + SubscribeSettings, + VonageException, + VonageVideoConnectorInputTransport, + VonageVideoConnectorOutputTransport, + VonageVideoConnectorTransport, + VonageVideoConnectorTransportParams, +) + + +@dataclass(frozen=True) +class SubscriberCallbacks: + on_error_cb: Callable[[MockSubscriber, str, int], None] + on_connected_cb: Callable[[MockSubscriber], None] + on_disconnected_cb: Callable[[MockSubscriber], None] + on_render_frame_cb: Callable[[MockSubscriber, MockVideoFrame], None] + + +@dataclass(frozen=True) +class ConnectCallbacks: + on_error_cb: Callable[[MockSession, str, int], None] + on_disconnected_cb: Callable[[MockSession], None] + on_ready_for_audio_cb: Callable[[MockSession], None] + + +class TestVonageVideoConnectorTransport: + """Test cases for Vonage Video Connector transport classes.""" + + def setup_method(self) -> None: + """Set up test fixtures.""" + self.VonageClient = VonageClient + self.VonageClientListener = VonageClientListener + self.VonageVideoConnectorInputTransport = VonageVideoConnectorInputTransport + self.VonageVideoConnectorOutputTransport = VonageVideoConnectorOutputTransport + self.VonageVideoConnectorTransport = VonageVideoConnectorTransport + self.VonageVideoConnectorTransportParams = VonageVideoConnectorTransportParams + + # Mock client instance + self.mock_client_instance = Mock() + vonage_video_mock.VonageVideoClient.return_value = self.mock_client_instance + + # Common test data + self.application_id = "test-app-id" + self.session_id = "test-session-id" + self.token = "test-token" + self._frame_processor_setup: Optional[FrameProcessorSetup] = None + self._executor = ThreadPoolExecutor(max_workers=1) + + # subscriber state + self._connect_callbacks: Optional[ConnectCallbacks] = None + self._subscriber_callbacks: dict[str, SubscriberCallbacks] = {} + + def _get_frame_processor_setup(self) -> FrameProcessorSetup: + if self._frame_processor_setup is not None: + return self._frame_processor_setup + + clock: SystemClock = SystemClock() # type: ignore[no-untyped-call] + task_manager = TaskManager() + task_manager.setup(TaskManagerParams(loop=asyncio.get_running_loop())) + self._frame_processor_setup = FrameProcessorSetup(clock=clock, task_manager=task_manager) + return self._frame_processor_setup + + async def _wait_for_condition( + self, + condition: Callable[[], bool], + timeout: timedelta = timedelta(seconds=1), + check_interval: timedelta = timedelta(milliseconds=10), + ) -> None: + """Wait for a condition to become true with timeout. + + Args: + condition: Callable that returns True when condition is met. + timeout: Maximum time to wait. + check_interval: How often to check the condition. + + Raises: + asyncio.TimeoutError: If condition is not met within timeout. + """ + start_time = asyncio.get_event_loop().time() + timeout_seconds = timeout.total_seconds() + check_interval_seconds = check_interval.total_seconds() + + while not condition(): + if asyncio.get_event_loop().time() - start_time > timeout_seconds: + raise asyncio.TimeoutError(f"Condition not met within {timeout}") + await asyncio.sleep(check_interval_seconds) + + def test_vonage_client_listener_defaults(self) -> None: + """Test VonageClientListener default values.""" + listener = self.VonageClientListener() + assert listener.on_connected is not None + assert listener.on_disconnected is not None + assert listener.on_error is not None + assert listener.on_audio_in is not None + assert listener.on_stream_received is not None + assert listener.on_stream_dropped is not None + assert listener.on_subscriber_connected is not None + assert listener.on_subscriber_disconnected is not None + + def test_vonage_transport_params_defaults(self) -> None: + """Test VonageVideoConnectorTransportParams default values.""" + params = self.VonageVideoConnectorTransportParams() + assert params.publisher_name == "Bot" + assert params.publisher_enable_opus_dtx is False + assert params.session_enable_migration is False + + def test_vonage_client_initialization(self) -> None: + """Test VonageClient initialization.""" + # Reset the mock for this specific test + vonage_video_mock.VonageVideoClient.reset_mock() + + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + assert client._application_id == self.application_id + assert client._session_id == self.session_id + assert client._token == self.token + assert client._params == params + assert client._connected is False + assert client._connection_counter == 0 + vonage_video_mock.VonageVideoClient.assert_called_once() + + # check getting the event loop before setup raises error + with pytest.raises(Exception) as exc_info: + _ = client._get_event_loop() + + assert "missing task manager" in str(exc_info.value) + + # check pushing events before setup raises error + async def mock_coro() -> None: + pass + + mock_task = mock_coro() + with pytest.raises(Exception) as exc_info: + client._sdk_event_cb_to_loop(mock_task) + + mock_task.close() + assert "missing event queue" in str(exc_info.value) + + def test_vonage_client_add_remove_listener(self) -> None: + """Test adding and removing listeners from VonageClient.""" + params = self.VonageVideoConnectorTransportParams() + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + listener = self.VonageClientListener() + listener_id = client.add_listener(listener) + + assert isinstance(listener_id, int) + assert listener_id in client._listeners + assert client._listeners[listener_id] == listener + + client.remove_listener(listener_id) + assert listener_id not in client._listeners + + def _setup_audio_ready_callback(self, client: VonageClient, call_ready_for_audio: bool) -> None: + """Helper to set up the audio ready callback.""" + + def connect_side_effect( + *_: Any, + on_error_cb: Callable[[MockSession, str, int], None], + on_disconnected_cb: Callable[[MockSession], None], + on_ready_for_audio_cb: Callable[[MockSession], None], + **__: Any, + ) -> bool: + if call_ready_for_audio: + on_ready_for_audio_cb(vonage_video_mock.models.Session(id="session")) + + self._connect_callbacks = ConnectCallbacks( + on_error_cb=on_error_cb, + on_disconnected_cb=on_disconnected_cb, + on_ready_for_audio_cb=on_ready_for_audio_cb, + ) + return True + + self.mock_client_instance.connect = MagicMock(side_effect=connect_side_effect) + + def _setup_subscriber_callbacks(self, client: VonageClient) -> None: + def subscribe_side_effect( + stream: MockStream, + on_error_cb: Callable[[MockSubscriber, str, int], None], + on_connected_cb: Callable[[MockSubscriber], None], + on_disconnected_cb: Callable[[MockSubscriber], None], + on_render_frame_cb: Callable[[MockSubscriber, MockVideoFrame], None], + **__: Any, + ) -> bool: + self._subscriber_callbacks[stream.id] = SubscriberCallbacks( + on_error_cb=on_error_cb, + on_connected_cb=on_connected_cb, + on_disconnected_cb=on_disconnected_cb, + on_render_frame_cb=on_render_frame_cb, + ) + return True + + self.mock_client_instance.subscribe = MagicMock(side_effect=subscribe_side_effect) + + async def _subscribe_n_handle_callbacks( + self, + client: VonageClient, + stream_id: str, + params: SubscribeSettings, + callback: Callable[[SubscriberCallbacks], None], + ) -> None: + task = asyncio.create_task(client.subscribe_to_stream(stream_id, params)) + await self._wait_for_condition( + lambda: stream_id in self._subscriber_callbacks, + timeout=timedelta(seconds=2), + ) + callback(self._subscriber_callbacks[stream_id]) + await task + + async def _create_client( + self, + params: Optional[VonageVideoConnectorTransportParams] = None, + setup_connect_mock: bool = True, + ) -> VonageClient: + params = params or VonageVideoConnectorTransportParams() + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + if setup_connect_mock: + self._setup_audio_ready_callback(client, call_ready_for_audio=True) + + self._setup_subscriber_callbacks(client) + + await client.setup(self._get_frame_processor_setup()) + + return client + + async def _run_in_thread(self, callback: Callable[[], Any]) -> Any: + """Helper to run a coroutine in a separate thread.""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, callback) + + async def _wait_client_async_tasks(self, client: VonageClient) -> None: + """Helper to wait for all async tasks in the client to complete.""" + # Wait for any pending tasks in the client's task manager + drain_event = asyncio.Event() + + async def set_event_when_drained() -> None: + # Wait for all queues to be joined (all tasks processed) + if client._event_queue: + await client._event_queue.join() + if client._audio_queue: + await client._audio_queue.join() + if client._video_queue: + await client._video_queue.join() + drain_event.set() + + # Schedule the drain coroutine in the event loop + asyncio.create_task(set_event_when_drained()) + await drain_event.wait() + + async def _create_output_transport( + self, params: VonageVideoConnectorTransportParams + ) -> VonageVideoConnectorOutputTransport: + client = self.VonageClient( + self.application_id, + self.session_id, + self.token, + params, + ) + transport = self.VonageVideoConnectorOutputTransport(client, params) + await transport.setup(self._get_frame_processor_setup()) + + return transport + + async def _create_input_transport( + self, params: VonageVideoConnectorTransportParams + ) -> VonageVideoConnectorInputTransport: + client = self.VonageClient( + self.application_id, + self.session_id, + self.token, + params, + ) + transport = self.VonageVideoConnectorInputTransport(client, params) + await transport.setup(self._get_frame_processor_setup()) + + return transport + + async def _create_transport( + self, params: VonageVideoConnectorTransportParams + ) -> VonageVideoConnectorTransport: + transport = VonageVideoConnectorTransport( + self.application_id, + self.session_id, + self.token, + params, + ) + await transport.input().setup(self._get_frame_processor_setup()) + await transport.output().setup(self._get_frame_processor_setup()) + + return transport + + @pytest.mark.asyncio + async def test_vonage_client_setup_n_cleanup(self) -> None: + """Test VonageClient setup and cleanup methods.""" + params = self.VonageVideoConnectorTransportParams() + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + # Before setup, task manager and queues should be None + assert client._task_manager is None + assert client._event_queue is None + assert client._event_task is None + assert client._audio_queue is None + assert client._audio_task is None + assert client._video_queue is None + assert client._video_task is None + + # Setup the client + setup = self._get_frame_processor_setup() + await client.setup(setup) + + # Mock connection + self.mock_client_instance.connect.return_value = True + client._connected = True + client._connection_counter = 1 + + # After setup, task manager and queues should be initialized + assert client._task_manager is not None + assert client._task_manager == setup.task_manager + assert client._event_queue is not None + assert client._event_task is not None + assert client._audio_queue is not None + assert client._audio_task is not None + assert client._video_queue is not None + assert client._video_task is not None + + # Test that calling setup again doesn't recreate the task manager + old_task_manager = client._task_manager + old_event_queue = client._event_queue + old_event_task = client._event_task + await client.setup(setup) + assert client._task_manager == old_task_manager + assert client._event_queue == old_event_queue + assert client._event_task == old_event_task + + # Test cleanup without being connected + await client.cleanup() + + # After cleanup, tasks should be cancelled + assert client._event_task is None + assert client._audio_task is None + assert client._video_task is None + + # Verify disconnect was called + self.mock_client_instance.disconnect.assert_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "has_audio, has_video", + [ + (True, False), + (False, True), + (True, True), + (False, False), + ], + ) + async def test_vonage_client_connect_first_time(self, has_audio: bool, has_video: bool) -> None: + """Test VonageClient connect method for first connection.""" + params = self.VonageVideoConnectorTransportParams() + + # make changes to params depending on the configuration to check the right value + # goes to the right destination + params.audio_in_channels = 1 if has_video else 2 + params.audio_out_channels = 2 if has_video else 1 + params.audio_in_sample_rate = 44100 if has_video else 22050 + params.audio_out_sample_rate = 22050 if has_video else 44100 + params.session_enable_migration = has_video + params.video_out_color_format = "YCbCr" if has_audio else "RGB" + params.video_out_framerate = 30 if has_audio else 15 + params.video_out_width = 1280 if has_audio else 640 + params.video_out_height = 720 if has_audio else 480 + params.audio_in_enabled = has_audio + params.audio_out_enabled = has_audio + params.video_in_enabled = has_video + params.video_out_enabled = has_video + params.video_connector_log_level = "WARN" if has_audio else "ERROR" + + client = await self._create_client(params) + + # Mock the connect method to return True + self.mock_client_instance.connect.return_value = True + + listener = self.VonageClientListener() + listener.on_connected = AsyncMock() + # only set this callback if we have audio enabled + self._setup_audio_ready_callback(client, has_audio) + listener_id = client.add_listener(listener) + await client.connect() + + assert isinstance(listener_id, int) + self.mock_client_instance.connect.assert_called_once() + + # Verify connect was called with correct parameters + call_args = self.mock_client_instance.connect.call_args + assert call_args[1]["application_id"] == self.application_id + assert call_args[1]["session_id"] == self.session_id + assert call_args[1]["token"] == self.token + assert call_args[1]["session_settings"] == MockSessionSettings( + av=MockSessionAVSettings( + audio_publisher=MockSessionAudioSettings( + sample_rate=params.audio_out_sample_rate, + number_of_channels=params.audio_out_channels, + ), + audio_subscribers_mix=MockSessionAudioSettings( + sample_rate=params.audio_in_sample_rate, + number_of_channels=params.audio_in_channels, + ), + video_publisher=MockSessionVideoPublisherSettings( + resolution=MockVideoResolution( + width=params.video_out_width, + height=params.video_out_height, + ), + fps=params.video_out_framerate, + format=client._video_out_color_format_vonage, + ), + ), + enable_migration=params.session_enable_migration, + logging=MockLoggingSettings(level=params.video_connector_log_level), + ) + assert self._connect_callbacks is not None + assert call_args[1]["on_audio_data_cb"] == client._on_session_audio_data_cb + assert call_args[1]["on_error_cb"] == self._connect_callbacks.on_error_cb + assert call_args[1]["on_connected_cb"] == client._on_session_connected_cb + assert call_args[1]["on_disconnected_cb"] == self._connect_callbacks.on_disconnected_cb + assert ( + call_args[1]["on_ready_for_audio_cb"] == self._connect_callbacks.on_ready_for_audio_cb + ) + assert call_args[1]["on_stream_received_cb"] == client._on_stream_received_cb + assert call_args[1]["on_stream_dropped_cb"] == client._on_stream_dropped_cb + + listener.on_connected.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "has_audio, has_video", + [ + (True, False), + (False, True), + (True, True), + (False, False), + ], + ) + async def test_vonage_client_publish_after_connect( + self, has_audio: bool, has_video: bool + ) -> None: + """Test VonageClient publishes after being connected method for first connection.""" + params = self.VonageVideoConnectorTransportParams() + + # make changes to params depending on the configuration to check the right value + # goes to the right destination + params.audio_in_enabled = has_audio + params.audio_out_enabled = has_audio + params.video_in_enabled = has_video + params.video_out_enabled = has_video + params.audio_out_channels = 2 if has_video else 1 + params.publisher_enable_opus_dtx = not has_video + params.publisher_name = "test-audio" if has_audio else "test-video" + + client = await self._create_client(params) + await client.connect() + + self.mock_client_instance.connect.assert_called_once() + + # trigger the _on_session_connected_cb to simulate being connected + await self._run_in_thread( + lambda: client._on_session_connected_cb(vonage_video_mock.models.Session(id="session")) + ) + await self._wait_client_async_tasks(client) + + # if no audio and no video, publish should not be called + if not has_audio and not has_video: + self.mock_client_instance.publish.assert_not_called() + return + + # Verify publish was called with correct parameters + self.mock_client_instance.publish.assert_called_once() + call_args = self.mock_client_instance.publish.call_args + assert call_args[1]["settings"] == MockPublisherSettings( + name=params.publisher_name, + audio_settings=MockPublisherAudioSettings( + enable_stereo_mode=params.audio_out_channels == 2, + enable_opus_dtx=params.publisher_enable_opus_dtx, + ), + has_audio=has_audio, + has_video=has_video, + ) + assert call_args[1]["on_error_cb"] == client._on_publisher_error_cb + assert call_args[1]["on_stream_created_cb"] == client._on_publisher_stream_created_cb + assert call_args[1]["on_stream_destroyed_cb"] == client._on_publisher_stream_destroyed_cb + + @pytest.mark.asyncio + async def test_vonage_client_connect_already_connected(self) -> None: + """Test VonageClient connect when already connected.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + # add some listeners, tests multiple listeners are notified, test no notifiaction after removal too + listener1 = self.VonageClientListener() + listener1.on_connected = AsyncMock() + client.add_listener(listener1) + listener2 = self.VonageClientListener() + listener2.on_connected = AsyncMock() + client.add_listener(listener2) + listener3 = self.VonageClientListener() + listener3.on_connected = AsyncMock() + listener_id3 = client.add_listener(listener3) + client.remove_listener(listener_id3) + + # First connection, connection is performed + await client.connect() + self.mock_client_instance.connect.assert_called_once() + listener1.on_connected.assert_called_once() + listener2.on_connected.assert_called_once() + + # Second connection, should not trigger a new connect call or raised any events + await client.connect() + self.mock_client_instance.connect.assert_called_once() + listener1.on_connected.assert_called_once() + listener2.on_connected.assert_called_once() + + # the removed listener should not have received any events + listener3.on_connected.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_client_connect_while_disconnecting(self) -> None: + """Test VonageClient waits for disconnect to complete before connecting.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + self.mock_client_instance.disconnect = MagicMock() + + # Simulate disconnect in progress + disconnect_future = asyncio.get_running_loop().create_future() + client._disconnecting_future = disconnect_future + + # Start connect task - it should block waiting for disconnect + connect_task = asyncio.create_task(client.connect()) + + # Give control to the event loop to let connect task start + await asyncio.sleep(0.2) + + self.mock_client_instance.connect.assert_not_called() + + # Resolve the disconnect future to unblock connect + disconnect_future.set_result(None) + + # Wait for connect to complete + await connect_task + + self.mock_client_instance.connect.assert_called_once() + + # Verify client state + assert client._connected is True + assert client._connection_counter == 1 + + @pytest.mark.asyncio + async def test_vonage_client_timeout_while_connecting(self) -> None: + """Test VonageClient handles timeout during connection.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params, setup_connect_mock=False) + + # Create an event that will block but can be interrupted + stop_event = threading.Event() + + # Mock the SDK connect method to block until interrupted + def connect_blocks_forever(*args: Any, **kwargs: Any) -> bool: + stop_event.wait(timeout=10) # Wait max 10 seconds but can be interrupted + return True + + self.mock_client_instance.connect.side_effect = connect_blocks_forever + + try: + # Patch the timeout to be very short for fast test execution + with patch( + "pipecat.transports.vonage.client.VIDEO_CONNECTOR_TIMEOUT", + timedelta(seconds=0.1), + ): + # Attempt to connect, should timeout + with pytest.raises(asyncio.TimeoutError): + await client.connect() + + # Verify client state after timeout + assert client._connected is False + assert client._connection_counter == 0 + assert client._connecting_future is None + finally: + # Stop the blocking thread + stop_event.set() + + @pytest.mark.asyncio + async def test_vonage_client_concurrent_connects(self) -> None: + """Test VonageClient concurrent connects.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + # Mock the connect method to return True and store the callback + connecting_future: asyncio.Future[Callable[[Any], None]] = ( + asyncio.get_running_loop().create_future() + ) + + def connect_side_effect( + *_: Any, on_ready_for_audio_cb: Optional[Callable[[Any], None]] = None, **__: Any + ) -> bool: + assert on_ready_for_audio_cb is not None + connecting_future.set_result(on_ready_for_audio_cb) + return True + + self.mock_client_instance.connect = MagicMock(side_effect=connect_side_effect) + + # create a listener + listener = self.VonageClientListener() + listener.on_connected = AsyncMock() + client.add_listener(listener) + + # send two parallel connect calls and let them get stuck awaiting + connect1_task = asyncio.create_task(client.connect()) + connect2_task = asyncio.create_task(client.connect()) + + audio_ready_cb = await connecting_future + + # Now both connects are waiting on the same promise, we can set it to complete + audio_ready_cb(vonage_video_mock.models.Session(id="session")) + + # await for the connections to now complete + await asyncio.gather(connect1_task, connect2_task) + + # SDK connect should only be called once + self.mock_client_instance.connect.assert_called_once() + listener.on_connected.assert_called_once() + + @pytest.mark.asyncio + async def test_vonage_client_connect_failure(self) -> None: + """Test VonageClient connect method when connection fails.""" + client = await self._create_client(setup_connect_mock=False) + + # Mock the connect method to return False + self.mock_client_instance.connect.return_value = False + + with pytest.raises(Exception) as exc_info: + await client.connect() + + assert "Could not connect to session" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_vonage_client_disconnect_before_connecting(self) -> None: + """Test VonageClient disconnect method before connecting.""" + client = await self._create_client() + + listener = self.VonageClientListener() + listener.on_disconnected = AsyncMock() + client.add_listener(listener) + + await client.disconnect() + + self.mock_client_instance.disconnect.assert_not_called() + listener.on_disconnected.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_client_disconnect(self) -> None: + """Test VonageClient disconnect method.""" + client = await self._create_client() + + # create a listener + listener = self.VonageClientListener() + listener.on_disconnected = AsyncMock() + client.add_listener(listener) + + # send two parallel connect calls and let them get stuck awaiting + connect_promise1 = client.connect() + connect_promise2 = client.connect() + + # await for the connections to now complete + await asyncio.gather(connect_promise1, connect_promise2) + + # Add some items to the queues before disconnect + assert client._event_queue is not None + assert client._audio_queue is not None + assert client._video_queue is not None + + async def mock_event_task() -> None: + pass + + async def mock_audio_task() -> None: + pass + + async def mock_video_task() -> None: + pass + + await client._event_queue.put(mock_event_task()) + await client._audio_queue.put(mock_audio_task()) + await client._video_queue.put(mock_video_task()) + + # Verify queues have items + assert client._event_queue.qsize() == 1 + assert client._audio_queue.qsize() == 1 + assert client._video_queue.qsize() == 1 + + # Mock the client's clear_media_buffers method + self.mock_client_instance.clear_media_buffers = MagicMock() + + # send the first disconnect call, we should still be connected + await client.disconnect() + self.mock_client_instance.disconnect.assert_not_called() + self.mock_client_instance.clear_media_buffers.assert_not_called() + listener.on_disconnected.assert_not_called() + + # Queues should still have items since we didn't actually disconnect + assert client._event_queue.qsize() == 1 + assert client._audio_queue.qsize() == 1 + assert client._video_queue.qsize() == 1 + + # check the second disconnect now disconnects for real + await client.disconnect() + self.mock_client_instance.disconnect.assert_called_once() + self.mock_client_instance.clear_media_buffers.assert_called_once() + listener.on_disconnected.assert_called_once() + + # Verify queues are now empty after disconnect + assert client._event_queue.qsize() == 0 + assert client._audio_queue.qsize() == 0 + assert client._video_queue.qsize() == 0 + + # an extra disconnect should not do anything + await client.disconnect() + self.mock_client_instance.disconnect.assert_called_once() + self.mock_client_instance.clear_media_buffers.assert_called_once() + listener.on_disconnected.assert_called_once() + + @pytest.mark.asyncio + async def test_vonage_client_disconnect_while_connecting(self) -> None: + """Test VonageClient waits for connect to complete before disconnecting.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + client._connected = True + client._connection_counter = 1 + self.mock_client_instance.connect = MagicMock() + + # Simulate connect in progress + connect_future = asyncio.get_running_loop().create_future() + client._connecting_future = connect_future + + # Start disconnect task - it should block waiting for disconnect + disconnect_task = asyncio.create_task(client.disconnect()) + + # Give control to the event loop to let disconnect task start + await asyncio.sleep(0.2) + + self.mock_client_instance.disconnect.assert_not_called() + + # Resolve the disconnect future to unblock connect + connect_future.set_result(None) + + # Wait for connect to complete + await disconnect_task + + self.mock_client_instance.disconnect.assert_called_once() + + # Verify client state + assert client._connected is False + assert client._connection_counter == 0 + + @pytest.mark.asyncio + async def test_vonage_client_timeout_while_disconnecting(self) -> None: + """Test VonageClient handles timeout during disconnection.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params, setup_connect_mock=False) + + await client.connect() + assert client._connection_counter == 1 + + # Create an event that will block but can be interrupted + stop_event = threading.Event() + + # Mock the SDK disconnect method to block until interrupted + def disconnect_blocks_forever(*args: Any, **kwargs: Any) -> bool: + stop_event.wait(timeout=10) # Wait max 10 seconds but can be interrupted + return True + + self.mock_client_instance.disconnect.side_effect = disconnect_blocks_forever + try: + # Patch the timeout to be very short for fast test execution + with patch( + "pipecat.transports.vonage.client.VIDEO_CONNECTOR_TIMEOUT", + timedelta(seconds=0.1), + ): + # Attempt to connect, should timeout + with pytest.raises(asyncio.TimeoutError): + await client.disconnect() + + # Verify client state after timeout + assert client._connected is True + assert client._connection_counter == 1 + assert client._disconnecting_future is None + finally: + # Stop the blocking thread + stop_event.set() + + @pytest.mark.asyncio + async def test_vonage_client_clear_media_buffers(self) -> None: + """Test VonageClient clear_media_buffers method.""" + params = self.VonageVideoConnectorTransportParams( + audio_out_channels=2, audio_out_sample_rate=48000 + ) + client = await self._create_client(params) + + # Add some items to the audio and video queues + assert client._audio_queue is not None + assert client._video_queue is not None + + # Create mock coroutines to add to queues + async def mock_audio_task() -> None: + pass + + async def mock_video_task() -> None: + pass + + # Put some items in the queues + await client._audio_queue.put(mock_audio_task()) + await client._audio_queue.put(mock_audio_task()) + await client._video_queue.put(mock_video_task()) + + # Verify queues have items + assert client._audio_queue.qsize() == 2 + assert client._video_queue.qsize() == 1 + + # Mock the client's clear_media_buffers method + self.mock_client_instance.clear_media_buffers = MagicMock() + + # Clear the buffers + client.clear_media_buffers() + + # Verify queues are now empty + assert client._audio_queue.qsize() == 0 + assert client._video_queue.qsize() == 0 + + # Verify the SDK client's clear_media_buffers was called + self.mock_client_instance.clear_media_buffers.assert_called_once() + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.VIDEO_QUEUE_MAXSIZE", 1) + async def test_vonage_client_sdk_cb_to_loop_full_queue(self) -> None: + """Test VonageClient SDK callback to loop filling up the queue.""" + params = self.VonageVideoConnectorTransportParams() + client = await self._create_client(params) + + # Ensure the loop thread ID is set + assert client._video_queue is not None + assert client._loop_thread_id == threading.get_ident() + + # Create a mock coroutine to queue + async def mock_task() -> None: + pass + + # Fill queue to max size + for _ in range(client._video_queue.maxsize): + await client._video_queue.put(mock_task()) + + # Queue should be full + assert client._video_queue.qsize() == client._video_queue.maxsize + # This should log an error and drop the event + async_task = mock_task() + client._sdk_cb_to_loop("test_event", client._video_queue, async_task) + + # Queue should still be full (no new item added) + assert client._video_queue.qsize() == client._video_queue.maxsize + # check the coroutine was closed and hence dropped + assert inspect.getcoroutinestate(async_task) == "CORO_CLOSED" + + # Clean up the coroutine + task = await client._video_queue.get() + task.close() + client._video_queue.task_done() + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.create_stream_resampler") + async def test_vonage_client_get_audio_with_resampling(self, mock_resampler: MagicMock) -> None: + """Test VonageClient get_audio method.""" + # Return resampled stereo data + resampled_data = b"\x07\x06\x05\x04\x03\x02\x01\x00" + mock_resampler_instance = Mock() + mock_resampler_instance.resample = AsyncMock(return_value=resampled_data) + mock_resampler.return_value = mock_resampler_instance + + params = self.VonageVideoConnectorTransportParams( + audio_in_channels=1, + audio_in_sample_rate=48000, + audio_in_enabled=True, + ) + client = await self._create_client(params) + listener = self.VonageClientListener() + on_audio_in_mock = AsyncMock() + listener.on_audio_in = on_audio_in_mock + client.add_listener(listener) + + await client.connect() + + mock_audio_data = vonage_video_mock.models.AudioData( + sample_buffer=memoryview(b"\x00\x01\x02\x03\x04\x05\x06\x07"), + number_of_frames=4, + number_of_channels=1, + sample_rate=16000, + ) + + session = vonage_video_mock.models.Session(id="test_session") + client._on_session_audio_data_cb(session, mock_audio_data) + await self._wait_for_condition(lambda: on_audio_in_mock.call_count > 0) + + listener.on_audio_in.assert_called_once_with(session, ANY) + frame = listener.on_audio_in.call_args[0][1] + assert frame.audio == resampled_data + assert frame.num_frames == 4 + assert frame.sample_rate == 48000 + assert frame.num_channels == 1 + + @pytest.mark.asyncio + async def test_vonage_client_get_video(self) -> None: + """Test VonageClient get video.""" + pass + + @pytest.mark.asyncio + async def test_vonage_client_write_audio(self) -> None: + """Test VonageClient write_audio method.""" + params = self.VonageVideoConnectorTransportParams( + audio_out_channels=2, audio_out_sample_rate=48000 + ) + client = await self._create_client(params) + + # Create mock audio data + audio_data = OutputAudioRawFrame( + audio=b"\x00\x01\x02\x03\x04\x05\x06\x07", + sample_rate=48000, + num_channels=2, + ) # 4 frames of 2-channel 16-bit audio + + await client.write_audio(audio_data) + + self.mock_client_instance.add_audio.assert_called_once() + call_args = self.mock_client_instance.add_audio.call_args[0][0] + assert call_args.sample_buffer.tobytes() == audio_data.audio + assert call_args.number_of_frames == 2 # 8 bytes / (2 channels * 2 bytes) + assert call_args.number_of_channels == 2 + assert call_args.sample_rate == 48000 + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.create_stream_resampler") + async def test_vonage_client_write_audio_with_resampling( + self, mock_resampler: MagicMock + ) -> None: + """Test VonageClient write_audio method.""" + # Return resampled stereo data + resampled_data = b"\x07\x06\x05\x04\x03\x02\x01\x00" + mock_resampler_instance = Mock() + mock_resampler_instance.resample = AsyncMock(return_value=resampled_data) + mock_resampler.return_value = mock_resampler_instance + + params = self.VonageVideoConnectorTransportParams( + audio_out_channels=1, audio_out_sample_rate=16000 + ) + client = await self._create_client(params) + + # Create mock audio data + audio_data = OutputAudioRawFrame( + audio=b"\x00\x01\x02\x03\x04\x05\x06\x07", + sample_rate=48000, + num_channels=1, + ) # 4 frames of 1-channel 16-bit audio + + await client.write_audio(audio_data) + + self.mock_client_instance.add_audio.assert_called_once() + call_args = self.mock_client_instance.add_audio.call_args[0][0] + assert call_args.sample_buffer.tobytes() == resampled_data + assert call_args.number_of_frames == 4 # 8 bytes / (1 channel * 2 bytes) + assert call_args.number_of_channels == 1 + assert call_args.sample_rate == 16000 + + @pytest.mark.asyncio + async def test_vonage_client_write_video(self) -> None: + """Test VonageClient write_video method.""" + params = self.VonageVideoConnectorTransportParams( + video_out_width=640, + video_out_height=480, + video_out_color_format="RGB", + ) + client = await self._create_client(params) + + # Create a test RGB image (640x480, 3 channels) + width, height = 640, 480 + # Create RGB data: simple gradient pattern + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + rgb_image[:, :, 0] = 100 # R channel + rgb_image[:, :, 1] = 150 # G channel + rgb_image[:, :, 2] = 200 # B channel + + rgb_bytes = rgb_image.tobytes() + + # Create OutputImageRawFrame + frame = OutputImageRawFrame(image=rgb_bytes, size=(width, height), format="RGB") + + # Mock the add_video method + self.mock_client_instance.add_video = MagicMock(return_value=True) + + result = await client.write_video(frame) + + # Verify add_video was called + assert result is True + self.mock_client_instance.add_video.assert_called_once() + + # Get the VideoFrame argument + call_args = self.mock_client_instance.add_video.call_args[0][0] + + # Verify the resolution + assert call_args.resolution.width == width + assert call_args.resolution.height == height + + # Verify the format + assert call_args.format == "RGB24" + + # Verify BGR conversion happened correctly + # Convert back from the buffer to verify + bgr_buffer = bytes(call_args.frame_buffer) + bgr_image = np.frombuffer(bgr_buffer, dtype=np.uint8).reshape(height, width, 3) + + # Check that RGB was converted to BGR (channels swapped) + assert bgr_image[0, 0, 0] == 200 # B channel (was R=200 in RGB) + assert bgr_image[0, 0, 1] == 150 # G channel (unchanged) + assert bgr_image[0, 0, 2] == 100 # R channel (was B=100 in RGB) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "has_audio, has_video", + [ + (True, False), + (False, True), + (True, True), + ], + ) + async def test_vonage_client_subscribe_to_stream( + self, has_audio: bool, has_video: bool + ) -> None: + """Test VonageClient subscribe_to_stream with a stream that exists in session.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + listener = self.VonageClientListener() + client.add_listener(listener) + on_subscriber_connected_mock = AsyncMock() + listener.on_subscriber_connected = on_subscriber_connected_mock + on_subscriber_disconnected_mock = AsyncMock() + listener.on_subscriber_disconnected = on_subscriber_disconnected_mock + + await client.connect() + + # Add a stream to the session + stream = vonage_video_mock.models.Stream(id="test_stream", connection=DUMMY_CONNECTION) + client._session_streams["test_stream"] = stream + + # Setup subscriber callbacks + self._setup_subscriber_callbacks(client) + + # Subscribe with audio and video + subscribe_params = SubscribeSettings( + subscribe_to_audio=has_audio, + subscribe_to_video=has_video, + preferred_resolution=(640, 480) if has_video else None, + preferred_framerate=30 if has_video else None, + ) + + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + await self._subscribe_n_handle_callbacks( + client, + "test_stream", + subscribe_params, + lambda callbacks: callbacks.on_connected_cb(subscriber), + ) + on_subscriber_connected_mock.assert_called_once_with(subscriber) + + # Verify subscribe was called with correct parameters + self.mock_client_instance.subscribe.assert_called_once() + call_args = self.mock_client_instance.subscribe.call_args + + expected_settings = MockSubscriberSettings( + subscribe_to_audio=has_audio, + subscribe_to_video=has_video, + video_settings=MockSubscriberVideoSettings( + preferred_resolution=MockVideoResolution( + width=subscribe_params.preferred_resolution[0], + height=subscribe_params.preferred_resolution[1], + ) + if subscribe_params.preferred_resolution + else None, + preferred_framerate=subscribe_params.preferred_framerate, + ), + ) + + assert call_args[0][0] == stream + assert call_args[1]["settings"] == expected_settings + + # Verify subscription was stored + assert "test_stream" in client._session_subscriptions + assert client._session_subscriptions["test_stream"] == subscribe_params + + # check we can get a disconnect event from the subscriber + self._subscriber_callbacks["test_stream"].on_disconnected_cb(subscriber) + await self._wait_for_condition(lambda: on_subscriber_disconnected_mock.call_count > 0) + on_subscriber_disconnected_mock.assert_called_once_with(subscriber) + + @pytest.mark.asyncio + async def test_vonage_client_subscribe_to_stream_same_and_different_params(self) -> None: + """Test VonageClient subscribe_to_stream when subscribing multiple times with same and different parameters.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + listener = self.VonageClientListener() + client.add_listener(listener) + on_subscriber_connected_mock = AsyncMock() + listener.on_subscriber_connected = on_subscriber_connected_mock + + await client.connect() + + # Add a stream to the session + stream = vonage_video_mock.models.Stream(id="test_stream", connection=DUMMY_CONNECTION) + client._session_streams["test_stream"] = stream + + # Setup subscriber callbacks + self._setup_subscriber_callbacks(client) + + # First subscription with specific parameters + subscribe_params_1 = SubscribeSettings( + subscribe_to_audio=True, + subscribe_to_video=True, + preferred_resolution=(640, 480), + preferred_framerate=30, + ) + + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + await self._subscribe_n_handle_callbacks( + client, + "test_stream", + subscribe_params_1, + lambda callbacks: callbacks.on_connected_cb(subscriber), + ) + on_subscriber_connected_mock.assert_called_once_with(subscriber) + + # Verify subscribe was called once + assert self.mock_client_instance.subscribe.call_count == 1 + assert "test_stream" in client._session_subscriptions + assert client._session_subscriptions["test_stream"] == subscribe_params_1 + + # Subscribe again with SAME parameters - should do nothing + await client.subscribe_to_stream("test_stream", subscribe_params_1) + + # Verify subscribe was still only called once (no new subscription) + assert self.mock_client_instance.subscribe.call_count == 1 + # Verify unsubscribe was NOT called + self.mock_client_instance.unsubscribe.assert_not_called() + # Subscription should remain unchanged + assert client._session_subscriptions["test_stream"] == subscribe_params_1 + + # Subscribe again with DIFFERENT parameters - should unsubscribe and resubscribe + subscribe_params_2 = SubscribeSettings( + subscribe_to_audio=False, + subscribe_to_video=True, + preferred_resolution=(1280, 720), + preferred_framerate=60, + ) + + # Track the current subscribe call count before resubscribing + initial_subscribe_count = self.mock_client_instance.subscribe.call_count + + # Start the resubscription + resubscribe_task = asyncio.create_task( + client.subscribe_to_stream("test_stream", subscribe_params_2) + ) + + # Wait for the new subscribe call to be made + await self._wait_for_condition( + lambda: self.mock_client_instance.subscribe.call_count > initial_subscribe_count, + timeout=timedelta(seconds=2), + ) + + # Now trigger the connected callback with the new subscription callbacks + self._subscriber_callbacks["test_stream"].on_connected_cb(subscriber) + await resubscribe_task + + # Verify unsubscribe was called once (to remove old subscription) + self.mock_client_instance.unsubscribe.assert_called_once_with(stream) + # Verify subscribe was called a second time (for new subscription) + assert self.mock_client_instance.subscribe.call_count == 2 + # Verify new subscription parameters are stored + assert client._session_subscriptions["test_stream"] == subscribe_params_2 + + # Verify the second subscribe call had the new parameters + second_call_args = self.mock_client_instance.subscribe.call_args + expected_settings = MockSubscriberSettings( + subscribe_to_audio=False, + subscribe_to_video=True, + video_settings=MockSubscriberVideoSettings( + preferred_resolution=MockVideoResolution(width=1280, height=720), + preferred_framerate=60, + ), + ) + assert second_call_args[0][0] == stream + assert second_call_args[1]["settings"] == expected_settings + + @pytest.mark.asyncio + async def test_vonage_client_subscribe_to_stream_timeout(self) -> None: + """Test VonageClient subscribe_to_stream when SDK subscribe times out.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + await client.connect() + + stream = vonage_video_mock.models.Stream(id="fail_stream", connection=DUMMY_CONNECTION) + client._session_streams["fail_stream"] = stream + + self._setup_subscriber_callbacks(client) + + subscribe_params = SubscribeSettings(subscribe_to_audio=True, subscribe_to_video=False) + + # Patch the timeout to be very short for fast test execution + # the call never gets on_connected_cb or any other callback, it will timeout + with patch( + "pipecat.transports.vonage.client.VIDEO_CONNECTOR_TIMEOUT", + timedelta(seconds=0.1), + ): + with pytest.raises(asyncio.TimeoutError): + await client.subscribe_to_stream("fail_stream", subscribe_params) + + @pytest.mark.asyncio + async def test_vonage_client_subscribe_to_stream_fails(self) -> None: + """Test VonageClient subscribe_to_stream when SDK subscribe fails.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + await client.connect() + + stream = vonage_video_mock.models.Stream(id="fail_stream", connection=DUMMY_CONNECTION) + client._session_streams["fail_stream"] = stream + + self._setup_subscriber_callbacks(client) + self.mock_client_instance.subscribe.side_effect = lambda *_, **__: False + + subscribe_params = SubscribeSettings(subscribe_to_audio=True, subscribe_to_video=False) + with pytest.raises(VonageException) as exc_info: + await client.subscribe_to_stream("fail_stream", subscribe_params) + + assert "Could not subscribe to stream" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_vonage_client_subscribe_to_stream_subscriber_error(self) -> None: + """Test VonageClient subscribe_to_stream when subscriber reports an error.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + await client.connect() + + stream = vonage_video_mock.models.Stream(id="error_stream", connection=DUMMY_CONNECTION) + client._session_streams["error_stream"] = stream + + self._setup_subscriber_callbacks(client) + + subscribe_params = SubscribeSettings(subscribe_to_audio=True, subscribe_to_video=False) + + # Subscription should raise an exception + with pytest.raises(VonageException) as exc_info: + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + await self._subscribe_n_handle_callbacks( + client, + "error_stream", + subscribe_params, + lambda callbacks: callbacks.on_error_cb(subscriber, "Connection failed", 1500), + ) + + assert "Subscriber error" in str(exc_info.value) + assert "Connection failed" in str(exc_info.value) + assert "(code 1500)" in str(exc_info.value) + + # Verify subscription was removed + assert "error_stream" not in client._session_subscriptions + + @pytest.mark.asyncio + async def test_vonage_client_subscribe_to_stream_subscriber_disconnected_before_connected( + self, + ) -> None: + """Test VonageClient subscribe_to_stream when subscriber disconnects before connecting.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = await self._create_client(params) + + await client.connect() + + stream = vonage_video_mock.models.Stream(id="dc_stream", connection=DUMMY_CONNECTION) + client._session_streams["dc_stream"] = stream + + self._setup_subscriber_callbacks(client) + + # Add listener to track disconnection + listener = self.VonageClientListener() + on_subscriber_disconnected_mock = AsyncMock() + listener.on_subscriber_disconnected = on_subscriber_disconnected_mock + client.add_listener(listener) + + subscribe_params = SubscribeSettings(subscribe_to_audio=True, subscribe_to_video=False) + + # Subscription should raise an exception + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + with pytest.raises(VonageException) as exc_info: + await self._subscribe_n_handle_callbacks( + client, + "dc_stream", + subscribe_params, + lambda callbacks: callbacks.on_disconnected_cb(subscriber), + ) + + assert "disconnected before connecting" in str(exc_info.value) + + # Verify subscription was removed + assert "dc_stream" not in client._session_subscriptions + + # Verify listener was called + listener.on_subscriber_disconnected.assert_awaited_once_with(subscriber) + + @pytest.mark.asyncio + async def test_vonage_client_on_stream_received_triggers_listeners(self) -> None: + """Test that _on_stream_received_cb triggers on_stream_received listener callbacks.""" + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=True, audio_in_auto_subscribe=False + ) + client = await self._create_client(params) + + # Add multiple listeners + listener1 = self.VonageClientListener() + on_stream_received_mock1 = AsyncMock() + listener1.on_stream_received = on_stream_received_mock1 + client.add_listener(listener1) + + listener2 = self.VonageClientListener() + on_stream_received_mock2 = AsyncMock() + listener2.on_stream_received = on_stream_received_mock2 + client.add_listener(listener2) + + await client.connect() + + session = vonage_video_mock.models.Session(id="test_session") + stream = vonage_video_mock.models.Stream(id="test_stream", connection=DUMMY_CONNECTION) + + # Trigger the callback + client._on_stream_received_cb(session, stream) + + # Wait for async processing + await self._wait_for_condition( + lambda: on_stream_received_mock1.await_count > 0 + and on_stream_received_mock2.await_count > 0 + ) + + # Verify both listeners were called + on_stream_received_mock1.assert_awaited_once_with(session, stream) + on_stream_received_mock2.assert_awaited_once_with(session, stream) + + # Verify stream was added to session streams + assert "test_stream" in client._session_streams + assert client._session_streams["test_stream"] == stream + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "auto_audio, auto_video", + [ + (True, False), + (False, True), + (True, True), + (False, False), + ], + ) + async def test_vonage_client_on_stream_received_auto_subscribe( + self, auto_audio: bool, auto_video: bool + ) -> None: + """Test that _on_stream_received_cb auto-subscribes when auto_subscribe is enabled.""" + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=True, + video_in_enabled=True, + audio_in_auto_subscribe=auto_audio, + video_in_auto_subscribe=auto_video, + video_in_preferred_resolution=(640, 480) if auto_video else None, + video_in_preferred_framerate=25 if auto_video else None, + ) + client = await self._create_client(params) + + await client.connect() + + self._setup_subscriber_callbacks(client) + + session = vonage_video_mock.models.Session(id="test_session") + stream = vonage_video_mock.models.Stream(id="auto_sub_stream", connection=DUMMY_CONNECTION) + + # Trigger the callback + client._on_stream_received_cb(session, stream) + await self._wait_for_condition(lambda: stream.id in client._session_streams) + + if not auto_audio and not auto_video: + await self._wait_client_async_tasks(client) + # No auto-subscribe should happen + await self._wait_client_async_tasks(client) + self.mock_client_instance.subscribe.assert_not_called() + return + + # Wait for auto-subscribe to happen + await self._wait_for_condition(lambda: "auto_sub_stream" in self._subscriber_callbacks) + + # Verify subscribe was called + self.mock_client_instance.subscribe.assert_called_once() + call_args = self.mock_client_instance.subscribe.call_args + + # Verify subscription settings + expected_settings = MockSubscriberSettings( + subscribe_to_audio=auto_audio, + subscribe_to_video=auto_video, + video_settings=MockSubscriberVideoSettings( + preferred_resolution=MockVideoResolution(width=640, height=480) + if auto_video + else None, + preferred_framerate=25 if auto_video else None, + ), + ) + assert call_args[0][0] == stream + assert call_args[1]["settings"] == expected_settings + + # Verify subscription was stored + assert "auto_sub_stream" in client._session_subscriptions + assert client._session_subscriptions["auto_sub_stream"].subscribe_to_audio == auto_audio + assert client._session_subscriptions["auto_sub_stream"].subscribe_to_video == auto_video + + self._subscriber_callbacks["auto_sub_stream"].on_connected_cb(MockSubscriber(stream=stream)) + await self._wait_client_async_tasks(client) + + @pytest.mark.asyncio + async def test_vonage_client_on_stream_received_skips_existing_subscription(self) -> None: + """Test that _on_stream_received_cb does not auto-subscribe if stream is already subscribed.""" + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=True, audio_in_auto_subscribe=True + ) + client = await self._create_client(params) + + listener = self.VonageClientListener() + on_stream_received_mock = AsyncMock() + listener.on_stream_received = on_stream_received_mock + client.add_listener(listener) + + await client.connect() + + self._setup_subscriber_callbacks(client) + + session = vonage_video_mock.models.Session(id="test_session") + stream = vonage_video_mock.models.Stream(id="existing_stream", connection=DUMMY_CONNECTION) + + # Manually add an existing subscription + client._session_subscriptions["existing_stream"] = SubscribeSettings( + subscribe_to_audio=True, subscribe_to_video=False + ) + + # Trigger the callback + client._on_stream_received_cb(session, stream) + + # Wait for listener to be called + await self._wait_for_condition(lambda: on_stream_received_mock.await_count > 0) + on_stream_received_mock.assert_awaited_once_with(session, stream) + + # Wait to ensure no subscription happens + await self._wait_client_async_tasks(client) + + # Verify subscribe was NOT called (because subscription already exists) + self.mock_client_instance.subscribe.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_client_events(self) -> None: + """Test VonageClient events""" + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_in_sample_rate=48000, + audio_in_channels=2, + ) + client = await self._create_client(params) + + # Mock the connect method to return True + self.mock_client_instance.connect.return_value = True + self._setup_audio_ready_callback(client, call_ready_for_audio=True) + self._setup_subscriber_callbacks(client) + + # create a listener + listener = self.VonageClientListener() + on_error_mock = AsyncMock() + listener.on_error = on_error_mock + on_audio_in_mock = AsyncMock() + listener.on_audio_in = on_audio_in_mock + on_stream_received_mock = AsyncMock() + listener.on_stream_received = on_stream_received_mock + on_stream_dropped_mock = AsyncMock() + listener.on_stream_dropped = on_stream_dropped_mock + on_subscriber_connected_mock = AsyncMock() + listener.on_subscriber_connected = on_subscriber_connected_mock + on_subscriber_disconnected_mock = AsyncMock() + listener.on_subscriber_disconnected = on_subscriber_disconnected_mock + + client.add_listener(listener) + + # connect + await client.connect() + + assert self._connect_callbacks is not None + + # Test _on_session_error_cb triggers on_error + session = vonage_video_mock.models.Session(id="test_session") + error_description = "Test error description" + error_code = 500 + + self._connect_callbacks.on_error_cb(session, error_description, error_code) + await self._wait_for_condition(lambda: on_error_mock.await_count > 0) + + listener.on_error.assert_called_once_with(session, error_description, error_code) + listener.on_error.reset_mock() + + # Test _on_session_audio_data_cb triggers on_audio_in + audio_buffer = np.array([100, 200, 300, 400], dtype=np.int16) + mock_audio_data = vonage_video_mock.models.AudioData( + sample_buffer=memoryview(audio_buffer), + number_of_frames=2, + number_of_channels=2, + sample_rate=48000, + ) + + client._on_session_audio_data_cb(session, mock_audio_data) + await self._wait_for_condition(lambda: on_audio_in_mock.await_count > 0) + + listener.on_audio_in.assert_awaited_once_with(session, ANY) + frame = listener.on_audio_in.call_args[0][1] + assert frame.audio == audio_buffer.tobytes() + assert frame.sample_rate == 48000 + assert frame.num_channels == 2 + listener.on_audio_in.reset_mock() + # Test _on_stream_received_cb triggers on_stream_received + stream = vonage_video_mock.models.Stream(id="test_stream", connection=DUMMY_CONNECTION) + + client._on_stream_received_cb(session, stream) + await self._wait_for_condition(lambda: on_stream_received_mock.await_count > 0) + listener.on_stream_received.assert_awaited_once_with(session, stream) + + await self._wait_for_condition(lambda: stream.id in self._subscriber_callbacks) + + assert stream.id in self._subscriber_callbacks + callbacks = self._subscriber_callbacks[stream.id] + self.mock_client_instance.subscribe.assert_called_once_with( + stream, + settings=ANY, + on_error_cb=callbacks.on_error_cb, + on_connected_cb=callbacks.on_connected_cb, + on_disconnected_cb=callbacks.on_disconnected_cb, + on_render_frame_cb=client._on_subscriber_video_data_cb, + ) + listener.on_stream_received.reset_mock() + + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + callbacks.on_connected_cb(subscriber) + await self._wait_for_condition(lambda: on_subscriber_connected_mock.await_count > 0) + listener.on_subscriber_connected.assert_awaited_once() + listener.on_subscriber_connected.reset_mock() + + # Test _on_subscriber_disconnected_cb triggers on_subscriber_disconnected + callbacks.on_disconnected_cb(subscriber) + await self._wait_for_condition(lambda: on_subscriber_disconnected_mock.await_count > 0) + + listener.on_subscriber_disconnected.assert_awaited_once_with(subscriber) + listener.on_subscriber_disconnected.reset_mock() + + # Test _on_stream_dropped_cb triggers on_stream_dropped + self.mock_client_instance.unsubscribe = MagicMock() + + client._on_stream_dropped_cb(session, stream) + await self._wait_for_condition(lambda: on_stream_dropped_mock.await_count > 0) + + listener.on_stream_dropped.assert_awaited_once_with(session, stream) + self.mock_client_instance.unsubscribe.assert_not_called() + listener.on_stream_dropped.reset_mock() + + # Test _on_subscriber_connected_cb triggers on_subscriber_connected + subscriber_stream = vonage_video_mock.models.Stream( + id="subscriber_stream", connection=DUMMY_CONNECTION + ) + subscriber = vonage_video_mock.models.Subscriber(stream=subscriber_stream) + + # Test error callbacks are logged but don't trigger listener events + # (these are internal error callbacks, not session errors) + publisher_stream = vonage_video_mock.models.Stream( + id="publisher_stream", connection=DUMMY_CONNECTION + ) + publisher = vonage_video_mock.models.Publisher(stream=publisher_stream) + + # These should not raise exceptions + client._on_publisher_error_cb(publisher, "publisher error", 400) + callbacks.on_error_cb(subscriber, "subscriber error", 401) + + await self._wait_client_async_tasks(client) + + @pytest.mark.asyncio + async def test_vonage_client_on_subscriber_video_data_cb_rgb_format(self) -> None: + """Test _on_subscriber_video_data_cb with RGB format video frames.""" + from pipecat.frames.frames import UserImageRawFrame + + params = self.VonageVideoConnectorTransportParams( + video_in_enabled=True, + video_in_auto_subscribe=False, + ) + client = await self._create_client(params) + + # Add listener to capture video frames + listener = self.VonageClientListener() + on_video_in_mock = AsyncMock() + listener.on_video_in = on_video_in_mock + client.add_listener(listener) + + await client.connect() + + # Create a test RGB video frame (4x4, 3 channels) + width, height = 4, 4 + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + rgb_image[:, :, 0] = 100 # R channel + rgb_image[:, :, 1] = 150 # G channel + rgb_image[:, :, 2] = 200 # B channel + + rgb_bytes = rgb_image.tobytes() + + # Create mock video frame with RGB24 format (which Vonage uses for RGB) + mock_video_frame = vonage_video_mock.models.VideoFrame( + frame_buffer=memoryview(rgb_bytes), + resolution=MockVideoResolution(width=width, height=height), + format="RGB24", + ) + + # Create mock subscriber + stream = vonage_video_mock.models.Stream(id="video_stream", connection=DUMMY_CONNECTION) + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + + # Trigger the callback + client._on_subscriber_video_data_cb(subscriber, mock_video_frame) + + # Wait for async processing + await self._wait_for_condition(lambda: on_video_in_mock.await_count > 0) + + # Verify listener was called + on_video_in_mock.assert_awaited_once() + call_args = on_video_in_mock.call_args[0] + assert call_args[0] == subscriber + + # Get the processed frame + processed_frame: UserImageRawFrame = call_args[1] + assert processed_frame.user_id == "video_stream" + assert processed_frame.size == (width, height) + assert processed_frame.format == "RGB" + + # Verify BGR to RGB conversion happened + processed_image = np.frombuffer(processed_frame.image, dtype=np.uint8).reshape( + height, width, 3 + ) + assert processed_image[0, 0, 0] == 200 # R channel (was B in BGR) + assert processed_image[0, 0, 1] == 150 # G channel (unchanged) + assert processed_image[0, 0, 2] == 100 # B channel (was R in BGR) + + @pytest.mark.asyncio + async def test_vonage_input_transport_initialization(self) -> None: + """Test VonageVideoConnectorInputTransport initialization.""" + params = self.VonageVideoConnectorTransportParams() + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + transport_params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + transport = self.VonageVideoConnectorInputTransport(client, transport_params) + + assert transport._client == client + assert transport._initialized is False + + @pytest.mark.asyncio + async def test_vonage_input_transport_start(self) -> None: + """Test VonageVideoConnectorInputTransport start method.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + transport = self.VonageVideoConnectorInputTransport(client, params) + + # Mock the client connect method + with ( + patch.object(client, "connect", AsyncMock(return_value=1)) as client_connect_mock, + patch.object(transport, "set_transport_ready", AsyncMock()) as set_transport_ready_mock, + ): + start_frame = StartFrame() + await transport.start(start_frame) + + assert transport._initialized is True + assert transport._connected is True + client_connect_mock.assert_called_once() + set_transport_ready_mock.assert_called_once_with(start_frame) + + @pytest.mark.asyncio + async def test_vonage_input_transport_stop(self) -> None: + """Test VonageVideoConnectorInputTransport stop method.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + transport = self.VonageVideoConnectorInputTransport(client, params) + transport._listener_id = 1 + transport._connected = True + + with ( + patch.object(client, "disconnect", AsyncMock()) as client_disconnect_mock, + patch.object(client, "remove_listener", MagicMock()) as remove_listener_mock, + ): + end_frame = EndFrame() + await transport.stop(end_frame) + + client_disconnect_mock.assert_called_once() + remove_listener_mock.assert_called_once_with(1) + assert not transport._connected + + @pytest.mark.asyncio + async def test_vonage_input_transport_cancel(self) -> None: + """Test VonageVideoConnectorInputTransport cancel method.""" + params = self.VonageVideoConnectorTransportParams(audio_in_enabled=True) + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + transport = self.VonageVideoConnectorInputTransport(client, params) + transport._listener_id = 1 + transport._connected = True + + # Mock the client disconnect method + with ( + patch.object(client, "disconnect", AsyncMock()) as client_disconnect_mock, + patch.object(client, "remove_listener", MagicMock()) as remove_listener_mock, + ): + cancel_frame = CancelFrame() + await transport.cancel(cancel_frame) + + client_disconnect_mock.assert_called_once() + remove_listener_mock.assert_called_once_with(1) + assert not transport._connected + + @pytest.mark.asyncio + async def test_vonage_output_transport_initialization(self) -> None: + """Test VonageVideoConnectorOutputTransport initialization.""" + params = self.VonageVideoConnectorTransportParams() + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + transport_params = self.VonageVideoConnectorTransportParams(audio_out_enabled=True) + transport = self.VonageVideoConnectorOutputTransport(client, transport_params) + + assert transport._client == client + assert transport._initialized is False + + @pytest.mark.asyncio + async def test_vonage_output_transport_start(self) -> None: + """Test VonageVideoConnectorOutputTransport start method.""" + params = self.VonageVideoConnectorTransportParams() + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + transport_params = self.VonageVideoConnectorTransportParams(audio_out_enabled=True) + transport = self.VonageVideoConnectorOutputTransport(client, transport_params) + + with ( + patch.object(client, "connect", AsyncMock(return_value=1)) as client_connect_mock, + patch.object(transport, "set_transport_ready", AsyncMock()) as set_transport_ready_mock, + ): + start_frame = StartFrame() + await transport.start(start_frame) + + assert transport._initialized is True + client_connect_mock.assert_called_once() + set_transport_ready_mock.assert_called_once_with(start_frame) + + @pytest.mark.asyncio + async def test_vonage_output_transport_write_audio_frame(self) -> None: + """Test VonageVideoConnectorOutputTransport write_audio_frame method.""" + + params = self.VonageVideoConnectorTransportParams( + audio_out_sample_rate=48000, audio_out_channels=2, audio_out_enabled=True + ) + client = self.VonageClient(self.application_id, self.session_id, self.token, params) + + with patch.object(client, "write_audio", AsyncMock()) as client_write_audio_mock: + transport_params = self.VonageVideoConnectorTransportParams(audio_out_enabled=True) + transport = self.VonageVideoConnectorOutputTransport(client, transport_params) + transport._connected = True + + # Create a mock audio frame + audio_frame = OutputAudioRawFrame( + audio=b"\x00\x01\x02\x03", sample_rate=16000, num_channels=1 + ) + + await transport.write_audio_frame(audio_frame) + + # Verify audio was written to client + client_write_audio_mock.assert_called_once_with(audio_frame) + + @pytest.mark.asyncio + async def test_vonage_output_transport_write_video_frame_not_connected(self) -> None: + """Test VonageVideoConnectorOutputTransport write_video_frame method.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams(video_out_enabled=True) + ) + client = transport._client + + # Create a test video frame + width, height = 640, 480 + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + rgb_image[:, :, 0] = 100 + rgb_image[:, :, 1] = 150 + rgb_image[:, :, 2] = 200 + + video_frame = OutputImageRawFrame( + image=rgb_image.tobytes(), size=(width, height), format="RGB" + ) + + with patch.object(client, "write_video", AsyncMock(return_value=True)) as write_video_mock: + await transport.stop(EndFrame()) + result = await transport.write_video_frame(video_frame) + + # Should return False when not connected + assert result is False + write_video_mock.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_output_transport_write_video_frame_connected(self) -> None: + """Test VonageVideoConnectorOutputTransport write_video_frame method when connected.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams( + video_out_enabled=True, + video_out_width=640, + video_out_height=480, + video_out_color_format="RGB", + ) + ) + client = transport._client + + # Create a test video frame + width, height = 640, 480 + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + rgb_image[:, :, 0] = 100 + rgb_image[:, :, 1] = 150 + rgb_image[:, :, 2] = 200 + + video_frame = OutputImageRawFrame( + image=rgb_image.tobytes(), size=(width, height), format="RGB" + ) + + with patch.object(client, "write_video", AsyncMock(return_value=True)) as write_video_mock: + transport._connected = True + result = await transport.write_video_frame(video_frame) + + # Should return True and call write_video when connected + assert result is True + write_video_mock.assert_called_once_with(video_frame) + + @pytest.mark.asyncio + async def test_vonage_output_transport_write_video_frame_invalid_size(self) -> None: + """Test VonageVideoConnectorOutputTransport write_video_frame with invalid frame size.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams( + video_out_enabled=True, + video_out_width=640, + video_out_height=480, + video_out_color_format="RGB", + ) + ) + + # Create a video frame with incorrect size + width, height = 320, 240 # Different from expected 640x480 + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + + video_frame = OutputImageRawFrame( + image=rgb_image.tobytes(), size=(width, height), format="RGB" + ) + + transport._connected = True + result = await transport.write_video_frame(video_frame) + + # Should return False for invalid size + assert result is False + + @pytest.mark.asyncio + async def test_vonage_output_transport_write_video_frame_invalid_format(self) -> None: + """Test VonageVideoConnectorOutputTransport write_video_frame with invalid color format.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams( + video_out_enabled=True, + video_out_width=640, + video_out_height=480, + video_out_color_format="YCbCr", + ) + ) + + # Create a video frame with incorrect size + width, height = 320, 240 # Different from expected 640x480 + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + + video_frame = OutputImageRawFrame( + image=rgb_image.tobytes(), size=(width, height), format="RGB" + ) + + transport._connected = True + result = await transport.write_video_frame(video_frame) + + # Should return False for invalid size + assert result is False + + @pytest.mark.asyncio + async def test_vonage_output_transport_process_frame_with_interruption(self) -> None: + """Test VonageVideoConnectorOutputTransport process_frame method with InterruptionFrame.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams(audio_out_enabled=True) + ) + client = transport._client + + with ( + patch.object(client, "clear_media_buffers") as clear_buffers_mock, + patch.object(client, "connect", AsyncMock()), + ): + await transport.start(StartFrame()) + interruption_frame = InterruptionFrame() + await transport.process_frame(interruption_frame, FrameDirection.DOWNSTREAM) + + # Verify clear_media_buffers was called + clear_buffers_mock.assert_called_once() + + @pytest.mark.asyncio + async def test_vonage_output_transport_process_frame_without_interruption(self) -> None: + """Test VonageVideoConnectorOutputTransport process_frame method with non-interruption frame.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams(audio_out_enabled=True) + ) + client = transport._client + + with patch.object(client, "clear_media_buffers") as clear_buffers_mock: + audio_frame = OutputAudioRawFrame( + audio=b"\x00\x01\x02\x03", sample_rate=16000, num_channels=1 + ) + await transport.process_frame(audio_frame, FrameDirection.DOWNSTREAM) + + # Verify clear_media_buffers was NOT called for non-interruption frames + clear_buffers_mock.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_output_transport_process_frame_when_not_connected(self) -> None: + """Test VonageVideoConnectorOutputTransport process_frame method when not connected.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams(audio_out_enabled=True) + ) + await transport.stop(EndFrame()) # Ensure transport is not connected + client = transport._client + + with patch.object(client, "clear_media_buffers") as clear_buffers_mock: + interruption_frame = InterruptionFrame() + await transport.process_frame(interruption_frame, FrameDirection.DOWNSTREAM) + + # Verify clear_media_buffers was NOT called when not connected + clear_buffers_mock.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_output_transport_interruption_with_clear_buffers_disabled(self) -> None: + """Test VonageVideoConnectorOutputTransport with clear_buffers_on_interruption=False.""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams( + audio_out_enabled=True, clear_buffers_on_interruption=False + ) + ) + client = transport._client + + with ( + patch.object(client, "clear_media_buffers") as clear_buffers_mock, + patch.object(client, "connect", AsyncMock()), + ): + await transport.start(StartFrame()) + interruption_frame = InterruptionFrame() + await transport.process_frame(interruption_frame, FrameDirection.DOWNSTREAM) + + # Verify clear_media_buffers was NOT called when clear_buffers_on_interruption is False + clear_buffers_mock.assert_not_called() + + @pytest.mark.asyncio + async def test_vonage_output_transport_interruption_with_clear_buffers_enabled(self) -> None: + """Test VonageVideoConnectorOutputTransport with clear_buffers_on_interruption=True (default).""" + transport = await self._create_output_transport( + params=self.VonageVideoConnectorTransportParams( + audio_out_enabled=True, clear_buffers_on_interruption=True + ) + ) + client = transport._client + + with ( + patch.object(client, "clear_media_buffers") as clear_buffers_mock, + patch.object(client, "connect", AsyncMock()), + ): + await transport.start(StartFrame()) + interruption_frame = InterruptionFrame() + await transport.process_frame(interruption_frame, FrameDirection.DOWNSTREAM) + + # Verify clear_media_buffers was called when clear_buffers_on_interruption is True + clear_buffers_mock.assert_called_once() + + @pytest.mark.asyncio + @pytest.mark.parametrize("transport_type", ["input", "output"]) + async def test_vonage_transport_sets_audio_sample_rates_from_start_frame( + self, transport_type: str + ) -> None: + """Test transport sets audio sample rates from StartFrame when params are None.""" + # Create params with None sample rates + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=(transport_type == "input"), + audio_out_enabled=(transport_type == "output"), + audio_in_sample_rate=None, + audio_out_sample_rate=None, + ) + transport: VonageVideoConnectorInputTransport | VonageVideoConnectorOutputTransport + if transport_type == "input": + transport = await self._create_input_transport(params=params) + else: + transport = await self._create_output_transport(params=params) + client = transport._client + + # Create a StartFrame with specific sample rates + start_frame = StartFrame(audio_in_sample_rate=22050, audio_out_sample_rate=44100) + + with patch.object(client, "_sdk_connect", AsyncMock()): + await transport.start(start_frame) + + # Verify both sample rates were set from the StartFrame + assert client._audio_in_sample_rate == 22050 + assert client._audio_out_sample_rate == 44100 + + @pytest.mark.asyncio + @pytest.mark.parametrize("transport_type", ["input", "output"]) + async def test_vonage_transport_doesnt_override_audio_sample_rates( + self, transport_type: str + ) -> None: + """Test transport doesn't override audio sample rates when already set in params.""" + # Create params with specific sample rates + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=(transport_type == "input"), + audio_out_enabled=(transport_type == "output"), + audio_in_sample_rate=48000, + audio_out_sample_rate=16000, + ) + transport: VonageVideoConnectorInputTransport | VonageVideoConnectorOutputTransport + if transport_type == "input": + transport = await self._create_input_transport(params=params) + else: + transport = await self._create_output_transport(params=params) + client = transport._client + + # Create a StartFrame with different sample rates + start_frame = StartFrame(audio_in_sample_rate=22050, audio_out_sample_rate=44100) + + with patch.object(client, "_sdk_connect", AsyncMock()): + await transport.start(start_frame) + + # Verify sample rates remain as originally set in params + assert client._audio_in_sample_rate == 48000 + assert client._audio_out_sample_rate == 16000 + + @pytest.mark.asyncio + async def test_vonage_transport_initialization(self) -> None: + """Test VonageVideoConnectorTransport initialization.""" + params = self.VonageVideoConnectorTransportParams( + audio_out_sample_rate=48000, + audio_out_channels=2, + audio_out_enabled=True, + session_enable_migration=True, + publisher_name="test-publisher", + publisher_enable_opus_dtx=True, + ) + transport = await self._create_transport(params=params) + + assert transport._client is not None + assert transport._one_stream_received is False + + # Verify vonage client was initialized with correct parameters + client_params = transport._client._params + assert client_params.audio_out_sample_rate == 48000 + assert client_params.audio_out_channels == 2 + assert client_params.session_enable_migration is True + + @pytest.mark.asyncio + async def test_vonage_transport_input_output_methods(self) -> None: + """Test VonageVideoConnectorTransport input and output methods.""" + params = self.VonageVideoConnectorTransportParams() + transport = self.VonageVideoConnectorTransport( + self.application_id, self.session_id, self.token, params + ) + + # Test input method + input_transport = transport.input() + assert isinstance(input_transport, self.VonageVideoConnectorInputTransport) + + # Test output method + output_transport = transport.output() + assert isinstance(output_transport, self.VonageVideoConnectorOutputTransport) + + # Verify they return the same instances on subsequent calls + assert transport.input() is input_transport + assert transport.output() is output_transport + + @pytest.mark.asyncio + async def test_vonage_input_audio_callback(self) -> None: + """Test audio input callback processing.""" + + params = self.VonageVideoConnectorTransportParams( + audio_in_enabled=True, + ) + transport = await self._create_input_transport(params) + client = transport._client + + with ( + patch.object(transport, "push_audio_frame", AsyncMock()) as mock_push_audio_frame, + patch.object(client, "connect", AsyncMock(return_value=1)), + ): + start_frame = StartFrame() + await transport.start(start_frame) + + # Create mock audio data + audio_buffer = np.array([100, 200, 300, 400], dtype=np.int16) + audio_frame = InputAudioRawFrame( + audio=audio_buffer.tobytes(), sample_rate=48000, num_channels=2 + ) + + # Call the audio callback + await transport._audio_in_cb( + vonage_video_mock.models.Session(id="session"), audio_frame + ) + + mock_push_audio_frame.assert_called_once_with(audio_frame) + + @pytest.mark.asyncio + async def test_vonage_input_video_callback(self) -> None: + """Test video input callback processing.""" + + params = self.VonageVideoConnectorTransportParams( + video_in_enabled=True, + ) + transport = await self._create_input_transport(params) + client = transport._client + + with ( + patch.object(transport, "push_video_frame", AsyncMock()) as mock_push_video_frame, + patch.object(client, "connect", AsyncMock(return_value=1)), + ): + start_frame = StartFrame() + await transport.start(start_frame) + + # Create mock video frame + width, height = 640, 480 + rgb_image = np.zeros((height, width, 3), dtype=np.uint8) + rgb_image[:, :, 0] = 100 # R channel + rgb_image[:, :, 1] = 150 # G channel + rgb_image[:, :, 2] = 200 # B channel + + video_frame = UserImageRawFrame( + user_id="test-user", image=rgb_image.tobytes(), size=(width, height), format="RGB" + ) + + # Create mock subscriber + stream = vonage_video_mock.models.Stream(id="video_stream", connection=DUMMY_CONNECTION) + subscriber = vonage_video_mock.models.Subscriber(stream=stream) + + # Call the video callback + await transport._video_in_cb(subscriber, video_frame) + + mock_push_video_frame.assert_called_once_with(video_frame) + + @pytest.mark.asyncio + async def test_vonage_transport_event_handlers(self) -> None: + """Test VonageVideoConnectorTransport event handlers.""" + params = self.VonageVideoConnectorTransportParams() + transport = await self._create_transport(params) + + with patch.object( + transport, "_call_event_handler", new_callable=AsyncMock + ) as mock_call_event_handler: + # Test session events + mock_session = Mock() + mock_session.id = "session-123" + + await transport._on_connected(mock_session) + mock_call_event_handler.assert_called_with("on_joined", {"sessionId": "session-123"}) + + await transport._on_disconnected(mock_session) + mock_call_event_handler.assert_called_with("on_left", {"sessionId": "session-123"}) + + await transport._on_error(mock_session, "test error", 500) + mock_call_event_handler.assert_called_with("on_error", "test error") + + # Test stream events + mock_connection = Mock() + mock_connection.data = "connection-data-123" + mock_stream = Mock() + mock_stream.id = "stream-456" + mock_stream.connection = mock_connection + + await transport._on_stream_received(mock_session, mock_stream) + # Should call both first participant and participant joined events + expected_calls = [ + call( + "on_first_participant_joined", + { + "sessionId": "session-123", + "streamId": "stream-456", + "connectionData": "connection-data-123", + }, + ), + call( + "on_participant_joined", + { + "sessionId": "session-123", + "streamId": "stream-456", + "connectionData": "connection-data-123", + }, + ), + ] + mock_call_event_handler.assert_has_calls(expected_calls) + + await transport._on_stream_dropped(mock_session, mock_stream) + mock_call_event_handler.assert_called_with( + "on_participant_left", + { + "sessionId": "session-123", + "streamId": "stream-456", + "connectionData": "connection-data-123", + }, + ) + + # Test subscriber events + mock_subscriber = Mock() + mock_subscriber.stream = Mock() + mock_subscriber.stream.id = "subscriber-789" + mock_subscriber.stream.connection = Mock() + mock_subscriber.stream.connection.data = "subscriber-conn-data" + + await transport._on_subscriber_connected(mock_subscriber) + mock_call_event_handler.assert_called_with( + "on_client_connected", + { + "subscriberId": "subscriber-789", + "streamId": "subscriber-789", + "connectionData": "subscriber-conn-data", + }, + ) + + await transport._on_subscriber_disconnected(mock_subscriber) + mock_call_event_handler.assert_called_with( + "on_client_disconnected", + { + "subscriberId": "subscriber-789", + "streamId": "subscriber-789", + "connectionData": "subscriber-conn-data", + }, + ) + + @pytest.mark.asyncio + async def test_vonage_transport_first_participant_flag(self) -> None: + """Test that first participant event is only called once.""" + params = self.VonageVideoConnectorTransportParams() + transport = await self._create_transport(params) + + with patch.object( + transport, "_call_event_handler", new_callable=AsyncMock + ) as mock_call_event_handler: + mock_session = Mock() + mock_session.id = "session-123" + + mock_connection1 = Mock() + mock_connection1.data = "conn-data-1" + mock_stream1 = Mock() + mock_stream1.id = "stream-456" + mock_stream1.connection = mock_connection1 + + mock_connection2 = Mock() + mock_connection2.data = "conn-data-2" + mock_stream2 = Mock() + mock_stream2.id = "stream-789" + mock_stream2.connection = mock_connection2 + + # First stream should trigger first participant event + await transport._on_stream_received(mock_session, mock_stream1) + assert transport._one_stream_received is True + + # Reset mock to check second stream + mock_call_event_handler.reset_mock() + + # Second stream should not trigger first participant event + await transport._on_stream_received(mock_session, mock_stream2) + mock_call_event_handler.assert_called_once_with( + "on_participant_joined", + { + "sessionId": "session-123", + "streamId": "stream-789", + "connectionData": "conn-data-2", + }, + ) + + +class TestAudioNormalization: + """Test cases for audio normalization functions.""" + + def setup_method(self) -> None: + """Set up test fixtures.""" + self.AudioProps = AudioProps + self.process_audio_channels = process_audio_channels + self.process_audio = process_audio + self.check_audio_data = check_audio_data + + def test_audio_props_creation(self) -> None: + """Test AudioProps dataclass creation.""" + props = self.AudioProps(sample_rate=48000, is_stereo=True) + assert props.sample_rate == 48000 + assert props.is_stereo is True + + props_mono = self.AudioProps(sample_rate=16000, is_stereo=False) + assert props_mono.sample_rate == 16000 + assert props_mono.is_stereo is False + + def test_process_audio_channels_mono_to_stereo(self) -> None: + """Test converting mono audio to stereo.""" + # Create mono audio (4 samples) + mono_audio = np.array([100, 200, 300, 400], dtype=np.int16) + + current = self.AudioProps(sample_rate=48000, is_stereo=False) + target = self.AudioProps(sample_rate=48000, is_stereo=True) + + result = self.process_audio_channels(mono_audio, current, target) + + # Should duplicate each sample + expected = np.array([100, 100, 200, 200, 300, 300, 400, 400], dtype=np.int16) + np.testing.assert_array_equal(result, expected) + + def test_process_audio_channels_stereo_to_mono(self) -> None: + """Test converting stereo audio to mono.""" + # Create stereo audio (2 frames, 4 samples total) + stereo_audio = np.array([100, 200, 300, 400], dtype=np.int16) + + current = self.AudioProps(sample_rate=48000, is_stereo=True) + target = self.AudioProps(sample_rate=48000, is_stereo=False) + + result = self.process_audio_channels(stereo_audio, current, target) + + # Should average each stereo pair: (100+200)/2=150, (300+400)/2=350 + expected = np.array([150, 350], dtype=np.int16) + np.testing.assert_array_equal(result, expected) + + def test_process_audio_channels_same_format(self) -> None: + """Test when source and target have the same channel format.""" + audio = np.array([100, 200, 300, 400], dtype=np.int16) + + # Test mono to mono + current = self.AudioProps(sample_rate=48000, is_stereo=False) + target = self.AudioProps(sample_rate=48000, is_stereo=False) + result = self.process_audio_channels(audio, current, target) + np.testing.assert_array_equal(result, audio) + + # Test stereo to stereo + current = self.AudioProps(sample_rate=48000, is_stereo=True) + target = self.AudioProps(sample_rate=48000, is_stereo=True) + result = self.process_audio_channels(audio, current, target) + np.testing.assert_array_equal(result, audio) + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.create_stream_resampler") + async def test_process_audio_same_sample_rate(self, mock_resampler: MagicMock) -> None: + """Test process_audio when sample rates are the same.""" + mock_resampler_instance = Mock() + mock_resampler.return_value = mock_resampler_instance + + audio = np.array([100, 200, 300, 400], dtype=np.int16) + current = self.AudioProps(sample_rate=48000, is_stereo=False) + target = self.AudioProps(sample_rate=48000, is_stereo=True) + + result = await self.process_audio(mock_resampler_instance, audio, current, target) + + # Should only do channel conversion, no resampling + expected = np.array([100, 100, 200, 200, 300, 300, 400, 400], dtype=np.int16) + np.testing.assert_array_equal(result, expected) + + # Resampler should not be called + mock_resampler_instance.resample.assert_not_called() + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.create_stream_resampler") + async def test_process_audio_different_sample_rate_mono( + self, mock_resampler: MagicMock + ) -> None: + """Test process_audio with different sample rates (mono).""" + mock_resampler_instance = Mock() + mock_resampler_instance.resample = AsyncMock( + return_value=b"\x64\x00\xc8\x00" + ) # 100, 200 in bytes + mock_resampler.return_value = mock_resampler_instance + + audio = np.array([150, 250, 350, 450], dtype=np.int16) + current = self.AudioProps(sample_rate=48000, is_stereo=False) + target = self.AudioProps(sample_rate=16000, is_stereo=False) + + result = await self.process_audio(mock_resampler_instance, audio, current, target) + + # Should resample the audio + expected = np.array([100, 200], dtype=np.int16) + np.testing.assert_array_equal(result, expected) + + # Resampler should be called with correct parameters + mock_resampler_instance.resample.assert_called_once_with(audio.tobytes(), 48000, 16000) + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.create_stream_resampler") + async def test_process_audio_different_sample_rate_stereo_to_mono( + self, mock_resampler: MagicMock + ) -> None: + """Test process_audio with different sample rates and channel conversion.""" + mock_resampler_instance = Mock() + # Return resampled mono data + mock_resampler_instance.resample = AsyncMock( + return_value=b"\x64\x00\xc8\x00" + ) # 100, 200 in bytes + mock_resampler.return_value = mock_resampler_instance + + # Stereo audio: 2 frames with left/right channels + audio = np.array([100, 200, 300, 400], dtype=np.int16) # L1=100, R1=200, L2=300, R2=400 + current = self.AudioProps(sample_rate=48000, is_stereo=True) + target = self.AudioProps(sample_rate=16000, is_stereo=False) + + result = await self.process_audio(mock_resampler_instance, audio, current, target) + + # Should convert to mono first, then resample + expected = np.array([100, 200], dtype=np.int16) + np.testing.assert_array_equal(result, expected) + + # Resampler should be called with mono audio + expected_mono = np.array([150, 350], dtype=np.int16) # (100+200)/2, (300+400)/2 + mock_resampler_instance.resample.assert_called_once_with( + expected_mono.tobytes(), 48000, 16000 + ) + + @pytest.mark.asyncio + @patch("pipecat.transports.vonage.client.create_stream_resampler") + async def test_process_audio_different_sample_rate_mono_to_stereo( + self, mock_resampler: MagicMock + ) -> None: + """Test process_audio with different sample rates converting mono to stereo.""" + mock_resampler_instance = Mock() + # Return resampled mono data + mock_resampler_instance.resample = AsyncMock( + return_value=b"\x64\x00\xc8\x00" + ) # 100, 200 in bytes + mock_resampler.return_value = mock_resampler_instance + + audio = np.array([150, 250], dtype=np.int16) + current = self.AudioProps(sample_rate=48000, is_stereo=False) + target = self.AudioProps(sample_rate=16000, is_stereo=True) + + result = await self.process_audio(mock_resampler_instance, audio, current, target) + + # Should resample first (mono), then convert to stereo + expected = np.array([100, 100, 200, 200], dtype=np.int16) + np.testing.assert_array_equal(result, expected) + + # Resampler should be called with mono audio + mock_resampler_instance.resample.assert_called_once_with(audio.tobytes(), 48000, 16000) + + def test_check_audio_data_valid_mono_bytes(self) -> None: + """Test check_audio_data with valid mono audio as bytes.""" + # 4 frames of mono 16-bit audio (8 bytes total) + buffer = b"\x00\x01\x02\x03\x04\x05\x06\x07" + + # Should not raise any exception + self.check_audio_data(buffer, 4, 1) + + def test_check_audio_data_valid_stereo_bytes(self) -> None: + """Test check_audio_data with valid stereo audio as bytes.""" + # 2 frames of stereo 16-bit audio (8 bytes total) + buffer = b"\x00\x01\x02\x03\x04\x05\x06\x07" + + # Should not raise any exception + self.check_audio_data(buffer, 2, 2) + + def test_check_audio_data_valid_memoryview(self) -> None: + """Test check_audio_data with valid audio as memoryview.""" + # Create int16 memoryview (2 bytes per sample) + array = np.array([100, 200, 300, 400], dtype=np.int16) + buffer = memoryview(array) + + # Should not raise any exception + self.check_audio_data(buffer, 4, 1) # 4 mono frames + self.check_audio_data(buffer, 2, 2) # 2 stereo frames + + def test_check_audio_data_invalid_channels(self) -> None: + """Test check_audio_data with invalid number of channels.""" + buffer = b"\x00\x01\x02\x03" + + # Should raise ValueError for invalid channel counts + with pytest.raises(ValueError) as exc_info: + self.check_audio_data(buffer, 2, 3) # 3 channels not supported + assert "mono or stereo" in str(exc_info.value) + + with pytest.raises(ValueError) as exc_info: + self.check_audio_data(buffer, 2, 0) # 0 channels not supported + assert "mono or stereo" in str(exc_info.value) + + def test_check_audio_data_invalid_bit_depth_bytes(self) -> None: + """Test check_audio_data with invalid bit depth using bytes.""" + # 2 frames of mono audio with 1 byte per sample (8-bit) + buffer = b"\x00\x01" + + with pytest.raises(ValueError) as exc_info: + self.check_audio_data(buffer, 2, 1) + assert "16 bit PCM" in str(exc_info.value) + assert "got 8 bit" in str(exc_info.value) + + def test_check_audio_data_invalid_bit_depth_memoryview(self) -> None: + """Test check_audio_data with invalid bit depth using memoryview.""" + # Create uint8 memoryview (1 byte per sample) + array = np.array([100, 200], dtype=np.uint8) + buffer = memoryview(array) + + with pytest.raises(ValueError) as exc_info: + self.check_audio_data(buffer, 2, 1) + assert "16 bit PCM" in str(exc_info.value) + assert "got 8 bit" in str(exc_info.value) + + def test_check_audio_data_buffer_size_mismatch(self) -> None: + """Test check_audio_data with buffer size that doesn't match expected size.""" + # 3 bytes total, but expecting 2 frames of mono 16-bit (should be 4 bytes) + buffer = b"\x00\x01\x02" + + with pytest.raises(ValueError) as exc_info: + self.check_audio_data(buffer, 2, 1) + # Should detect that 3 bytes / (2 frames * 1 channel) = 1.5 bytes per sample + # which gets truncated to 1 byte per sample = 8 bit + assert "16 bit PCM" in str(exc_info.value) + + +class TestColorspaceConversion: + """Test cases for image colorspace conversion functions.""" + + def test_same_format_no_conversion(self) -> None: + """Test that conversion with same source and target format returns original image.""" + width, height = 4, 4 + image_data = np.random.randint(0, 256, width * height * 3, dtype=np.uint8).tobytes() + + # Test all formats with themselves + for fmt in ImageFormat: + result = image_colorspace_conversion(image_data, (width, height), fmt, fmt) + assert result == image_data + + def test_rgb_to_bgr_conversion(self) -> None: + """Test RGB to BGR conversion.""" + width, height = 2, 2 + # Create a simple RGB image with distinct colors + rgb_image = np.array( + [ + [[255, 0, 0], [0, 255, 0]], # Red, Green + [[0, 0, 255], [255, 255, 0]], # Blue, Yellow + ], + dtype=np.uint8, + ) + + result = image_colorspace_conversion( + rgb_image.tobytes(), + (width, height), + ImageFormat.RGB, + ImageFormat.BGR, + ) + + # Expected BGR: R and B channels swapped + expected = np.array( + [ + [[0, 0, 255], [0, 255, 0]], # Blue, Green (unchanged) + [[255, 0, 0], [0, 255, 255]], # Red, Cyan + ], + dtype=np.uint8, + ) + + assert result is not None + result_array = np.frombuffer(result, dtype=np.uint8).reshape(height, width, 3) + np.testing.assert_array_equal(result_array, expected) + + def test_bgr_to_rgb_conversion(self) -> None: + """Test BGR to RGB conversion (should be same as RGB to BGR).""" + width, height = 2, 2 + bgr_image = np.array( + [ + [[255, 0, 0], [0, 255, 0]], + [[0, 0, 255], [255, 255, 0]], + ], + dtype=np.uint8, + ) + + result = image_colorspace_conversion( + bgr_image.tobytes(), + (width, height), + ImageFormat.BGR, + ImageFormat.RGB, + ) + + # R and B channels should be swapped + expected = np.array( + [ + [[0, 0, 255], [0, 255, 0]], + [[255, 0, 0], [0, 255, 255]], + ], + dtype=np.uint8, + ) + + assert result is not None + result_array = np.frombuffer(result, dtype=np.uint8).reshape(height, width, 3) + np.testing.assert_array_equal(result_array, expected) + + def test_rgba_to_bgra_conversion(self) -> None: + """Test RGBA to BGRA conversion.""" + width, height = 2, 2 + rgba_image = np.array( + [ + [[255, 0, 0, 255], [0, 255, 0, 200]], # Red opaque, Green semi-transparent + [[0, 0, 255, 150], [255, 255, 0, 100]], # Blue, Yellow + ], + dtype=np.uint8, + ) + + result = image_colorspace_conversion( + rgba_image.tobytes(), + (width, height), + ImageFormat.RGBA, + ImageFormat.BGRA, + ) + + # Expected: R and B swapped, alpha unchanged + expected = np.array( + [ + [[0, 0, 255, 255], [0, 255, 0, 200]], + [[255, 0, 0, 150], [0, 255, 255, 100]], + ], + dtype=np.uint8, + ) + + assert result is not None + result_array = np.frombuffer(result, dtype=np.uint8).reshape(height, width, 4) + np.testing.assert_array_equal(result_array, expected) + + def test_bgra_to_rgba_conversion(self) -> None: + """Test BGRA to RGBA conversion.""" + width, height = 2, 2 + bgra_image = np.array( + [ + [[255, 0, 0, 255], [0, 255, 0, 200]], + [[0, 0, 255, 150], [255, 255, 0, 100]], + ], + dtype=np.uint8, + ) + + result = image_colorspace_conversion( + bgra_image.tobytes(), + (width, height), + ImageFormat.BGRA, + ImageFormat.RGBA, + ) + + expected = np.array( + [ + [[0, 0, 255, 255], [0, 255, 0, 200]], + [[255, 0, 0, 150], [0, 255, 255, 100]], + ], + dtype=np.uint8, + ) + + assert result is not None + result_array = np.frombuffer(result, dtype=np.uint8).reshape(height, width, 4) + np.testing.assert_array_equal(result_array, expected) + + def test_planar_yuv420_to_packed_yuv444_conversion(self) -> None: + """Test planar YUV420 to packed YUV444 conversion.""" + width, height = 4, 4 + + # Create YUV420 planar data + # Y plane: 4x4 = 16 bytes + y_plane = np.array( + [100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250], + dtype=np.uint8, + ) + + # U plane: 2x2 = 4 bytes (subsampled) + u_plane = np.array([50, 60, 70, 80], dtype=np.uint8) + + # V plane: 2x2 = 4 bytes (subsampled) + v_plane = np.array([90, 100, 110, 120], dtype=np.uint8) + + yuv420_data = np.concatenate([y_plane, u_plane, v_plane]) + + result = image_colorspace_conversion( + yuv420_data.tobytes(), + (width, height), + ImageFormat.PLANAR_YUV420, + ImageFormat.PACKED_YUV444, + ) + + assert result is not None + result_array = np.frombuffer(result, dtype=np.uint8).reshape(height, width, 3) + + # Check that Y plane values are preserved + assert result_array[0, 0, 0] == 100 + assert result_array[0, 1, 0] == 110 + assert result_array[3, 3, 0] == 250 + + # Check that U and V planes are upsampled (each 2x2 block should have same U/V values) + # Top-left 2x2 block should have U=50, V=90 + assert result_array[0, 0, 1] == 50 + assert result_array[0, 0, 2] == 90 + assert result_array[0, 1, 1] == 50 + assert result_array[0, 1, 2] == 90 + assert result_array[1, 0, 1] == 50 + assert result_array[1, 0, 2] == 90 + assert result_array[1, 1, 1] == 50 + assert result_array[1, 1, 2] == 90 + + # Top-right 2x2 block should have U=60, V=100 + assert result_array[0, 2, 1] == 60 + assert result_array[0, 2, 2] == 100 + + def test_packed_yuv444_to_planar_yuv420_conversion(self) -> None: + """Test packed YUV444 to planar YUV420 conversion.""" + width, height = 4, 4 + + # Create packed YUV444 data (interleaved YUVYUVYUV...) + # Each pixel has Y, U, V values + packed_yuv444 = np.zeros((height, width, 3), dtype=np.uint8) + + # Set Y values + packed_yuv444[:, :, 0] = np.arange(100, 100 + width * height, dtype=np.uint8).reshape( + height, width + ) + + # Set U values (will be downsampled) + packed_yuv444[0:2, 0:2, 1] = 50 # Top-left block + packed_yuv444[0:2, 2:4, 1] = 60 # Top-right block + packed_yuv444[2:4, 0:2, 1] = 70 # Bottom-left block + packed_yuv444[2:4, 2:4, 1] = 80 # Bottom-right block + + # Set V values (will be downsampled) + packed_yuv444[0:2, 0:2, 2] = 90 + packed_yuv444[0:2, 2:4, 2] = 100 + packed_yuv444[2:4, 0:2, 2] = 110 + packed_yuv444[2:4, 2:4, 2] = 120 + + result = image_colorspace_conversion( + packed_yuv444.tobytes(), + (width, height), + ImageFormat.PACKED_YUV444, + ImageFormat.PLANAR_YUV420, + ) + + assert result is not None + + # Parse the planar YUV420 result + y_plane_size = width * height + uv_plane_size = (width // 2) * (height // 2) + + result_array = np.frombuffer(result, dtype=np.uint8) + y_result = result_array[:y_plane_size] + u_result = result_array[y_plane_size : y_plane_size + uv_plane_size] + v_result = result_array[y_plane_size + uv_plane_size :] + + # Check Y plane is preserved + expected_y = np.arange(100, 100 + width * height, dtype=np.uint8) + np.testing.assert_array_equal(y_result, expected_y) + + # Check U plane is downsampled (should be 2x2) + expected_u = np.array([50, 60, 70, 80], dtype=np.uint8) + np.testing.assert_array_equal(u_result, expected_u) + + # Check V plane is downsampled + expected_v = np.array([90, 100, 110, 120], dtype=np.uint8) + np.testing.assert_array_equal(v_result, expected_v) + + def test_unsupported_conversion_returns_none(self) -> None: + """Test that unsupported conversions return None.""" + width, height = 4, 4 + image_data = np.random.randint(0, 256, width * height * 3, dtype=np.uint8).tobytes() + + # Test some unsupported conversions + result = image_colorspace_conversion( + image_data, + (width, height), + ImageFormat.RGB, + ImageFormat.PLANAR_YUV420, + ) + assert result is None + + result = image_colorspace_conversion( + image_data, + (width, height), + ImageFormat.RGBA, + ImageFormat.RGB, + ) + assert result is None + + result = image_colorspace_conversion( + image_data, + (width, height), + ImageFormat.PLANAR_YUV420, + ImageFormat.BGR, + ) + assert result is None + + def test_conversion_with_different_sizes(self) -> None: + """Test conversions work with different image sizes.""" + test_sizes = [(2, 2), (4, 4), (8, 8), (16, 16)] + + for width, height in test_sizes: + # Test RGB to BGR + rgb_image = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) + result = image_colorspace_conversion( + rgb_image.tobytes(), + (width, height), + ImageFormat.RGB, + ImageFormat.BGR, + ) + assert result is not None + assert len(result) == width * height * 3 + + def test_yuv420_to_yuv444_roundtrip_preserves_y_plane(self) -> None: + """Test that Y plane is preserved in YUV420 -> YUV444 -> YUV420 conversion.""" + width, height = 4, 4 + + # Create original YUV420 data + y_plane_orig = np.arange(0, width * height, dtype=np.uint8) + u_plane_orig = np.array([50, 60, 70, 80], dtype=np.uint8) + v_plane_orig = np.array([90, 100, 110, 120], dtype=np.uint8) + yuv420_orig = np.concatenate([y_plane_orig, u_plane_orig, v_plane_orig]) + + # Convert to YUV444 + yuv444 = image_colorspace_conversion( + yuv420_orig.tobytes(), + (width, height), + ImageFormat.PLANAR_YUV420, + ImageFormat.PACKED_YUV444, + ) + assert yuv444 is not None + + # Convert back to YUV420 + yuv420_result = image_colorspace_conversion( + yuv444, + (width, height), + ImageFormat.PACKED_YUV444, + ImageFormat.PLANAR_YUV420, + ) + assert yuv420_result is not None + + # Extract Y plane from result + result_array = np.frombuffer(yuv420_result, dtype=np.uint8) + y_plane_result = result_array[: width * height] + + # Y plane should be identical after roundtrip + np.testing.assert_array_equal(y_plane_result, y_plane_orig) + + def test_rgb_bgr_roundtrip(self) -> None: + """Test that RGB -> BGR -> RGB conversion preserves data.""" + width, height = 4, 4 + rgb_orig = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) + + # Convert to BGR + bgr = image_colorspace_conversion( + rgb_orig.tobytes(), + (width, height), + ImageFormat.RGB, + ImageFormat.BGR, + ) + assert bgr is not None + + # Convert back to RGB + rgb_result = image_colorspace_conversion( + bgr, + (width, height), + ImageFormat.BGR, + ImageFormat.RGB, + ) + assert rgb_result is not None + + result_array = np.frombuffer(rgb_result, dtype=np.uint8).reshape(height, width, 3) + np.testing.assert_array_equal(result_array, rgb_orig) + + def test_rgba_bgra_roundtrip(self) -> None: + """Test that RGBA -> BGRA -> RGBA conversion preserves data.""" + width, height = 4, 4 + rgba_orig = np.random.randint(0, 256, (height, width, 4), dtype=np.uint8) + + # Convert to BGRA + bgra = image_colorspace_conversion( + rgba_orig.tobytes(), + (width, height), + ImageFormat.RGBA, + ImageFormat.BGRA, + ) + assert bgra is not None + + # Convert back to RGBA + rgba_result = image_colorspace_conversion( + bgra, + (width, height), + ImageFormat.BGRA, + ImageFormat.RGBA, + ) + assert rgba_result is not None + + result_array = np.frombuffer(rgba_result, dtype=np.uint8).reshape(height, width, 4) + np.testing.assert_array_equal(result_array, rgba_orig)