Migrate SimliVideoService to AIService with Settings pattern
Align Simli with HeyGen/Tavus by extending AIService instead of FrameProcessor and using a ServiceSettings dataclass. InputParams is preserved but deprecated; its fields are promoted to direct init params. Lifecycle handling moves to start()/stop()/cancel() methods.
This commit is contained in:
@@ -8,6 +8,7 @@
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import warnings
|
import warnings
|
||||||
|
from dataclasses import dataclass
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -20,11 +21,14 @@ from pipecat.frames.frames import (
|
|||||||
Frame,
|
Frame,
|
||||||
InterruptionFrame,
|
InterruptionFrame,
|
||||||
OutputImageRawFrame,
|
OutputImageRawFrame,
|
||||||
|
StartFrame,
|
||||||
TTSAudioRawFrame,
|
TTSAudioRawFrame,
|
||||||
TTSStoppedFrame,
|
TTSStoppedFrame,
|
||||||
UserStartedSpeakingFrame,
|
UserStartedSpeakingFrame,
|
||||||
)
|
)
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame
|
from pipecat.processors.frame_processor import FrameDirection
|
||||||
|
from pipecat.services.ai_service import AIService
|
||||||
|
from pipecat.services.settings import ServiceSettings
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from av.audio.frame import AudioFrame
|
from av.audio.frame import AudioFrame
|
||||||
@@ -36,7 +40,14 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
class SimliVideoService(FrameProcessor):
|
@dataclass
|
||||||
|
class SimliVideoSettings(ServiceSettings):
|
||||||
|
"""Settings for the Simli video service."""
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class SimliVideoService(AIService):
|
||||||
"""Simli video service for real-time avatar generation.
|
"""Simli video service for real-time avatar generation.
|
||||||
|
|
||||||
Provides real-time avatar video generation by processing audio frames
|
Provides real-time avatar video generation by processing audio frames
|
||||||
@@ -44,9 +55,15 @@ class SimliVideoService(FrameProcessor):
|
|||||||
audio resampling, video frame processing, and connection management.
|
audio resampling, video frame processing, and connection management.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
Settings = SimliVideoSettings
|
||||||
|
_settings: Settings
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
"""Input parameters for Simli video configuration.
|
"""Input parameters for Simli video configuration.
|
||||||
|
|
||||||
|
.. deprecated:: 0.0.105
|
||||||
|
Use ``SimliVideoService.Settings(...)`` instead.
|
||||||
|
|
||||||
Parameters:
|
Parameters:
|
||||||
enable_logging: Whether to enable Simli logging.
|
enable_logging: Whether to enable Simli logging.
|
||||||
max_session_length: Absolute maximum session duration in seconds.
|
max_session_length: Absolute maximum session duration in seconds.
|
||||||
@@ -66,10 +83,13 @@ class SimliVideoService(FrameProcessor):
|
|||||||
face_id: Optional[str] = None,
|
face_id: Optional[str] = None,
|
||||||
simli_config: Optional[SimliConfig] = None,
|
simli_config: Optional[SimliConfig] = None,
|
||||||
use_turn_server: bool = False,
|
use_turn_server: bool = False,
|
||||||
latency_interval: int = 0,
|
|
||||||
simli_url: str = "https://api.simli.ai",
|
simli_url: str = "https://api.simli.ai",
|
||||||
is_trinity_avatar: bool = False,
|
is_trinity_avatar: bool = False,
|
||||||
params: Optional[InputParams] = None,
|
params: Optional[InputParams] = None,
|
||||||
|
max_session_length: Optional[int] = None,
|
||||||
|
max_idle_time: Optional[int] = None,
|
||||||
|
enable_logging: Optional[bool] = None,
|
||||||
|
settings: Optional[Settings] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
"""Initialize the Simli video service.
|
"""Initialize the Simli video service.
|
||||||
@@ -90,18 +110,42 @@ class SimliVideoService(FrameProcessor):
|
|||||||
.. deprecated:: 0.0.95
|
.. deprecated:: 0.0.95
|
||||||
The 'use_turn_server' parameter is deprecated and will be removed in a future version.
|
The 'use_turn_server' parameter is deprecated and will be removed in a future version.
|
||||||
|
|
||||||
latency_interval: Latency interval setting for sending health checks to check
|
|
||||||
the latency to Simli Servers. Defaults to 0.
|
|
||||||
simli_url: URL of the simli servers. Can be changed for custom deployments
|
simli_url: URL of the simli servers. Can be changed for custom deployments
|
||||||
of enterprise users.
|
of enterprise users.
|
||||||
is_trinity_avatar: Boolean to tell simli client that this is a Trinity avatar
|
is_trinity_avatar: Boolean to tell simli client that this is a Trinity avatar
|
||||||
which reduces latency when using Trinity.
|
which reduces latency when using Trinity.
|
||||||
params: Additional input parameters for session configuration.
|
params: Additional input parameters for session configuration.
|
||||||
**kwargs: Additional arguments passed to the parent FrameProcessor.
|
|
||||||
"""
|
|
||||||
super().__init__(**kwargs)
|
|
||||||
|
|
||||||
params = params or SimliVideoService.InputParams()
|
.. deprecated:: 0.0.105
|
||||||
|
Use ``settings=SimliVideoService.Settings(...)`` instead.
|
||||||
|
|
||||||
|
max_session_length: Absolute maximum session duration in seconds.
|
||||||
|
Avatar will disconnect after this time even if it's speaking.
|
||||||
|
max_idle_time: Maximum duration in seconds the avatar is not speaking
|
||||||
|
before the avatar disconnects.
|
||||||
|
enable_logging: Whether to enable Simli logging.
|
||||||
|
settings: Service settings.
|
||||||
|
**kwargs: Additional arguments passed to the parent AIService.
|
||||||
|
"""
|
||||||
|
# 1. Default settings
|
||||||
|
default_settings = ServiceSettings(model=None)
|
||||||
|
|
||||||
|
# 2. Apply deprecated params overrides
|
||||||
|
if params is not None:
|
||||||
|
self._warn_init_param_moved_to_settings("params")
|
||||||
|
if max_session_length is None and hasattr(params, "max_session_length"):
|
||||||
|
max_session_length = params.max_session_length
|
||||||
|
if max_idle_time is None and hasattr(params, "max_idle_time"):
|
||||||
|
max_idle_time = params.max_idle_time
|
||||||
|
if enable_logging is None and hasattr(params, "enable_logging"):
|
||||||
|
enable_logging = params.enable_logging
|
||||||
|
|
||||||
|
# 3. Apply settings delta
|
||||||
|
if settings is not None:
|
||||||
|
default_settings.apply_update(settings)
|
||||||
|
|
||||||
|
# 4. Call super
|
||||||
|
super().__init__(settings=default_settings, **kwargs)
|
||||||
|
|
||||||
# Handle deprecated simli_config parameter
|
# Handle deprecated simli_config parameter
|
||||||
if simli_config is not None:
|
if simli_config is not None:
|
||||||
@@ -133,10 +177,10 @@ class SimliVideoService(FrameProcessor):
|
|||||||
config_kwargs = {
|
config_kwargs = {
|
||||||
"faceId": face_id,
|
"faceId": face_id,
|
||||||
}
|
}
|
||||||
if params.max_session_length is not None:
|
if max_session_length is not None:
|
||||||
config_kwargs["maxSessionLength"] = params.max_session_length
|
config_kwargs["maxSessionLength"] = max_session_length
|
||||||
if params.max_idle_time is not None:
|
if max_idle_time is not None:
|
||||||
config_kwargs["maxIdleTime"] = params.max_idle_time
|
config_kwargs["maxIdleTime"] = max_idle_time
|
||||||
|
|
||||||
config = SimliConfig(**config_kwargs)
|
config = SimliConfig(**config_kwargs)
|
||||||
|
|
||||||
@@ -168,6 +212,33 @@ class SimliVideoService(FrameProcessor):
|
|||||||
self._previously_interrupted = is_trinity_avatar
|
self._previously_interrupted = is_trinity_avatar
|
||||||
self._audio_buffer = bytearray()
|
self._audio_buffer = bytearray()
|
||||||
|
|
||||||
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the Simli video service.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
|
await super().start(frame)
|
||||||
|
await self._start_connection()
|
||||||
|
|
||||||
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the Simli video service.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame.
|
||||||
|
"""
|
||||||
|
await super().stop(frame)
|
||||||
|
await self._stop_connection()
|
||||||
|
|
||||||
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the Simli video service.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame.
|
||||||
|
"""
|
||||||
|
await super().cancel(frame)
|
||||||
|
await self._stop_connection()
|
||||||
|
|
||||||
async def _start_connection(self):
|
async def _start_connection(self):
|
||||||
"""Start the connection to Simli service and begin processing tasks."""
|
"""Start the connection to Simli service and begin processing tasks."""
|
||||||
try:
|
try:
|
||||||
@@ -222,9 +293,7 @@ class SimliVideoService(FrameProcessor):
|
|||||||
direction: The direction of frame processing.
|
direction: The direction of frame processing.
|
||||||
"""
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
if isinstance(frame, StartFrame):
|
if isinstance(frame, TTSAudioRawFrame):
|
||||||
await self._start_connection()
|
|
||||||
elif isinstance(frame, TTSAudioRawFrame):
|
|
||||||
# Send audio frame to Simli
|
# Send audio frame to Simli
|
||||||
try:
|
try:
|
||||||
old_frame = AudioFrame.from_ndarray(
|
old_frame = AudioFrame.from_ndarray(
|
||||||
@@ -268,8 +337,6 @@ class SimliVideoService(FrameProcessor):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
await self.push_error(error_msg=f"Error stopping TTS: {e}", exception=e)
|
await self.push_error(error_msg=f"Error stopping TTS: {e}", exception=e)
|
||||||
return
|
return
|
||||||
elif isinstance(frame, (EndFrame, CancelFrame)):
|
|
||||||
await self._stop()
|
|
||||||
elif isinstance(frame, (InterruptionFrame, UserStartedSpeakingFrame)):
|
elif isinstance(frame, (InterruptionFrame, UserStartedSpeakingFrame)):
|
||||||
if not self._previously_interrupted:
|
if not self._previously_interrupted:
|
||||||
await self._simli_client.clearBuffer()
|
await self._simli_client.clearBuffer()
|
||||||
@@ -277,7 +344,7 @@ class SimliVideoService(FrameProcessor):
|
|||||||
|
|
||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
async def _stop(self):
|
async def _stop_connection(self):
|
||||||
"""Stop the Simli client and cancel processing tasks."""
|
"""Stop the Simli client and cancel processing tasks."""
|
||||||
await self._simli_client.stop()
|
await self._simli_client.stop()
|
||||||
if self._audio_task:
|
if self._audio_task:
|
||||||
|
|||||||
Reference in New Issue
Block a user