wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update more examples, misc services

Browse Source
This commit is contained in:
Mark Backman
2026-03-04 23:29:50 -05:00
parent 62554a2390
commit 8a203dd98f
36 changed files with 221 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/pipecat/services/azure/image.py
View File

@@ -12,7 +12,7 @@ using REST endpoints for creating images from text prompts.
import asyncio
import io
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import AsyncGenerator, Optional
import aiohttp
@@ -20,7 +20,7 @@ from PIL import Image
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings, _warn_deprecated_param
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, _warn_deprecated_param
@dataclass
@@ -29,8 +29,11 @@ class AzureImageGenSettings(ImageGenSettings):
Parameters:
model: Azure image generation model identifier.
image_size: Target size for generated images.
"""
image_size: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class AzureImageGenServiceREST(ImageGenService):
"""Azure OpenAI REST-based image generation service.
@@ -40,10 +43,12 @@ class AzureImageGenServiceREST(ImageGenService):
and automatic image download and processing.
"""
_settings: AzureImageGenSettings
def __init__(
self,
*,
image_size: str,
image_size: Optional[str] = None,
api_key: str,
endpoint: str,
model: Optional[str] = None,
@@ -55,6 +60,10 @@ class AzureImageGenServiceREST(ImageGenService):
Args:
image_size: Size specification for generated images (e.g., "1024x1024").
.. deprecated:: 0.0.105
Use ``settings=AzureImageGenSettings(image_size=...)`` instead.
api_key: Azure OpenAI API key for authentication.
endpoint: Azure OpenAI endpoint URL.
model: The image generation model to use.
@@ -67,10 +76,22 @@ class AzureImageGenServiceREST(ImageGenService):
settings: Runtime-updatable settings. When provided alongside deprecated
parameters, ``settings`` values take precedence.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = AzureImageGenSettings(
model=None,
image_size=None,
)
# 2. Apply direct init arg overrides (deprecated)
if model is not None:
_warn_deprecated_param("model", AzureImageGenSettings, "model")
default_settings.model = model
default_settings = AzureImageGenSettings(model=model)
if image_size is not None:
_warn_deprecated_param("image_size", AzureImageGenSettings, "image_size")
default_settings.image_size = image_size
# 4. Apply settings delta (canonical API, always wins)
if settings is not None:
default_settings.apply_update(settings)
@@ -79,7 +100,6 @@ class AzureImageGenServiceREST(ImageGenService):
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self._image_size = image_size
self._aiohttp_session = aiohttp_session
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
@@ -97,12 +117,13 @@ class AzureImageGenServiceREST(ImageGenService):
headers = {"api-key": self._api_key, "Content-Type": "application/json"}
body = {
# Enter your prompt text here
"prompt": prompt,
"size": self._image_size,
"n": 1,
}
if self._settings.image_size is not None:
body["size"] = self._settings.image_size
async with self._aiohttp_session.post(url, headers=headers, json=body) as submission:
# We never get past this line, because this header isn't
# defined on a 429 response, but something is eating our

70
src/pipecat/services/fal/image.py
View File

@@ -13,8 +13,8 @@ for creating images from text prompts using various AI models.
import asyncio
import io
import os
from dataclasses import dataclass
from typing import AsyncGenerator, Dict, Optional, Union
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Dict, Optional, Union
import aiohttp
from loguru import logger
@@ -23,7 +23,7 @@ from pydantic import BaseModel
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings, _warn_deprecated_param
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, _warn_deprecated_param
@dataclass
@@ -32,8 +32,36 @@ class FalImageGenSettings(ImageGenSettings):
Parameters:
model: Fal.ai model identifier.
seed: Random seed for reproducible generation. ``None`` uses a random seed.
num_inference_steps: Number of inference steps for generation.
num_images: Number of images to generate.
image_size: Image dimensions as a string preset or dict with width/height.
expand_prompt: Whether to automatically expand/enhance the prompt.
enable_safety_checker: Whether to enable content safety filtering.
format: Output image format.
"""
seed: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
num_inference_steps: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
num_images: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
image_size: str | Dict[str, int] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
expand_prompt: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
enable_safety_checker: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
format: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
def to_api_arguments(self) -> Dict[str, Any]:
"""Build the Fal API arguments dict from settings, excluding None values."""
args: Dict[str, Any] = {}
if self.seed is not None:
args["seed"] = self.seed
args["num_inference_steps"] = self.num_inference_steps
args["num_images"] = self.num_images
args["image_size"] = self.image_size
args["expand_prompt"] = self.expand_prompt
args["enable_safety_checker"] = self.enable_safety_checker
args["format"] = self.format
return args
class FalImageGenService(ImageGenService):
"""Fal's image generation service.
@@ -45,6 +73,9 @@ class FalImageGenService(ImageGenService):
class InputParams(BaseModel):
"""Input parameters for Fal.ai image generation.
.. deprecated:: 0.0.105
Use ``settings=FalImageGenSettings(...)`` instead.
Parameters:
seed: Random seed for reproducible generation. If None, uses random seed.
num_inference_steps: Number of inference steps for generation. Defaults to 8.
@@ -63,10 +94,12 @@ class FalImageGenService(ImageGenService):
enable_safety_checker: bool = True
format: str = "png"
_settings: FalImageGenSettings
def __init__(
self,
*,
params: InputParams,
params: Optional[InputParams] = None,
aiohttp_session: aiohttp.ClientSession,
model: Optional[str] = None,
key: Optional[str] = None,
@@ -77,6 +110,10 @@ class FalImageGenService(ImageGenService):
Args:
params: Input parameters for image generation configuration.
.. deprecated:: 0.0.105
Use ``settings=FalImageGenSettings(...)`` instead.
aiohttp_session: HTTP client session for downloading generated images.
model: The Fal.ai model to use for generation. Defaults to "fal-ai/fast-sdxl".
@@ -89,19 +126,38 @@ class FalImageGenService(ImageGenService):
**kwargs: Additional arguments passed to parent ImageGenService.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = FalImageGenSettings(model="fal-ai/fast-sdxl")
default_settings = FalImageGenSettings(
model="fal-ai/fast-sdxl",
seed=None,
num_inference_steps=8,
num_images=1,
image_size="square_hd",
expand_prompt=False,
enable_safety_checker=True,
format="png",
)
# 2. Apply direct init arg overrides (deprecated)
if model is not None:
_warn_deprecated_param("model", FalImageGenSettings, "model")
default_settings.model = model
if params is not None:
_warn_deprecated_param("params", FalImageGenSettings)
if not settings:
default_settings.seed = params.seed
default_settings.num_inference_steps = params.num_inference_steps
default_settings.num_images = params.num_images
default_settings.image_size = params.image_size
default_settings.expand_prompt = params.expand_prompt
default_settings.enable_safety_checker = params.enable_safety_checker
default_settings.format = params.format
# 4. Apply settings delta (canonical API, always wins)
if settings is not None:
default_settings.apply_update(settings)
super().__init__(settings=default_settings, **kwargs)
self._params = params
self._aiohttp_session = aiohttp_session
self._api_key = key or os.getenv("FAL_KEY", "")
if key:
@@ -129,7 +185,7 @@ class FalImageGenService(ImageGenService):
"Authorization": f"Key {self._api_key}",
"Content-Type": "application/json",
}
payload = {"prompt": prompt, **self._params.model_dump(exclude_none=True)}
payload = {"prompt": prompt, **self._settings.to_api_arguments()}
async with self._aiohttp_session.post(
f"https://fal.run/{self._settings.model}",

33
src/pipecat/services/google/image.py
View File

@@ -16,7 +16,7 @@ import os
# Suppress gRPC fork warnings
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Any, AsyncGenerator, Optional
from loguru import logger
@@ -26,7 +26,7 @@ from pydantic import BaseModel, Field
from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
from pipecat.services.google.utils import update_google_client_http_options
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings, _warn_deprecated_param
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, _warn_deprecated_param
try:
from google import genai
@@ -43,8 +43,13 @@ class GoogleImageGenSettings(ImageGenSettings):
Parameters:
model: Google Imagen model identifier.
number_of_images: Number of images to generate per request.
negative_prompt: Text describing what not to include in generated images.
"""
number_of_images: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
negative_prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class GoogleImageGenService(ImageGenService):
"""Google AI image generation service using Imagen models.
@@ -57,6 +62,9 @@ class GoogleImageGenService(ImageGenService):
class InputParams(BaseModel):
"""Configuration parameters for Google image generation.
.. deprecated:: 0.0.105
Use ``settings=GoogleImageGenSettings(...)`` instead.
Parameters:
number_of_images: Number of images to generate (1-8). Defaults to 1.
model: Google Imagen model to use. Defaults to "imagen-3.0-generate-002".
@@ -67,6 +75,8 @@ class GoogleImageGenService(ImageGenService):
model: str = Field(default="imagen-3.0-generate-002")
negative_prompt: Optional[str] = Field(default=None)
_settings: GoogleImageGenSettings
def __init__(
self,
*,
@@ -83,7 +93,7 @@ class GoogleImageGenService(ImageGenService):
params: Configuration parameters for image generation.
.. deprecated:: 0.0.105
Use ``settings=GoogleImageGenSettings(model=...)`` instead.
Use ``settings=GoogleImageGenSettings(...)`` instead.
http_options: HTTP options for the client.
settings: Runtime-updatable settings. When provided alongside deprecated
@@ -91,20 +101,25 @@ class GoogleImageGenService(ImageGenService):
**kwargs: Additional arguments passed to the parent ImageGenService.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = GoogleImageGenSettings(model="imagen-3.0-generate-002")
default_settings = GoogleImageGenSettings(
model="imagen-3.0-generate-002",
number_of_images=1,
negative_prompt=None,
)
# 3. Apply params overrides — only if settings not provided
# 2. Apply params overrides (deprecated)
if params is not None:
_warn_deprecated_param("params", GoogleImageGenSettings)
if not settings:
default_settings.model = params.model
default_settings.number_of_images = params.number_of_images
default_settings.negative_prompt = params.negative_prompt
# 4. Apply settings delta (canonical API, always wins)
if settings is not None:
default_settings.apply_update(settings)
super().__init__(settings=default_settings, **kwargs)
self._params = params or GoogleImageGenService.InputParams()
# Add client header
http_options = update_google_client_http_options(http_options)
@@ -137,11 +152,11 @@ class GoogleImageGenService(ImageGenService):
try:
response = await self._client.aio.models.generate_images(
model=self._params.model,
model=self._settings.model,
prompt=prompt,
config=types.GenerateImagesConfig(
number_of_images=self._params.number_of_images,
negative_prompt=self._params.negative_prompt,
number_of_images=self._settings.number_of_images,
negative_prompt=self._settings.negative_prompt,
),
)
await self.stop_ttfb_metrics()

10
src/pipecat/services/heygen/video.py
View File

@@ -12,6 +12,7 @@ audio/video streaming capabilities through the HeyGen API.
"""
import asyncio
from dataclasses import dataclass
from typing import Optional, Union
import aiohttp
@@ -52,6 +53,13 @@ from pipecat.transports.base_transport import TransportParams
AVATAR_VAD_STOP_SECS = 0.35
@dataclass
class HeyGenVideoSettings(ServiceSettings):
"""Settings for the HeyGen video service."""
pass
class HeyGenVideoService(AIService):
"""A service that integrates HeyGen's interactive avatar capabilities into the pipeline.
@@ -81,7 +89,7 @@ class HeyGenVideoService(AIService):
session: aiohttp.ClientSession,
session_request: Optional[Union[LiveAvatarNewSessionRequest, NewSessionRequest]] = None,
service_type: Optional[ServiceType] = None,
settings: Optional[ServiceSettings] = None,
settings: Optional[HeyGenVideoSettings] = None,
**kwargs,
) -> None:
"""Initialize the HeyGen video service.

31
src/pipecat/services/openai/image.py
View File

@@ -11,7 +11,7 @@ for creating images from text prompts.
"""
import io
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import AsyncGenerator, Literal, Optional
import aiohttp
@@ -25,7 +25,7 @@ from pipecat.frames.frames import (
URLImageRawFrame,
)
from pipecat.services.image_service import ImageGenService
from pipecat.services.settings import ImageGenSettings, _warn_deprecated_param
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, _warn_deprecated_param
@dataclass
@@ -34,8 +34,11 @@ class OpenAIImageGenSettings(ImageGenSettings):
Parameters:
model: DALL-E model identifier.
image_size: Target size for generated images.
"""
image_size: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
class OpenAIImageGenService(ImageGenService):
"""OpenAI DALL-E image generation service.
@@ -45,13 +48,17 @@ class OpenAIImageGenService(ImageGenService):
with configurable quality and style parameters.
"""
_settings: OpenAIImageGenSettings
def __init__(
self,
*,
api_key: str,
base_url: Optional[str] = None,
aiohttp_session: aiohttp.ClientSession,
image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"],
image_size: Optional[
Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"]
] = None,
model: Optional[str] = None,
settings: Optional[OpenAIImageGenSettings] = None,
):
@@ -61,7 +68,11 @@ class OpenAIImageGenService(ImageGenService):
api_key: OpenAI API key for authentication.
base_url: Custom base URL for OpenAI API. If None, uses default.
aiohttp_session: HTTP session for downloading generated images.
image_size: Target size for generated images.
image_size: Target size for generated images. Defaults to "1024x1024".
.. deprecated:: 0.0.105
Use ``settings=OpenAIImageGenSettings(image_size=...)`` instead.
model: DALL-E model to use for generation. Defaults to "dall-e-3".
.. deprecated:: 0.0.105
@@ -71,19 +82,25 @@ class OpenAIImageGenService(ImageGenService):
parameters, ``settings`` values take precedence.
"""
# 1. Initialize default_settings with hardcoded defaults
default_settings = OpenAIImageGenSettings(model="dall-e-3")
default_settings = OpenAIImageGenSettings(
model="dall-e-3",
image_size=None,
)
# 2. Apply direct init arg overrides (deprecated)
if model is not None:
_warn_deprecated_param("model", OpenAIImageGenSettings, "model")
default_settings.model = model
if image_size is not None:
_warn_deprecated_param("image_size", OpenAIImageGenSettings, "image_size")
default_settings.image_size = image_size
# 4. Apply settings delta (canonical API, always wins)
if settings is not None:
default_settings.apply_update(settings)
super().__init__(settings=default_settings)
self._image_size = image_size
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
self._aiohttp_session = aiohttp_session
@@ -99,7 +116,7 @@ class OpenAIImageGenService(ImageGenService):
logger.debug(f"Generating image from prompt: {prompt}")
image = await self._client.images.generate(
prompt=prompt, model=self._settings.model, n=1, size=self._image_size
prompt=prompt, model=self._settings.model, n=1, size=self._settings.image_size
)
image_url = image.data[0].url

10
src/pipecat/services/tavus/video.py
View File

@@ -11,6 +11,7 @@ avatar functionality through Tavus's streaming API.
"""
import asyncio
from dataclasses import dataclass
from typing import Optional
import aiohttp
@@ -38,6 +39,13 @@ from pipecat.services.settings import ServiceSettings
from pipecat.transports.tavus.transport import TavusCallbacks, TavusParams, TavusTransportClient
@dataclass
class TavusVideoSettings(ServiceSettings):
"""Settings for the Tavus video service."""
pass
class TavusVideoService(AIService):
"""Service that proxies audio to Tavus and receives audio and video in return.
@@ -58,7 +66,7 @@ class TavusVideoService(AIService):
replica_id: str,
persona_id: str = "pipecat-stream",
session: aiohttp.ClientSession,
settings: Optional[ServiceSettings] = None,
settings: Optional[TavusVideoSettings] = None,
**kwargs,
) -> None:
"""Initialize the Tavus video service.