Merge pull request #4378 from pipecat-ai/pk/more-pyright-fixes
More pyright fixes
This commit is contained in:
4
.github/workflows/format.yaml
vendored
4
.github/workflows/format.yaml
vendored
@@ -32,7 +32,9 @@ jobs:
|
|||||||
run: uv python install 3.12
|
run: uv python install 3.12
|
||||||
|
|
||||||
- name: Install development dependencies
|
- name: Install development dependencies
|
||||||
run: uv sync --group dev --extra daily --extra tracing
|
# `--all-extras` (matching the dev setup in README.md) so pyright can
|
||||||
|
# resolve types from various optional dependencies.
|
||||||
|
run: uv sync --group dev --all-extras --no-extra gstreamer --no-extra local
|
||||||
|
|
||||||
- name: Ruff formatter
|
- name: Ruff formatter
|
||||||
id: ruff-format
|
id: ruff-format
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ Pipecat is an open-source Python framework for building real-time voice and mult
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Setup development environment
|
# Setup development environment
|
||||||
uv sync --group dev --all-extras --no-extra gstreamer
|
uv sync --group dev --all-extras --no-extra gstreamer --no-extra local
|
||||||
|
|
||||||
# Install pre-commit hooks
|
# Install pre-commit hooks
|
||||||
uv run pre-commit install
|
uv run pre-commit install
|
||||||
|
|||||||
@@ -6,115 +6,54 @@
|
|||||||
"exclude": ["**/*_pb2.py", "**/__pycache__"],
|
"exclude": ["**/*_pb2.py", "**/__pycache__"],
|
||||||
"ignore": [
|
"ignore": [
|
||||||
"tests",
|
"tests",
|
||||||
"src/pipecat/adapters/services/anthropic_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/aws_nova_sonic_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/bedrock_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/gemini_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/grok_realtime_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/inworld_realtime_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/open_ai_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/open_ai_realtime_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/open_ai_responses_adapter.py",
|
|
||||||
"src/pipecat/adapters/services/perplexity_adapter.py",
|
|
||||||
"src/pipecat/audio/dtmf/utils.py",
|
|
||||||
"src/pipecat/audio/filters/aic_filter.py",
|
"src/pipecat/audio/filters/aic_filter.py",
|
||||||
"src/pipecat/audio/filters/krisp_viva_filter.py",
|
"src/pipecat/audio/filters/krisp_viva_filter.py",
|
||||||
"src/pipecat/audio/filters/rnnoise_filter.py",
|
|
||||||
"src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py",
|
"src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py",
|
||||||
"src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py",
|
"src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py",
|
||||||
"src/pipecat/audio/vad/silero.py",
|
"src/pipecat/audio/vad/silero.py",
|
||||||
"src/pipecat/processors/aggregators/llm_context.py",
|
|
||||||
"src/pipecat/processors/aggregators/llm_response_universal.py",
|
"src/pipecat/processors/aggregators/llm_response_universal.py",
|
||||||
"src/pipecat/processors/frame_processor.py",
|
"src/pipecat/processors/frame_processor.py",
|
||||||
"src/pipecat/processors/frameworks/langchain.py",
|
|
||||||
"src/pipecat/processors/frameworks/rtvi/observer.py",
|
"src/pipecat/processors/frameworks/rtvi/observer.py",
|
||||||
"src/pipecat/processors/frameworks/rtvi/processor.py",
|
|
||||||
"src/pipecat/processors/frameworks/strands_agents.py",
|
|
||||||
"src/pipecat/services/anthropic/llm.py",
|
"src/pipecat/services/anthropic/llm.py",
|
||||||
"src/pipecat/services/assemblyai/stt.py",
|
|
||||||
"src/pipecat/services/aws/agent_core.py",
|
|
||||||
"src/pipecat/services/aws/llm.py",
|
"src/pipecat/services/aws/llm.py",
|
||||||
"src/pipecat/services/aws/nova_sonic/llm.py",
|
"src/pipecat/services/aws/nova_sonic/llm.py",
|
||||||
"src/pipecat/services/aws/sagemaker/bidi_client.py",
|
"src/pipecat/services/aws/sagemaker/bidi_client.py",
|
||||||
"src/pipecat/services/aws/stt.py",
|
|
||||||
"src/pipecat/services/aws/tts.py",
|
|
||||||
"src/pipecat/services/aws/utils.py",
|
|
||||||
"src/pipecat/services/azure/stt.py",
|
|
||||||
"src/pipecat/services/azure/tts.py",
|
"src/pipecat/services/azure/tts.py",
|
||||||
"src/pipecat/services/cartesia/stt.py",
|
|
||||||
"src/pipecat/services/deepgram/flux/base.py",
|
|
||||||
"src/pipecat/services/deepgram/flux/sagemaker/stt.py",
|
"src/pipecat/services/deepgram/flux/sagemaker/stt.py",
|
||||||
"src/pipecat/services/deepgram/flux/stt.py",
|
|
||||||
"src/pipecat/services/deepgram/sagemaker/stt.py",
|
"src/pipecat/services/deepgram/sagemaker/stt.py",
|
||||||
"src/pipecat/services/deepgram/sagemaker/tts.py",
|
"src/pipecat/services/deepgram/sagemaker/tts.py",
|
||||||
"src/pipecat/services/deepgram/tts.py",
|
|
||||||
"src/pipecat/services/elevenlabs/stt.py",
|
|
||||||
"src/pipecat/services/elevenlabs/tts.py",
|
|
||||||
"src/pipecat/services/fish/tts.py",
|
|
||||||
"src/pipecat/services/gladia/stt.py",
|
|
||||||
"src/pipecat/services/google/gemini_live/llm.py",
|
"src/pipecat/services/google/gemini_live/llm.py",
|
||||||
"src/pipecat/services/google/gemini_live/vertex/llm.py",
|
|
||||||
"src/pipecat/services/google/image.py",
|
|
||||||
"src/pipecat/services/google/llm.py",
|
"src/pipecat/services/google/llm.py",
|
||||||
"src/pipecat/services/google/stt.py",
|
"src/pipecat/services/google/stt.py",
|
||||||
"src/pipecat/services/google/tts.py",
|
"src/pipecat/services/google/tts.py",
|
||||||
"src/pipecat/services/gradium/stt.py",
|
|
||||||
"src/pipecat/services/groq/tts.py",
|
|
||||||
"src/pipecat/services/heygen/api_interactive_avatar.py",
|
|
||||||
"src/pipecat/services/heygen/base_api.py",
|
|
||||||
"src/pipecat/services/heygen/client.py",
|
"src/pipecat/services/heygen/client.py",
|
||||||
"src/pipecat/services/heygen/video.py",
|
"src/pipecat/services/heygen/video.py",
|
||||||
"src/pipecat/services/hume/tts.py",
|
|
||||||
"src/pipecat/services/inworld/realtime/llm.py",
|
"src/pipecat/services/inworld/realtime/llm.py",
|
||||||
"src/pipecat/services/inworld/tts.py",
|
|
||||||
"src/pipecat/services/kokoro/tts.py",
|
|
||||||
"src/pipecat/services/llm_service.py",
|
"src/pipecat/services/llm_service.py",
|
||||||
"src/pipecat/services/lmnt/tts.py",
|
|
||||||
"src/pipecat/services/mem0/memory.py",
|
"src/pipecat/services/mem0/memory.py",
|
||||||
"src/pipecat/services/mistral/stt.py",
|
|
||||||
"src/pipecat/services/mistral/tts.py",
|
"src/pipecat/services/mistral/tts.py",
|
||||||
"src/pipecat/services/moondream/vision.py",
|
|
||||||
"src/pipecat/services/neuphonic/tts.py",
|
|
||||||
"src/pipecat/services/nvidia/stt.py",
|
"src/pipecat/services/nvidia/stt.py",
|
||||||
"src/pipecat/services/nvidia/tts.py",
|
"src/pipecat/services/nvidia/tts.py",
|
||||||
"src/pipecat/services/openai/base_llm.py",
|
"src/pipecat/services/openai/base_llm.py",
|
||||||
"src/pipecat/services/openai/image.py",
|
|
||||||
"src/pipecat/services/openai/llm.py",
|
|
||||||
"src/pipecat/services/openai/realtime/llm.py",
|
"src/pipecat/services/openai/realtime/llm.py",
|
||||||
"src/pipecat/services/openai/responses/llm.py",
|
|
||||||
"src/pipecat/services/openai/stt.py",
|
|
||||||
"src/pipecat/services/openai/tts.py",
|
|
||||||
"src/pipecat/services/openrouter/llm.py",
|
|
||||||
"src/pipecat/services/piper/tts.py",
|
|
||||||
"src/pipecat/services/resembleai/tts.py",
|
|
||||||
"src/pipecat/services/rime/tts.py",
|
"src/pipecat/services/rime/tts.py",
|
||||||
"src/pipecat/services/sambanova/llm.py",
|
"src/pipecat/services/sambanova/llm.py",
|
||||||
"src/pipecat/services/sarvam/stt.py",
|
"src/pipecat/services/sarvam/stt.py",
|
||||||
"src/pipecat/services/sarvam/tts.py",
|
|
||||||
"src/pipecat/services/simli/video.py",
|
"src/pipecat/services/simli/video.py",
|
||||||
"src/pipecat/services/smallest/tts.py",
|
|
||||||
"src/pipecat/services/soniox/stt.py",
|
|
||||||
"src/pipecat/services/speechmatics/stt.py",
|
"src/pipecat/services/speechmatics/stt.py",
|
||||||
"src/pipecat/services/stt_service.py",
|
"src/pipecat/services/stt_service.py",
|
||||||
"src/pipecat/services/tavus/video.py",
|
"src/pipecat/services/tavus/video.py",
|
||||||
"src/pipecat/services/tts_service.py",
|
"src/pipecat/services/tts_service.py",
|
||||||
"src/pipecat/services/ultravox/llm.py",
|
"src/pipecat/services/ultravox/llm.py",
|
||||||
"src/pipecat/services/websocket_service.py",
|
|
||||||
"src/pipecat/services/whisper/stt.py",
|
"src/pipecat/services/whisper/stt.py",
|
||||||
"src/pipecat/services/xai/realtime/llm.py",
|
"src/pipecat/services/xai/realtime/llm.py",
|
||||||
"src/pipecat/services/xtts/tts.py",
|
|
||||||
"src/pipecat/transports/base_output.py",
|
|
||||||
"src/pipecat/transports/daily/transport.py",
|
"src/pipecat/transports/daily/transport.py",
|
||||||
"src/pipecat/transports/heygen/transport.py",
|
|
||||||
"src/pipecat/transports/lemonslice/transport.py",
|
"src/pipecat/transports/lemonslice/transport.py",
|
||||||
"src/pipecat/transports/livekit/transport.py",
|
"src/pipecat/transports/livekit/transport.py",
|
||||||
"src/pipecat/transports/smallwebrtc/connection.py",
|
"src/pipecat/transports/smallwebrtc/connection.py",
|
||||||
"src/pipecat/transports/smallwebrtc/request_handler.py",
|
|
||||||
"src/pipecat/transports/smallwebrtc/transport.py",
|
"src/pipecat/transports/smallwebrtc/transport.py",
|
||||||
"src/pipecat/transports/tavus/transport.py",
|
"src/pipecat/transports/tavus/transport.py",
|
||||||
"src/pipecat/transports/websocket/client.py",
|
"src/pipecat/transports/websocket/server.py"
|
||||||
"src/pipecat/transports/websocket/server.py",
|
|
||||||
"src/pipecat/transports/whatsapp/client.py"
|
|
||||||
],
|
],
|
||||||
"reportMissingImports": false
|
"reportMissingImports": false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, TypedDict, TypeGuard, TypeVar
|
from typing import Any, TypedDict, TypeGuard, TypeVar, cast
|
||||||
|
|
||||||
from anthropic import NOT_GIVEN, NotGiven
|
from anthropic import NOT_GIVEN, NotGiven
|
||||||
from anthropic.types.message_param import MessageParam
|
from anthropic.types.message_param import MessageParam
|
||||||
@@ -121,16 +121,20 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
messages = self._from_universal_context_messages(self.get_messages(context)).messages
|
messages = self._from_universal_context_messages(self.get_messages(context)).messages
|
||||||
|
|
||||||
# Sanitize messages for logging
|
# Sanitize messages for logging
|
||||||
messages_for_logging = []
|
messages_for_logging: list[dict[str, Any]] = []
|
||||||
for message in messages:
|
for message in messages:
|
||||||
msg = copy.deepcopy(message)
|
msg: dict[str, Any] = copy.deepcopy(dict(message))
|
||||||
if "content" in msg:
|
content = msg.get("content")
|
||||||
if isinstance(msg["content"], list):
|
if isinstance(content, list):
|
||||||
for item in msg["content"]:
|
for item in content:
|
||||||
if item["type"] == "image":
|
if not isinstance(item, dict):
|
||||||
item["source"]["data"] = "..."
|
continue
|
||||||
if item["type"] == "thinking" and item.get("signature"):
|
if item.get("type") == "image":
|
||||||
item["signature"] = "..."
|
source = item.get("source")
|
||||||
|
if isinstance(source, dict):
|
||||||
|
source["data"] = "..."
|
||||||
|
if item.get("type") == "thinking" and item.get("signature"):
|
||||||
|
item["signature"] = "..."
|
||||||
messages_for_logging.append(msg)
|
messages_for_logging.append(msg)
|
||||||
return messages_for_logging
|
return messages_for_logging
|
||||||
|
|
||||||
@@ -185,8 +189,13 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
]
|
]
|
||||||
if isinstance(next_message["content"], str):
|
if isinstance(next_message["content"], str):
|
||||||
next_message["content"] = [{"type": "text", "text": next_message["content"]}]
|
next_message["content"] = [{"type": "text", "text": next_message["content"]}]
|
||||||
# Concatenate the content
|
# Concatenate the content. MessageParam types content as
|
||||||
current_message["content"].extend(next_message["content"])
|
# `str | Iterable[...]`, but this codebase assumes it's
|
||||||
|
# either a str or a list. The str case is handled above, so
|
||||||
|
# we assume that both are lists here.
|
||||||
|
cast(list[Any], current_message["content"]).extend(
|
||||||
|
cast(list[Any], next_message["content"])
|
||||||
|
)
|
||||||
# Remove the next message from the list
|
# Remove the next message from the list
|
||||||
messages.pop(i + 1)
|
messages.pop(i + 1)
|
||||||
else:
|
else:
|
||||||
@@ -239,7 +248,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Fall back to assuming that the message is already in Anthropic format
|
# Fall back to assuming that the message is already in Anthropic format
|
||||||
return copy.deepcopy(message.message)
|
return cast(MessageParam, copy.deepcopy(message.message))
|
||||||
|
|
||||||
def _from_standard_message(self, message: LLMStandardMessage) -> MessageParam:
|
def _from_standard_message(self, message: LLMStandardMessage) -> MessageParam:
|
||||||
"""Convert standard universal context message to Anthropic format.
|
"""Convert standard universal context message to Anthropic format.
|
||||||
@@ -280,20 +289,26 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
message = copy.deepcopy(message)
|
# ChatCompletionMessageParam (input) and MessageParam (output) are
|
||||||
if message["role"] == "tool":
|
# different TypedDicts — work with the message as a plain dict for the
|
||||||
return {
|
# transformations below and cast back to MessageParam at return sites.
|
||||||
"role": "user",
|
msg = cast(dict[str, Any], copy.deepcopy(message))
|
||||||
"content": [
|
if msg["role"] == "tool":
|
||||||
{
|
return cast(
|
||||||
"type": "tool_result",
|
MessageParam,
|
||||||
"tool_use_id": message["tool_call_id"],
|
{
|
||||||
"content": message["content"],
|
"role": "user",
|
||||||
},
|
"content": [
|
||||||
],
|
{
|
||||||
}
|
"type": "tool_result",
|
||||||
if message.get("tool_calls"):
|
"tool_use_id": msg["tool_call_id"],
|
||||||
tc = message["tool_calls"]
|
"content": msg["content"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if msg.get("tool_calls"):
|
||||||
|
tc = msg["tool_calls"]
|
||||||
ret = {"role": "assistant", "content": []}
|
ret = {"role": "assistant", "content": []}
|
||||||
for tool_call in tc:
|
for tool_call in tc:
|
||||||
function = tool_call["function"]
|
function = tool_call["function"]
|
||||||
@@ -305,8 +320,8 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
"input": arguments,
|
"input": arguments,
|
||||||
}
|
}
|
||||||
ret["content"].append(new_tool_use)
|
ret["content"].append(new_tool_use)
|
||||||
return ret
|
return cast(MessageParam, ret)
|
||||||
content = message.get("content")
|
content = msg.get("content")
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
# fix empty text
|
# fix empty text
|
||||||
if content == "":
|
if content == "":
|
||||||
@@ -354,7 +369,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
image_item = content.pop(img_idx)
|
image_item = content.pop(img_idx)
|
||||||
content.insert(first_txt_idx, image_item)
|
content.insert(first_txt_idx, image_item)
|
||||||
|
|
||||||
return message
|
return cast(MessageParam, msg)
|
||||||
|
|
||||||
def _with_cache_control_markers(self, messages: list[MessageParam]) -> list[MessageParam]:
|
def _with_cache_control_markers(self, messages: list[MessageParam]) -> list[MessageParam]:
|
||||||
"""Add cache control markers to messages for prompt caching.
|
"""Add cache control markers to messages for prompt caching.
|
||||||
@@ -369,7 +384,16 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]):
|
|||||||
def add_cache_control_marker(message: MessageParam):
|
def add_cache_control_marker(message: MessageParam):
|
||||||
if isinstance(message["content"], str):
|
if isinstance(message["content"], str):
|
||||||
message["content"] = [{"type": "text", "text": message["content"]}]
|
message["content"] = [{"type": "text", "text": message["content"]}]
|
||||||
message["content"][-1]["cache_control"] = {"type": "ephemeral"}
|
# Assumptions on the next line:
|
||||||
|
# - content is a list (str case handled above; this codebase only
|
||||||
|
# ever constructs content as a str or a list)
|
||||||
|
# - the list is non-empty (guaranteed by the empty-content
|
||||||
|
# replacement in `_from_universal_context_messages`)
|
||||||
|
# - the last item is a dict. The standard-message path enforces
|
||||||
|
# this via TypedDicts (which are dicts at runtime); the
|
||||||
|
# LLMSpecificMessage passthrough doesn't, but in practice
|
||||||
|
# callers use dicts.
|
||||||
|
cast(list[Any], message["content"])[-1]["cache_control"] = {"type": "ephemeral"}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Add cache control markers to the most recent two user messages.
|
# Add cache control markers to the most recent two user messages.
|
||||||
|
|||||||
@@ -8,9 +8,9 @@
|
|||||||
|
|
||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, TypedDict
|
from typing import Any, TypedDict, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -110,7 +110,10 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
|
|||||||
Returns:
|
Returns:
|
||||||
List of messages in a format ready for logging about AWS Nova Sonic.
|
List of messages in a format ready for logging about AWS Nova Sonic.
|
||||||
"""
|
"""
|
||||||
return self._from_universal_context_messages(self.get_messages(context)).messages
|
return [
|
||||||
|
asdict(m)
|
||||||
|
for m in self._from_universal_context_messages(self.get_messages(context)).messages
|
||||||
|
]
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ConvertedMessages:
|
class ConvertedMessages:
|
||||||
@@ -123,18 +126,27 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
|
|||||||
self, universal_context_messages: list[LLMContextMessage]
|
self, universal_context_messages: list[LLMContextMessage]
|
||||||
) -> ConvertedMessages:
|
) -> ConvertedMessages:
|
||||||
system_instruction = None
|
system_instruction = None
|
||||||
messages = []
|
messages: list[AWSNovaSonicConversationHistoryMessage] = []
|
||||||
|
|
||||||
# Bail if there are no messages
|
# Bail if there are no messages
|
||||||
if not universal_context_messages:
|
if not universal_context_messages:
|
||||||
return self.ConvertedMessages()
|
return self.ConvertedMessages(messages=[])
|
||||||
|
|
||||||
universal_context_messages = copy.deepcopy(universal_context_messages)
|
# NOTE: This adapter does not yet handle ``LLMSpecificMessage`` —
|
||||||
|
# those are filtered out below (the role-extraction and conversion
|
||||||
|
# logic only applies to standard message dicts). If/when this
|
||||||
|
# adapter grows a per-provider passthrough like the Anthropic
|
||||||
|
# adapter has, LLMSpecific items can flow through.
|
||||||
|
ucm: list[dict[str, Any]] = [
|
||||||
|
cast(dict[str, Any], m)
|
||||||
|
for m in copy.deepcopy(universal_context_messages)
|
||||||
|
if isinstance(m, dict)
|
||||||
|
]
|
||||||
|
|
||||||
# If we have a "system" message as our first message,
|
# If we have a "system" message as our first message,
|
||||||
# pull that out into "instruction"
|
# pull that out into "instruction"
|
||||||
if universal_context_messages[0].get("role") == "system":
|
if ucm and ucm[0].get("role") == "system":
|
||||||
system = universal_context_messages.pop(0)
|
system = ucm.pop(0)
|
||||||
content = system.get("content")
|
content = system.get("content")
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
system_instruction = content
|
system_instruction = content
|
||||||
@@ -145,19 +157,21 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
|
|||||||
|
|
||||||
# Convert any remaining "system"/"developer" messages to "user",
|
# Convert any remaining "system"/"developer" messages to "user",
|
||||||
# as Nova Sonic only supports "user" and "assistant" in history.
|
# as Nova Sonic only supports "user" and "assistant" in history.
|
||||||
for msg in universal_context_messages:
|
for msg in ucm:
|
||||||
if msg.get("role") in ("system", "developer"):
|
if msg.get("role") in ("system", "developer"):
|
||||||
msg["role"] = "user"
|
msg["role"] = "user"
|
||||||
|
|
||||||
# Process remaining messages to fill out conversation history.
|
# Process remaining messages to fill out conversation history.
|
||||||
for universal_context_message in universal_context_messages:
|
for universal_context_message in ucm:
|
||||||
message = self._from_universal_context_message(universal_context_message)
|
message = self._from_universal_context_message(universal_context_message)
|
||||||
if message:
|
if message:
|
||||||
messages.append(message)
|
messages.append(message)
|
||||||
|
|
||||||
return self.ConvertedMessages(messages=messages, system_instruction=system_instruction)
|
return self.ConvertedMessages(messages=messages, system_instruction=system_instruction)
|
||||||
|
|
||||||
def _from_universal_context_message(self, message) -> AWSNovaSonicConversationHistoryMessage:
|
def _from_universal_context_message(
|
||||||
|
self, message: dict[str, Any]
|
||||||
|
) -> AWSNovaSonicConversationHistoryMessage | None:
|
||||||
"""Convert standard message format to Nova Sonic format.
|
"""Convert standard message format to Nova Sonic format.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -167,17 +181,18 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]):
|
|||||||
Nova Sonic conversation history message, or None if not convertible.
|
Nova Sonic conversation history message, or None if not convertible.
|
||||||
"""
|
"""
|
||||||
role = message.get("role")
|
role = message.get("role")
|
||||||
if message.get("role") == "user" or message.get("role") == "assistant":
|
if role == "user" or role == "assistant":
|
||||||
content = message.get("content")
|
content = message.get("content")
|
||||||
if isinstance(message.get("content"), list):
|
if isinstance(content, list):
|
||||||
content = ""
|
text_parts = []
|
||||||
for c in message.get("content"):
|
for c in content:
|
||||||
if c.get("type") == "text":
|
if c.get("type") == "text":
|
||||||
content += " " + c.get("text")
|
text_parts.append(c.get("text"))
|
||||||
else:
|
else:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unhandled content type in context message: {c.get('type')} - {message}"
|
f"Unhandled content type in context message: {c.get('type')} - {message}"
|
||||||
)
|
)
|
||||||
|
content = " ".join(t for t in text_parts if t)
|
||||||
# There won't be content if this is an assistant tool call entry.
|
# There won't be content if this is an assistant tool call entry.
|
||||||
# We're ignoring those since they can't be loaded into AWS Nova Sonic conversation
|
# We're ignoring those since they can't be loaded into AWS Nova Sonic conversation
|
||||||
# history
|
# history
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import base64
|
|||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, TypedDict
|
from typing import Any, TypedDict, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -68,16 +68,19 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
|
|||||||
system_instruction,
|
system_instruction,
|
||||||
discard_context_system=True,
|
discard_context_system=True,
|
||||||
)
|
)
|
||||||
return {
|
return cast(
|
||||||
"system": [{"text": effective_system}] if effective_system else None,
|
AWSBedrockLLMInvocationParams,
|
||||||
"messages": converted.messages,
|
{
|
||||||
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
|
"system": [{"text": effective_system}] if effective_system else None,
|
||||||
"tools": self.from_standard_tools(context.tools) or [],
|
"messages": converted.messages,
|
||||||
# To avoid refactoring in AWSBedrockLLMService, we just pass through tool_choice.
|
# NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
|
||||||
# Eventually (when we don't have to maintain the non-LLMContext code path) we should do
|
"tools": self.from_standard_tools(context.tools) or [],
|
||||||
# the conversion to Bedrock's expected format here rather than in AWSBedrockLLMService.
|
# To avoid refactoring in AWSBedrockLLMService, we just pass through tool_choice.
|
||||||
"tool_choice": context.tool_choice,
|
# Eventually (when we don't have to maintain the non-LLMContext code path) we should do
|
||||||
}
|
# the conversion to Bedrock's expected format here rather than in AWSBedrockLLMService.
|
||||||
|
"tool_choice": context.tool_choice,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
def get_messages_for_logging(self, context) -> list[dict[str, Any]]:
|
def get_messages_for_logging(self, context) -> list[dict[str, Any]]:
|
||||||
"""Get messages from a universal LLM context in a format ready for logging about AWS Bedrock.
|
"""Get messages from a universal LLM context in a format ready for logging about AWS Bedrock.
|
||||||
@@ -213,35 +216,36 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
message = copy.deepcopy(message)
|
# ChatCompletionMessageParam (input) and the dict shape Bedrock expects
|
||||||
if message["role"] == "tool":
|
# are different — work with the deepcopied message as a plain dict for
|
||||||
|
# the transformations below.
|
||||||
|
msg = cast(dict[str, Any], copy.deepcopy(message))
|
||||||
|
if msg["role"] == "tool":
|
||||||
# Try to parse the content as JSON if it looks like JSON
|
# Try to parse the content as JSON if it looks like JSON
|
||||||
try:
|
try:
|
||||||
if message["content"].strip().startswith("{") and message[
|
if msg["content"].strip().startswith("{") and msg["content"].strip().endswith("}"):
|
||||||
"content"
|
content_json = json.loads(msg["content"])
|
||||||
].strip().endswith("}"):
|
|
||||||
content_json = json.loads(message["content"])
|
|
||||||
tool_result_content = [{"json": content_json}]
|
tool_result_content = [{"json": content_json}]
|
||||||
else:
|
else:
|
||||||
tool_result_content = [{"text": message["content"]}]
|
tool_result_content = [{"text": msg["content"]}]
|
||||||
except (json.JSONDecodeError, ValueError, AttributeError):
|
except (json.JSONDecodeError, ValueError, AttributeError):
|
||||||
tool_result_content = [{"text": message["content"]}]
|
tool_result_content = [{"text": msg["content"]}]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": [
|
"content": [
|
||||||
{
|
{
|
||||||
"toolResult": {
|
"toolResult": {
|
||||||
"toolUseId": message["tool_call_id"],
|
"toolUseId": msg["tool_call_id"],
|
||||||
"content": tool_result_content,
|
"content": tool_result_content,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
if message.get("tool_calls"):
|
if msg.get("tool_calls"):
|
||||||
tc = message["tool_calls"]
|
tc = msg["tool_calls"]
|
||||||
ret = {"role": "assistant", "content": []}
|
ret: dict[str, Any] = {"role": "assistant", "content": []}
|
||||||
for tool_call in tc:
|
for tool_call in tc:
|
||||||
function = tool_call["function"]
|
function = tool_call["function"]
|
||||||
arguments = json.loads(function["arguments"])
|
arguments = json.loads(function["arguments"])
|
||||||
@@ -256,12 +260,12 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
|
|||||||
return ret
|
return ret
|
||||||
|
|
||||||
# Handle text content
|
# Handle text content
|
||||||
content = message.get("content")
|
content = msg.get("content")
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
if content == "":
|
if content == "":
|
||||||
return {"role": message["role"], "content": [{"text": "(empty)"}]}
|
return {"role": msg["role"], "content": [{"text": "(empty)"}]}
|
||||||
else:
|
else:
|
||||||
return {"role": message["role"], "content": [{"text": content}]}
|
return {"role": msg["role"], "content": [{"text": content}]}
|
||||||
elif isinstance(content, list):
|
elif isinstance(content, list):
|
||||||
new_content = []
|
new_content = []
|
||||||
for item in content:
|
for item in content:
|
||||||
@@ -300,9 +304,9 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
|
|||||||
# Move image before the first text
|
# Move image before the first text
|
||||||
image_item = new_content.pop(img_idx)
|
image_item = new_content.pop(img_idx)
|
||||||
new_content.insert(first_txt_idx, image_item)
|
new_content.insert(first_txt_idx, image_item)
|
||||||
return {"role": message["role"], "content": new_content}
|
return {"role": msg["role"], "content": new_content}
|
||||||
|
|
||||||
return message
|
return msg
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_bedrock_function_format(function: FunctionSchema) -> dict[str, Any]:
|
def _to_bedrock_function_format(function: FunctionSchema) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, TypedDict
|
from typing import Any, TypedDict, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai import NotGiven
|
from openai import NotGiven
|
||||||
@@ -154,9 +154,12 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
messages = self._from_universal_context_messages(self.get_messages(context)).messages
|
messages = self._from_universal_context_messages(self.get_messages(context)).messages
|
||||||
|
|
||||||
# Sanitize messages for logging
|
# Sanitize messages for logging
|
||||||
messages_for_logging = []
|
messages_for_logging: list[dict[str, Any]] = []
|
||||||
for message in messages:
|
for message in messages:
|
||||||
obj = message.to_json_dict()
|
# `to_json_dict()` returns `dict[str, object]`; treat as a plain
|
||||||
|
# dict for the value indexing/mutation below. The broad `except`
|
||||||
|
# below is the safety net if any item isn't shaped as expected.
|
||||||
|
obj: dict[str, Any] = cast(dict[str, Any], message.to_json_dict())
|
||||||
try:
|
try:
|
||||||
if "parts" in obj:
|
if "parts" in obj:
|
||||||
for part in obj["parts"]:
|
for part in obj["parts"]:
|
||||||
@@ -274,7 +277,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
# Check if we only have function-related messages (no regular text)
|
# Check if we only have function-related messages (no regular text)
|
||||||
effective_system = extracted_system or system_instruction
|
effective_system = extracted_system or system_instruction
|
||||||
has_regular_messages = any(
|
has_regular_messages = any(
|
||||||
len(msg.parts) == 1
|
msg.parts is not None
|
||||||
|
and len(msg.parts) == 1
|
||||||
and getattr(msg.parts[0], "text", None)
|
and getattr(msg.parts[0], "text", None)
|
||||||
and not getattr(msg.parts[0], "function_call", None)
|
and not getattr(msg.parts[0], "function_call", None)
|
||||||
and not getattr(msg.parts[0], "function_response", None)
|
and not getattr(msg.parts[0], "function_response", None)
|
||||||
@@ -346,8 +350,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))]
|
parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))]
|
||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
role = message["role"]
|
# ChatCompletionMessageParam (a union of TypedDicts) doesn't allow
|
||||||
content = message.get("content", [])
|
# the dict-style key access used below; treat it as a plain dict.
|
||||||
|
msg = cast(dict[str, Any], message)
|
||||||
|
role = msg["role"]
|
||||||
|
content = msg.get("content", [])
|
||||||
|
|
||||||
# Convert non-initial system/developer messages to user role,
|
# Convert non-initial system/developer messages to user role,
|
||||||
# as Gemini doesn't support these as input messages.
|
# as Gemini doesn't support these as input messages.
|
||||||
@@ -359,8 +366,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
parts = []
|
parts = []
|
||||||
tool_call_id_to_name_mapping = {}
|
tool_call_id_to_name_mapping = {}
|
||||||
|
|
||||||
if message.get("tool_calls"):
|
if msg.get("tool_calls"):
|
||||||
for tc in message["tool_calls"]:
|
for tc in msg["tool_calls"]:
|
||||||
id = tc["id"]
|
id = tc["id"]
|
||||||
name = tc["function"]["name"]
|
name = tc["function"]["name"]
|
||||||
tool_call_id_to_name_mapping[id] = name
|
tool_call_id_to_name_mapping[id] = name
|
||||||
@@ -376,7 +383,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
elif role == "tool":
|
elif role == "tool":
|
||||||
role = "user"
|
role = "user"
|
||||||
try:
|
try:
|
||||||
response = json.loads(message["content"])
|
response = json.loads(msg["content"])
|
||||||
if isinstance(response, dict):
|
if isinstance(response, dict):
|
||||||
response_dict = response
|
response_dict = response
|
||||||
else:
|
else:
|
||||||
@@ -384,10 +391,10 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Response might not be JSON-deserializable.
|
# Response might not be JSON-deserializable.
|
||||||
# This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string.
|
# This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string.
|
||||||
response_dict = {"value": message["content"]}
|
response_dict = {"value": msg["content"]}
|
||||||
|
|
||||||
# Get function name from mapping using tool_call_id, or fallback
|
# Get function name from mapping using tool_call_id, or fallback
|
||||||
tool_call_id = message.get("tool_call_id")
|
tool_call_id = msg.get("tool_call_id")
|
||||||
function_name = "tool_call_result" # Default fallback
|
function_name = "tool_call_result" # Default fallback
|
||||||
if tool_call_id and tool_call_id in params.tool_call_id_to_name_mapping:
|
if tool_call_id and tool_call_id in params.tool_call_id_to_name_mapping:
|
||||||
function_name = params.tool_call_id_to_name_mapping[tool_call_id]
|
function_name = params.tool_call_id_to_name_mapping[tool_call_id]
|
||||||
@@ -491,7 +498,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
|
|
||||||
def is_tool_call_message(msg: Content) -> bool:
|
def is_tool_call_message(msg: Content) -> bool:
|
||||||
"""Check if message contains only function_call parts."""
|
"""Check if message contains only function_call parts."""
|
||||||
return (
|
return bool(
|
||||||
msg.role == "model"
|
msg.role == "model"
|
||||||
and msg.parts
|
and msg.parts
|
||||||
and all(getattr(part, "function_call", None) for part in msg.parts)
|
and all(getattr(part, "function_call", None) for part in msg.parts)
|
||||||
@@ -499,6 +506,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
|
|
||||||
def message_has_thought_signature(msg: Content) -> bool:
|
def message_has_thought_signature(msg: Content) -> bool:
|
||||||
"""Check if any part in the message has a thought_signature."""
|
"""Check if any part in the message has a thought_signature."""
|
||||||
|
if msg.parts is None:
|
||||||
|
return False
|
||||||
return any(getattr(part, "thought_signature", None) for part in msg.parts)
|
return any(getattr(part, "thought_signature", None) for part in msg.parts)
|
||||||
|
|
||||||
merged_messages = []
|
merged_messages = []
|
||||||
@@ -564,6 +573,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
logger.debug(f"Thought signatures to apply: {len(thought_signature_dicts)}")
|
logger.debug(f"Thought signatures to apply: {len(thought_signature_dicts)}")
|
||||||
for ts in thought_signature_dicts:
|
for ts in thought_signature_dicts:
|
||||||
bookmark = ts.get("bookmark")
|
bookmark = ts.get("bookmark")
|
||||||
|
if bookmark is None:
|
||||||
|
continue
|
||||||
if bookmark.get("function_call"):
|
if bookmark.get("function_call"):
|
||||||
logger.trace(f" - To function call: {bookmark['function_call']}")
|
logger.trace(f" - To function call: {bookmark['function_call']}")
|
||||||
elif bookmark.get("text"):
|
elif bookmark.get("text"):
|
||||||
@@ -665,6 +676,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
|
|||||||
if (
|
if (
|
||||||
hasattr(part, "inline_data")
|
hasattr(part, "inline_data")
|
||||||
and part.inline_data
|
and part.inline_data
|
||||||
|
and part.inline_data.data is not None
|
||||||
|
and bookmark_inline_data.data is not None
|
||||||
# Comparing length should be good enough for matching inline data,
|
# Comparing length should be good enough for matching inline data,
|
||||||
# especially since we're already matching thought signatures in
|
# especially since we're already matching thought signatures in
|
||||||
# strict message order. Comparing actual data is expensive.
|
# strict message order. Comparing actual data is expensive.
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ Grok's Voice Agent API.
|
|||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, TypedDict
|
from typing import Any, TypedDict, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -85,7 +85,10 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
Returns:
|
Returns:
|
||||||
List of messages with sensitive data redacted.
|
List of messages with sensitive data redacted.
|
||||||
"""
|
"""
|
||||||
return self.get_messages(context, truncate_large_values=True)
|
return cast(
|
||||||
|
list[dict[str, Any]],
|
||||||
|
self.get_messages(context, truncate_large_values=True),
|
||||||
|
)
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ConvertedMessages:
|
class ConvertedMessages:
|
||||||
@@ -111,11 +114,20 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
if not universal_context_messages:
|
if not universal_context_messages:
|
||||||
return self.ConvertedMessages(messages=[])
|
return self.ConvertedMessages(messages=[])
|
||||||
|
|
||||||
messages = copy.deepcopy(universal_context_messages)
|
# NOTE: This adapter does not yet handle ``LLMSpecificMessage`` —
|
||||||
|
# those are filtered out below. Other adapters (e.g. Anthropic)
|
||||||
|
# dispatch LLMSpecific items through a per-provider passthrough.
|
||||||
|
# The pack-into-single-text-message strategy here doesn't compose
|
||||||
|
# with opaque per-provider payloads.
|
||||||
|
messages: list[dict[str, Any]] = [
|
||||||
|
cast(dict[str, Any], m)
|
||||||
|
for m in copy.deepcopy(universal_context_messages)
|
||||||
|
if isinstance(m, dict)
|
||||||
|
]
|
||||||
system_instruction = None
|
system_instruction = None
|
||||||
|
|
||||||
# Extract system message as session instructions
|
# Extract system message as session instructions
|
||||||
if messages[0].get("role") == "system":
|
if messages and messages[0].get("role") == "system":
|
||||||
system = messages.pop(0)
|
system = messages.pop(0)
|
||||||
content = system.get("content")
|
content = system.get("content")
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
@@ -133,7 +145,9 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
# Single user message can be sent normally
|
# Single user message can be sent normally
|
||||||
if len(messages) == 1 and messages[0].get("role") == "user":
|
if len(messages) == 1 and messages[0].get("role") == "user":
|
||||||
return self.ConvertedMessages(
|
return self.ConvertedMessages(
|
||||||
messages=[self._from_universal_context_message(messages[0])],
|
messages=[
|
||||||
|
self._from_universal_context_message(cast(LLMContextMessage, messages[0]))
|
||||||
|
],
|
||||||
system_instruction=system_instruction,
|
system_instruction=system_instruction,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -181,26 +195,29 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
Returns:
|
Returns:
|
||||||
ConversationItem formatted for Grok Realtime API.
|
ConversationItem formatted for Grok Realtime API.
|
||||||
"""
|
"""
|
||||||
if message.get("role") == "user":
|
# NOTE: ``LLMSpecificMessage`` is not yet handled here — see the
|
||||||
content = message.get("content")
|
# corresponding note in `_from_universal_context_messages`.
|
||||||
|
msg = cast(dict[str, Any], message)
|
||||||
|
if msg.get("role") == "user":
|
||||||
|
content = msg.get("content")
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
text_content = ""
|
text_parts = []
|
||||||
for c in content:
|
for c in content:
|
||||||
if c.get("type") == "text":
|
if c.get("type") == "text":
|
||||||
text_content += " " + c.get("text")
|
text_parts.append(c.get("text"))
|
||||||
else:
|
else:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unhandled content type in context message: {c.get('type')} - {message}"
|
f"Unhandled content type in context message: {c.get('type')} - {msg}"
|
||||||
)
|
)
|
||||||
content = text_content.strip()
|
content = " ".join(t for t in text_parts if t).strip()
|
||||||
return events.ConversationItem(
|
return events.ConversationItem(
|
||||||
role="user",
|
role="user",
|
||||||
type="message",
|
type="message",
|
||||||
content=[events.ItemContent(type="input_text", text=content)],
|
content=[events.ItemContent(type="input_text", text=content)],
|
||||||
)
|
)
|
||||||
|
|
||||||
if message.get("role") == "assistant" and message.get("tool_calls"):
|
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||||
tc = message.get("tool_calls")[0]
|
tc = msg["tool_calls"][0]
|
||||||
return events.ConversationItem(
|
return events.ConversationItem(
|
||||||
type="function_call",
|
type="function_call",
|
||||||
call_id=tc["id"],
|
call_id=tc["id"],
|
||||||
@@ -208,7 +225,7 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
arguments=tc["function"]["arguments"],
|
arguments=tc["function"]["arguments"],
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.error(f"Unhandled message type in _from_universal_context_message: {message}")
|
raise ValueError(f"Unhandled message type in _from_universal_context_message: {msg}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_grok_function_format(function: FunctionSchema) -> dict[str, Any]:
|
def _to_grok_function_format(function: FunctionSchema) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ Inworld's Realtime API.
|
|||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, TypedDict
|
from typing import Any, TypedDict, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -85,7 +85,10 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
Returns:
|
Returns:
|
||||||
List of messages with sensitive data redacted.
|
List of messages with sensitive data redacted.
|
||||||
"""
|
"""
|
||||||
return self.get_messages(context, truncate_large_values=True)
|
return cast(
|
||||||
|
list[dict[str, Any]],
|
||||||
|
self.get_messages(context, truncate_large_values=True),
|
||||||
|
)
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ConvertedMessages:
|
class ConvertedMessages:
|
||||||
@@ -111,11 +114,20 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
if not universal_context_messages:
|
if not universal_context_messages:
|
||||||
return self.ConvertedMessages(messages=[])
|
return self.ConvertedMessages(messages=[])
|
||||||
|
|
||||||
messages = copy.deepcopy(universal_context_messages)
|
# NOTE: This adapter does not yet handle ``LLMSpecificMessage`` —
|
||||||
|
# those are filtered out below. Other adapters (e.g. Anthropic)
|
||||||
|
# dispatch LLMSpecific items through a per-provider passthrough.
|
||||||
|
# The pack-into-single-text-message strategy here doesn't compose
|
||||||
|
# with opaque per-provider payloads.
|
||||||
|
messages: list[dict[str, Any]] = [
|
||||||
|
cast(dict[str, Any], m)
|
||||||
|
for m in copy.deepcopy(universal_context_messages)
|
||||||
|
if isinstance(m, dict)
|
||||||
|
]
|
||||||
system_instruction = None
|
system_instruction = None
|
||||||
|
|
||||||
# Extract system message as session instructions
|
# Extract system message as session instructions
|
||||||
if messages[0].get("role") == "system":
|
if messages and messages[0].get("role") == "system":
|
||||||
system = messages.pop(0)
|
system = messages.pop(0)
|
||||||
content = system.get("content")
|
content = system.get("content")
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
@@ -133,7 +145,9 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
# Single user message can be sent normally
|
# Single user message can be sent normally
|
||||||
if len(messages) == 1 and messages[0].get("role") == "user":
|
if len(messages) == 1 and messages[0].get("role") == "user":
|
||||||
return self.ConvertedMessages(
|
return self.ConvertedMessages(
|
||||||
messages=[self._from_universal_context_message(messages[0])],
|
messages=[
|
||||||
|
self._from_universal_context_message(cast(LLMContextMessage, messages[0]))
|
||||||
|
],
|
||||||
system_instruction=system_instruction,
|
system_instruction=system_instruction,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -181,26 +195,29 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
Returns:
|
Returns:
|
||||||
ConversationItem formatted for Inworld Realtime API.
|
ConversationItem formatted for Inworld Realtime API.
|
||||||
"""
|
"""
|
||||||
if message.get("role") == "user":
|
# NOTE: ``LLMSpecificMessage`` is not yet handled here — see the
|
||||||
content = message.get("content")
|
# corresponding note in `_from_universal_context_messages`.
|
||||||
|
msg = cast(dict[str, Any], message)
|
||||||
|
if msg.get("role") == "user":
|
||||||
|
content = msg.get("content")
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
text_content = ""
|
text_parts = []
|
||||||
for c in content:
|
for c in content:
|
||||||
if c.get("type") == "text":
|
if c.get("type") == "text":
|
||||||
text_content += " " + c.get("text")
|
text_parts.append(c.get("text"))
|
||||||
else:
|
else:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unhandled content type in context message: {c.get('type')} - {message}"
|
f"Unhandled content type in context message: {c.get('type')} - {msg}"
|
||||||
)
|
)
|
||||||
content = text_content.strip()
|
content = " ".join(t for t in text_parts if t).strip()
|
||||||
return events.ConversationItem(
|
return events.ConversationItem(
|
||||||
role="user",
|
role="user",
|
||||||
type="message",
|
type="message",
|
||||||
content=[events.ItemContent(type="input_text", text=content)],
|
content=[events.ItemContent(type="input_text", text=content)],
|
||||||
)
|
)
|
||||||
|
|
||||||
if message.get("role") == "assistant" and message.get("tool_calls"):
|
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||||
tc = message.get("tool_calls")[0]
|
tc = msg["tool_calls"][0]
|
||||||
return events.ConversationItem(
|
return events.ConversationItem(
|
||||||
type="function_call",
|
type="function_call",
|
||||||
call_id=tc["id"],
|
call_id=tc["id"],
|
||||||
@@ -208,7 +225,7 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
arguments=tc["function"]["arguments"],
|
arguments=tc["function"]["arguments"],
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.error(f"Unhandled message type in _from_universal_context_message: {message}")
|
raise ValueError(f"Unhandled message type in _from_universal_context_message: {msg}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_inworld_function_format(function: FunctionSchema) -> dict[str, Any]:
|
def _to_inworld_function_format(function: FunctionSchema) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -127,12 +127,15 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if system_instruction:
|
if system_instruction:
|
||||||
# Detect initial system message for warning purposes (don't extract)
|
# Detect initial system message for warning purposes (don't extract).
|
||||||
initial_content = (
|
# ChatCompletionMessageParam.content is `str | Iterable[...]`; we
|
||||||
messages[0].get("content", "")
|
# only forward it for warning purposes, so coerce non-strings to
|
||||||
if messages and messages[0].get("role") == "system"
|
# None — the resolver handles None.
|
||||||
else None
|
initial_content: str | None = None
|
||||||
)
|
if messages and messages[0].get("role") == "system":
|
||||||
|
raw_content = messages[0].get("content", "")
|
||||||
|
if isinstance(raw_content, str):
|
||||||
|
initial_content = raw_content
|
||||||
self._resolve_system_instruction(
|
self._resolve_system_instruction(
|
||||||
initial_content,
|
initial_content,
|
||||||
system_instruction,
|
system_instruction,
|
||||||
@@ -140,12 +143,15 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
|
|||||||
)
|
)
|
||||||
messages = [{"role": "system", "content": system_instruction}] + messages
|
messages = [{"role": "system", "content": system_instruction}] + messages
|
||||||
|
|
||||||
return {
|
return cast(
|
||||||
"messages": messages,
|
OpenAILLMInvocationParams,
|
||||||
# NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
|
{
|
||||||
"tools": self.from_standard_tools(context.tools),
|
"messages": messages,
|
||||||
"tool_choice": _openai_from_llm_context_tool_choice(context.tool_choice),
|
# NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN)
|
||||||
}
|
"tools": self.from_standard_tools(context.tools),
|
||||||
|
"tool_choice": _openai_from_llm_context_tool_choice(context.tool_choice),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> list[ChatCompletionToolParam]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> list[ChatCompletionToolParam]:
|
||||||
"""Convert function schemas to OpenAI's function-calling format.
|
"""Convert function schemas to OpenAI's function-calling format.
|
||||||
@@ -158,13 +164,19 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
|
|||||||
with ChatCompletion API.
|
with ChatCompletion API.
|
||||||
"""
|
"""
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
formatted_standard_tools = [
|
# `function=...` expects a `FunctionDefinition` TypedDict; the dict
|
||||||
ChatCompletionToolParam(type="function", function=func.to_default_dict())
|
# produced by `to_default_dict()` is structurally compatible. Cast at
|
||||||
|
# the boundary.
|
||||||
|
formatted_standard_tools: list[ChatCompletionToolParam] = [
|
||||||
|
ChatCompletionToolParam(type="function", function=cast(Any, func.to_default_dict()))
|
||||||
for func in functions_schema
|
for func in functions_schema
|
||||||
]
|
]
|
||||||
custom_openai_tools = []
|
custom_openai_tools: list[ChatCompletionToolParam] = []
|
||||||
if tools_schema.custom_tools:
|
if tools_schema.custom_tools:
|
||||||
custom_openai_tools = tools_schema.custom_tools.get(AdapterType.OPENAI, [])
|
custom_openai_tools = cast(
|
||||||
|
list[ChatCompletionToolParam],
|
||||||
|
tools_schema.custom_tools.get(AdapterType.OPENAI, []),
|
||||||
|
)
|
||||||
return formatted_standard_tools + custom_openai_tools
|
return formatted_standard_tools + custom_openai_tools
|
||||||
|
|
||||||
def get_messages_for_logging(self, context: LLMContext) -> list[dict[str, Any]]:
|
def get_messages_for_logging(self, context: LLMContext) -> list[dict[str, Any]]:
|
||||||
@@ -178,7 +190,10 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]):
|
|||||||
Returns:
|
Returns:
|
||||||
List of messages in a format ready for logging about OpenAI.
|
List of messages in a format ready for logging about OpenAI.
|
||||||
"""
|
"""
|
||||||
return self.get_messages(context, truncate_large_values=True)
|
return cast(
|
||||||
|
list[dict[str, Any]],
|
||||||
|
self.get_messages(context, truncate_large_values=True),
|
||||||
|
)
|
||||||
|
|
||||||
def _from_universal_context_messages(
|
def _from_universal_context_messages(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
import copy
|
import copy
|
||||||
import json
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, TypedDict
|
from typing import Any, TypedDict, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -81,7 +81,7 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
Returns:
|
Returns:
|
||||||
List of messages in a format ready for logging about OpenAI Realtime.
|
List of messages in a format ready for logging about OpenAI Realtime.
|
||||||
"""
|
"""
|
||||||
return self.get_messages(context, truncate_large_values=True)
|
return cast(list[dict[str, Any]], self.get_messages(context, truncate_large_values=True))
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ConvertedMessages:
|
class ConvertedMessages:
|
||||||
@@ -101,12 +101,24 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
if not universal_context_messages:
|
if not universal_context_messages:
|
||||||
return self.ConvertedMessages(messages=[])
|
return self.ConvertedMessages(messages=[])
|
||||||
|
|
||||||
messages = copy.deepcopy(universal_context_messages)
|
# NOTE: This adapter does not yet handle ``LLMSpecificMessage`` — those
|
||||||
|
# are filtered out below. Other adapters (e.g. Anthropic) dispatch
|
||||||
|
# LLMSpecific items through a per-provider passthrough. For OpenAI
|
||||||
|
# Realtime, the strategy here packs a multi-message history into a
|
||||||
|
# single text message (see comment further down), which doesn't
|
||||||
|
# compose with opaque per-provider payloads. If/when this adapter
|
||||||
|
# adopts the per-message strategy, LLMSpecific items can flow
|
||||||
|
# through `_from_universal_context_message` like in other adapters.
|
||||||
|
messages: list[dict[str, Any]] = [
|
||||||
|
cast(dict[str, Any], m)
|
||||||
|
for m in copy.deepcopy(universal_context_messages)
|
||||||
|
if isinstance(m, dict)
|
||||||
|
]
|
||||||
system_instruction = None
|
system_instruction = None
|
||||||
|
|
||||||
# If we have a "system" message as our first message,
|
# If we have a "system" message as our first message,
|
||||||
# pull that out into session "instructions"
|
# pull that out into session "instructions"
|
||||||
if messages[0].get("role") == "system":
|
if messages and messages[0].get("role") == "system":
|
||||||
system = messages.pop(0)
|
system = messages.pop(0)
|
||||||
content = system.get("content")
|
content = system.get("content")
|
||||||
if isinstance(content, str):
|
if isinstance(content, str):
|
||||||
@@ -124,7 +136,9 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
# If we have just a single "user" item, we can just send it normally
|
# If we have just a single "user" item, we can just send it normally
|
||||||
if len(messages) == 1 and messages[0].get("role") == "user":
|
if len(messages) == 1 and messages[0].get("role") == "user":
|
||||||
return self.ConvertedMessages(
|
return self.ConvertedMessages(
|
||||||
messages=[self._from_universal_context_message(messages[0])],
|
messages=[
|
||||||
|
self._from_universal_context_message(cast(LLMContextMessage, messages[0]))
|
||||||
|
],
|
||||||
system_instruction=system_instruction,
|
system_instruction=system_instruction,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -142,18 +156,18 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
|
|
||||||
return self.ConvertedMessages(
|
return self.ConvertedMessages(
|
||||||
messages=[
|
messages=[
|
||||||
{
|
events.ConversationItem(
|
||||||
"role": "user",
|
role="user",
|
||||||
"type": "message",
|
type="message",
|
||||||
"content": [
|
content=[
|
||||||
{
|
events.ItemContent(
|
||||||
"type": "input_text",
|
type="input_text",
|
||||||
"text": "\n\n".join(
|
text="\n\n".join(
|
||||||
[intro_text, json.dumps(messages, indent=2), trailing_text]
|
[intro_text, json.dumps(messages, indent=2), trailing_text]
|
||||||
),
|
),
|
||||||
}
|
)
|
||||||
],
|
],
|
||||||
}
|
)
|
||||||
],
|
],
|
||||||
system_instruction=system_instruction,
|
system_instruction=system_instruction,
|
||||||
)
|
)
|
||||||
@@ -161,31 +175,34 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
def _from_universal_context_message(
|
def _from_universal_context_message(
|
||||||
self, message: LLMContextMessage
|
self, message: LLMContextMessage
|
||||||
) -> events.ConversationItem:
|
) -> events.ConversationItem:
|
||||||
if message.get("role") == "user":
|
# NOTE: ``LLMSpecificMessage`` is not yet handled here — see the
|
||||||
content = message.get("content")
|
# corresponding note in `_from_universal_context_messages`.
|
||||||
if isinstance(message.get("content"), list):
|
msg = cast(dict[str, Any], message)
|
||||||
|
if msg.get("role") == "user":
|
||||||
|
content = msg.get("content")
|
||||||
|
if isinstance(content, list):
|
||||||
content = ""
|
content = ""
|
||||||
for c in message.get("content"):
|
for c in msg.get("content", []):
|
||||||
if c.get("type") == "text":
|
if c.get("type") == "text":
|
||||||
content += " " + c.get("text")
|
content += " " + c.get("text")
|
||||||
else:
|
else:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Unhandled content type in context message: {c.get('type')} - {message}"
|
f"Unhandled content type in context message: {c.get('type')} - {msg}"
|
||||||
)
|
)
|
||||||
return events.ConversationItem(
|
return events.ConversationItem(
|
||||||
role="user",
|
role="user",
|
||||||
type="message",
|
type="message",
|
||||||
content=[events.ItemContent(type="input_text", text=content)],
|
content=[events.ItemContent(type="input_text", text=content)],
|
||||||
)
|
)
|
||||||
if message.get("role") == "assistant" and message.get("tool_calls"):
|
if msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||||
tc = message.get("tool_calls")[0]
|
tc = msg["tool_calls"][0]
|
||||||
return events.ConversationItem(
|
return events.ConversationItem(
|
||||||
type="function_call",
|
type="function_call",
|
||||||
call_id=tc["id"],
|
call_id=tc["id"],
|
||||||
name=tc["function"]["name"],
|
name=tc["function"]["name"],
|
||||||
arguments=tc["function"]["arguments"],
|
arguments=tc["function"]["arguments"],
|
||||||
)
|
)
|
||||||
logger.error(f"Unhandled message type in _from_universal_context_message: {message}")
|
raise ValueError(f"Unhandled message type in _from_universal_context_message: {msg}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_openai_realtime_function_format(function: FunctionSchema) -> dict[str, Any]:
|
def _to_openai_realtime_function_format(function: FunctionSchema) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
"""OpenAI Responses API adapter for Pipecat."""
|
"""OpenAI Responses API adapter for Pipecat."""
|
||||||
|
|
||||||
from typing import Any, TypedDict
|
from typing import Any, Required, TypedDict, cast
|
||||||
|
|
||||||
from openai._types import NotGiven as OpenAINotGiven
|
from openai._types import NotGiven as OpenAINotGiven
|
||||||
from openai.types.responses import FunctionToolParam, ResponseInputItemParam, ToolParam
|
from openai.types.responses import FunctionToolParam, ResponseInputItemParam, ToolParam
|
||||||
@@ -23,8 +23,10 @@ from pipecat.processors.aggregators.llm_context import (
|
|||||||
class OpenAIResponsesLLMInvocationParams(TypedDict, total=False):
|
class OpenAIResponsesLLMInvocationParams(TypedDict, total=False):
|
||||||
"""Context-based parameters for invoking OpenAI Responses API."""
|
"""Context-based parameters for invoking OpenAI Responses API."""
|
||||||
|
|
||||||
input: list[ResponseInputItemParam]
|
# `input` and `tools` are always populated by `get_llm_invocation_params`;
|
||||||
tools: list[ToolParam] | OpenAINotGiven
|
# `instructions` is only set when a system instruction is present.
|
||||||
|
input: Required[list[ResponseInputItemParam]]
|
||||||
|
tools: Required[list[ToolParam] | OpenAINotGiven]
|
||||||
instructions: str
|
instructions: str
|
||||||
|
|
||||||
|
|
||||||
@@ -64,8 +66,11 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam
|
|||||||
if system_instruction and messages:
|
if system_instruction and messages:
|
||||||
first_msg = messages[0] if not isinstance(messages[0], LLMSpecificMessage) else None
|
first_msg = messages[0] if not isinstance(messages[0], LLMSpecificMessage) else None
|
||||||
if first_msg and first_msg.get("role") == "system":
|
if first_msg and first_msg.get("role") == "system":
|
||||||
|
# `content` is `str | Iterable[...]`; we only forward it for
|
||||||
|
# warning purposes. Coerce non-strings to None.
|
||||||
|
first_content = first_msg.get("content", "")
|
||||||
self._resolve_system_instruction(
|
self._resolve_system_instruction(
|
||||||
first_msg.get("content", ""),
|
first_content if isinstance(first_content, str) else None,
|
||||||
system_instruction,
|
system_instruction,
|
||||||
discard_context_system=False,
|
discard_context_system=False,
|
||||||
)
|
)
|
||||||
@@ -143,7 +148,10 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam
|
|||||||
Returns:
|
Returns:
|
||||||
List of messages in a format ready for logging.
|
List of messages in a format ready for logging.
|
||||||
"""
|
"""
|
||||||
return self.get_messages(context, truncate_large_values=True)
|
return cast(
|
||||||
|
list[dict[str, Any]],
|
||||||
|
self.get_messages(context, truncate_large_values=True),
|
||||||
|
)
|
||||||
|
|
||||||
def _convert_messages_to_input(
|
def _convert_messages_to_input(
|
||||||
self, messages: list[LLMContextMessage]
|
self, messages: list[LLMContextMessage]
|
||||||
@@ -169,13 +177,15 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam
|
|||||||
content = message.get("content", "")
|
content = message.get("content", "")
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
content = self._convert_multimodal_content(content)
|
content = self._convert_multimodal_content(content)
|
||||||
result.append({"role": "developer", "content": content})
|
result.append(
|
||||||
|
cast(ResponseInputItemParam, {"role": "developer", "content": content})
|
||||||
|
)
|
||||||
|
|
||||||
elif role == "user":
|
elif role == "user":
|
||||||
content = message.get("content", "")
|
content = message.get("content", "")
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
content = self._convert_multimodal_content(content)
|
content = self._convert_multimodal_content(content)
|
||||||
result.append({"role": "user", "content": content})
|
result.append(cast(ResponseInputItemParam, {"role": "user", "content": content}))
|
||||||
|
|
||||||
elif role == "assistant":
|
elif role == "assistant":
|
||||||
tool_calls = message.get("tool_calls")
|
tool_calls = message.get("tool_calls")
|
||||||
@@ -194,7 +204,9 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam
|
|||||||
content = message.get("content", "")
|
content = message.get("content", "")
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
content = self._convert_multimodal_content(content)
|
content = self._convert_multimodal_content(content)
|
||||||
result.append({"role": "assistant", "content": content})
|
result.append(
|
||||||
|
cast(ResponseInputItemParam, {"role": "assistant", "content": content})
|
||||||
|
)
|
||||||
|
|
||||||
elif role == "tool":
|
elif role == "tool":
|
||||||
content = message.get("content", "")
|
content = message.get("content", "")
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ the messages are sent to Perplexity's API.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
from openai.types.chat import ChatCompletionMessageParam
|
from openai.types.chat import ChatCompletionMessageParam
|
||||||
|
|
||||||
@@ -116,7 +117,11 @@ class PerplexityLLMAdapter(OpenAILLMAdapter):
|
|||||||
if not messages:
|
if not messages:
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
messages = copy.deepcopy(messages)
|
# ChatCompletionMessageParam is a union of TypedDicts; the
|
||||||
|
# transformations below mutate by key/index in ways those TypedDicts
|
||||||
|
# don't permit. Work against a plain-dict view for the duration of
|
||||||
|
# the transformation and cast back at the return site.
|
||||||
|
msgs: list[dict[str, Any]] = cast(list[dict[str, Any]], copy.deepcopy(messages))
|
||||||
|
|
||||||
# Note: "developer" → "user" conversion is handled by the parent adapter
|
# Note: "developer" → "user" conversion is handled by the parent adapter
|
||||||
# via the convert_developer_to_user parameter.
|
# via the convert_developer_to_user parameter.
|
||||||
@@ -125,10 +130,10 @@ class PerplexityLLMAdapter(OpenAILLMAdapter):
|
|||||||
# Perplexity allows system messages at the start, but rejects them
|
# Perplexity allows system messages at the start, but rejects them
|
||||||
# after any non-system message.
|
# after any non-system message.
|
||||||
in_initial_system_block = True
|
in_initial_system_block = True
|
||||||
for i in range(len(messages)):
|
for i in range(len(msgs)):
|
||||||
if messages[i].get("role") == "system":
|
if msgs[i].get("role") == "system":
|
||||||
if not in_initial_system_block:
|
if not in_initial_system_block:
|
||||||
messages[i]["role"] = "user"
|
msgs[i]["role"] = "user"
|
||||||
else:
|
else:
|
||||||
in_initial_system_block = False
|
in_initial_system_block = False
|
||||||
|
|
||||||
@@ -137,9 +142,9 @@ class PerplexityLLMAdapter(OpenAILLMAdapter):
|
|||||||
# messages that violate Perplexity's strict alternation requirement.
|
# messages that violate Perplexity's strict alternation requirement.
|
||||||
# Skip consecutive system messages at the start — Perplexity allows those.
|
# Skip consecutive system messages at the start — Perplexity allows those.
|
||||||
i = 0
|
i = 0
|
||||||
while i < len(messages) - 1:
|
while i < len(msgs) - 1:
|
||||||
current = messages[i]
|
current = msgs[i]
|
||||||
next_msg = messages[i + 1]
|
next_msg = msgs[i + 1]
|
||||||
if current["role"] == next_msg["role"] == "system":
|
if current["role"] == next_msg["role"] == "system":
|
||||||
# Perplexity allows multiple initial system messages, don't merge
|
# Perplexity allows multiple initial system messages, don't merge
|
||||||
i += 1
|
i += 1
|
||||||
@@ -154,7 +159,7 @@ class PerplexityLLMAdapter(OpenAILLMAdapter):
|
|||||||
next_msg.get("content"), list
|
next_msg.get("content"), list
|
||||||
):
|
):
|
||||||
current["content"].extend(next_msg["content"])
|
current["content"].extend(next_msg["content"])
|
||||||
messages.pop(i + 1)
|
msgs.pop(i + 1)
|
||||||
else:
|
else:
|
||||||
i += 1
|
i += 1
|
||||||
|
|
||||||
@@ -162,7 +167,7 @@ class PerplexityLLMAdapter(OpenAILLMAdapter):
|
|||||||
# Perplexity requires the last message to be "user" or "tool".
|
# Perplexity requires the last message to be "user" or "tool".
|
||||||
# OpenAI appears to silently ignore trailing assistant messages
|
# OpenAI appears to silently ignore trailing assistant messages
|
||||||
# server-side, so removing them preserves equivalent behavior.
|
# server-side, so removing them preserves equivalent behavior.
|
||||||
while messages and messages[-1].get("role") == "assistant":
|
while msgs and msgs[-1].get("role") == "assistant":
|
||||||
messages.pop()
|
msgs.pop()
|
||||||
|
|
||||||
return messages
|
return cast(list[ChatCompletionMessageParam], msgs)
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ in-memory after first load to improve performance on subsequent accesses.
|
|||||||
import asyncio
|
import asyncio
|
||||||
import io
|
import io
|
||||||
import wave
|
import wave
|
||||||
from importlib.resources import files
|
from importlib.resources import as_file, files
|
||||||
|
|
||||||
import aiofiles
|
import aiofiles
|
||||||
|
|
||||||
@@ -52,10 +52,12 @@ async def load_dtmf_audio(button: KeypadEntry, *, sample_rate: int = 8000) -> by
|
|||||||
__DTMF_RESAMPLER__ = create_file_resampler()
|
__DTMF_RESAMPLER__ = create_file_resampler()
|
||||||
|
|
||||||
dtmf_file_name = __DTMF_FILE_NAME.get(button, f"dtmf-{button.value}.wav")
|
dtmf_file_name = __DTMF_FILE_NAME.get(button, f"dtmf-{button.value}.wav")
|
||||||
dtmf_file_path = files("pipecat.audio.dtmf").joinpath(dtmf_file_name)
|
# `as_file` materialises the resource as a real filesystem `Path`,
|
||||||
|
# which aiofiles can open. (For installed packages this is just the
|
||||||
async with aiofiles.open(dtmf_file_path, "rb") as f:
|
# bundled file; for zipped distributions it would extract to a temp.)
|
||||||
data = await f.read()
|
with as_file(files("pipecat.audio.dtmf").joinpath(dtmf_file_name)) as dtmf_file_path:
|
||||||
|
async with aiofiles.open(dtmf_file_path, "rb") as f:
|
||||||
|
data = await f.read()
|
||||||
|
|
||||||
with io.BytesIO(data) as buffer:
|
with io.BytesIO(data) as buffer:
|
||||||
with wave.open(buffer, "rb") as wf:
|
with wave.open(buffer, "rb") as wf:
|
||||||
|
|||||||
@@ -60,7 +60,12 @@ class RNNoiseFilter(BaseAudioFilter):
|
|||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# RNNoise always requires 48kHz
|
# The module-level import sets `RNNoise` to `None` if pyrnnoise
|
||||||
|
# isn't installed; raise instead of calling `None(...)` so the
|
||||||
|
# except clause handles it cleanly.
|
||||||
|
if RNNoise is None:
|
||||||
|
raise ImportError("pyrnnoise is not installed")
|
||||||
|
# RNNoise always requires 48kHz.
|
||||||
self._rnnoise = RNNoise(sample_rate=48000)
|
self._rnnoise = RNNoise(sample_rate=48000)
|
||||||
self._rnnoise_ready = True
|
self._rnnoise_ready = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -107,7 +112,7 @@ class RNNoiseFilter(BaseAudioFilter):
|
|||||||
Returns:
|
Returns:
|
||||||
Noise-suppressed audio data as bytes.
|
Noise-suppressed audio data as bytes.
|
||||||
"""
|
"""
|
||||||
if not self._rnnoise_ready or not self._filtering:
|
if not self._rnnoise_ready or not self._filtering or self._rnnoise is None:
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
# Resample input if needed
|
# Resample input if needed
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import io
|
|||||||
import wave
|
import wave
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, TypeAlias, TypeGuard, TypeVar
|
from typing import Any, TypeAlias, TypeGuard, TypeVar, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
|
from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
|
||||||
@@ -129,13 +129,13 @@ class LLMContext:
|
|||||||
url: The URL of the image.
|
url: The URL of the image.
|
||||||
text: Optional text to include with the image.
|
text: Optional text to include with the image.
|
||||||
"""
|
"""
|
||||||
content = []
|
content: list[dict[str, Any]] = []
|
||||||
if text:
|
if text:
|
||||||
content.append({"type": "text", "text": text})
|
content.append({"type": "text", "text": text})
|
||||||
|
|
||||||
content.append({"type": "image_url", "image_url": {"url": url}})
|
content.append({"type": "image_url", "image_url": {"url": url}})
|
||||||
|
|
||||||
return {"role": role, "content": content}
|
return cast(LLMContextMessage, {"role": role, "content": content})
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def create_image_message(
|
async def create_image_message(
|
||||||
@@ -187,7 +187,7 @@ class LLMContext:
|
|||||||
audio_frames: List of audio frame objects to include.
|
audio_frames: List of audio frame objects to include.
|
||||||
text: Optional text to include with the audio.
|
text: Optional text to include with the audio.
|
||||||
"""
|
"""
|
||||||
content = [{"type": "text", "text": text}]
|
content: list[dict[str, Any]] = [{"type": "text", "text": text}]
|
||||||
|
|
||||||
def encode_audio():
|
def encode_audio():
|
||||||
sample_rate = audio_frames[0].sample_rate
|
sample_rate = audio_frames[0].sample_rate
|
||||||
@@ -214,7 +214,7 @@ class LLMContext:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
return {"role": role, "content": content}
|
return cast(LLMContextMessage, {"role": role, "content": content})
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def messages(self) -> list[LLMContextMessage]:
|
def messages(self) -> list[LLMContextMessage]:
|
||||||
@@ -295,7 +295,10 @@ class LLMContext:
|
|||||||
result.append(msg_copy)
|
result.append(msg_copy)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
msg = copy.deepcopy(message)
|
# The standard message variant is a union of TypedDicts; the
|
||||||
|
# mutations below operate on plain dicts at runtime. Treat as
|
||||||
|
# such for the duration of the redaction loop.
|
||||||
|
msg: dict[str, Any] = cast(dict[str, Any], copy.deepcopy(message))
|
||||||
content = msg.get("content")
|
content = msg.get("content")
|
||||||
if isinstance(content, list):
|
if isinstance(content, list):
|
||||||
for item in content:
|
for item in content:
|
||||||
|
|||||||
@@ -67,9 +67,20 @@ class LangchainProcessor(FrameProcessor):
|
|||||||
# The last one by the human is the one we want to send to the LLM.
|
# The last one by the human is the one we want to send to the LLM.
|
||||||
logger.debug(f"Got transcription frame {frame}")
|
logger.debug(f"Got transcription frame {frame}")
|
||||||
messages = frame.context.get_messages()
|
messages = frame.context.get_messages()
|
||||||
text: str = messages[-1]["content"]
|
# Historically this processor has only handled plain-text user
|
||||||
|
# messages; the guards below make that contract explicit for the
|
||||||
|
# type checker. TODO: maybe handle other message shapes (provider-
|
||||||
|
# specific messages, multi-modal content lists, etc.).
|
||||||
|
last_message = messages[-1] if messages else None
|
||||||
|
if not isinstance(last_message, dict):
|
||||||
|
await self.push_frame(frame, direction)
|
||||||
|
return
|
||||||
|
content = last_message.get("content")
|
||||||
|
if not isinstance(content, str):
|
||||||
|
await self.push_frame(frame, direction)
|
||||||
|
return
|
||||||
|
|
||||||
await self._ainvoke(text.strip())
|
await self._ainvoke(content.strip())
|
||||||
else:
|
else:
|
||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
@@ -87,7 +98,10 @@ class LangchainProcessor(FrameProcessor):
|
|||||||
case str():
|
case str():
|
||||||
return text
|
return text
|
||||||
case AIMessageChunk():
|
case AIMessageChunk():
|
||||||
return text.content
|
# `content` is `str | list[...]` (multi-modal); stringify if
|
||||||
|
# it's a list, since downstream consumers want plain text.
|
||||||
|
content = text.content
|
||||||
|
return content if isinstance(content, str) else str(content)
|
||||||
case _:
|
case _:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|||||||
@@ -102,7 +102,7 @@ class RTVIProcessor(FrameProcessor):
|
|||||||
self._client_ready = True
|
self._client_ready = True
|
||||||
await self._call_event_handler("on_client_ready")
|
await self._call_event_handler("on_client_ready")
|
||||||
|
|
||||||
async def set_bot_ready(self, about: Mapping[str, Any] = None):
|
async def set_bot_ready(self, about: Mapping[str, Any] | None = None):
|
||||||
"""Mark the bot as ready and send the bot-ready message.
|
"""Mark the bot as ready and send the bot-ready message.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -404,7 +404,7 @@ class RTVIProcessor(FrameProcessor):
|
|||||||
)
|
)
|
||||||
await self.push_frame(frame)
|
await self.push_frame(frame)
|
||||||
|
|
||||||
async def _send_bot_ready(self, about: Mapping[str, Any] = None):
|
async def _send_bot_ready(self, about: Mapping[str, Any] | None = None):
|
||||||
"""Send the bot-ready message to the client.
|
"""Send the bot-ready message to the client.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
@@ -71,9 +71,15 @@ class StrandsAgentsProcessor(FrameProcessor):
|
|||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
if isinstance(frame, LLMContextFrame):
|
if isinstance(frame, LLMContextFrame):
|
||||||
messages = frame.context.get_messages()
|
messages = frame.context.get_messages()
|
||||||
if messages:
|
# Historically this processor has only handled plain-text user
|
||||||
last_message = messages[-1]
|
# messages; the guards below make that contract explicit for the
|
||||||
await self._ainvoke(str(last_message["content"]).strip())
|
# type checker. TODO: handle other message shapes (provider-
|
||||||
|
# specific messages, multi-modal content lists, etc.).
|
||||||
|
last_message = messages[-1] if messages else None
|
||||||
|
if isinstance(last_message, dict):
|
||||||
|
content = last_message.get("content")
|
||||||
|
if isinstance(content, str):
|
||||||
|
await self._ainvoke(content.strip())
|
||||||
else:
|
else:
|
||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
@@ -91,6 +97,9 @@ class StrandsAgentsProcessor(FrameProcessor):
|
|||||||
await self.start_ttfb_metrics()
|
await self.start_ttfb_metrics()
|
||||||
|
|
||||||
if self.graph:
|
if self.graph:
|
||||||
|
# `__init__` asserts `graph_exit_node` is set whenever `graph`
|
||||||
|
# is, so this can't be None here.
|
||||||
|
assert self.graph_exit_node is not None
|
||||||
# Graph does not stream; await full result then emit assistant text
|
# Graph does not stream; await full result then emit assistant text
|
||||||
graph_result = await self.graph.invoke_async(text)
|
graph_result = await self.graph.invoke_async(text)
|
||||||
if ttfb_tracking:
|
if ttfb_tracking:
|
||||||
@@ -115,6 +124,9 @@ class StrandsAgentsProcessor(FrameProcessor):
|
|||||||
except Exception as parse_err:
|
except Exception as parse_err:
|
||||||
logger.warning(f"Failed to extract messages from GraphResult: {parse_err}")
|
logger.warning(f"Failed to extract messages from GraphResult: {parse_err}")
|
||||||
else:
|
else:
|
||||||
|
# `__init__` asserts at least one of `agent`/`graph` is set,
|
||||||
|
# and we're in the `not self.graph` branch.
|
||||||
|
assert self.agent is not None
|
||||||
# Agent supports streaming events via async iterator
|
# Agent supports streaming events via async iterator
|
||||||
async for event in self.agent.stream_async(text):
|
async for event in self.agent.stream_async(text):
|
||||||
# Push to TTS service
|
# Push to TTS service
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ class AnthropicLLMSettings(LLMSettings):
|
|||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
|
||||||
class AnthropicLLMService(LLMService):
|
class AnthropicLLMService(LLMService[AnthropicLLMAdapter]):
|
||||||
"""LLM service for Anthropic's Claude models.
|
"""LLM service for Anthropic's Claude models.
|
||||||
|
|
||||||
Provides inference capabilities with Claude models including support for
|
Provides inference capabilities with Claude models including support for
|
||||||
@@ -293,7 +293,7 @@ class AnthropicLLMService(LLMService):
|
|||||||
effective_instruction = system_instruction or assert_given(
|
effective_instruction = system_instruction or assert_given(
|
||||||
self._settings.system_instruction
|
self._settings.system_instruction
|
||||||
)
|
)
|
||||||
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
invocation_params = adapter.get_llm_invocation_params(
|
invocation_params = adapter.get_llm_invocation_params(
|
||||||
context,
|
context,
|
||||||
enable_prompt_caching=assert_given(self._settings.enable_prompt_caching),
|
enable_prompt_caching=assert_given(self._settings.enable_prompt_caching),
|
||||||
@@ -328,8 +328,8 @@ class AnthropicLLMService(LLMService):
|
|||||||
return next((block.text for block in response.content if hasattr(block, "text")), None)
|
return next((block.text for block in response.content if hasattr(block, "text")), None)
|
||||||
|
|
||||||
def _get_llm_invocation_params(self, context: LLMContext) -> AnthropicLLMInvocationParams:
|
def _get_llm_invocation_params(self, context: LLMContext) -> AnthropicLLMInvocationParams:
|
||||||
adapter: AnthropicLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
params: AnthropicLLMInvocationParams = adapter.get_llm_invocation_params(
|
params = adapter.get_llm_invocation_params(
|
||||||
context,
|
context,
|
||||||
enable_prompt_caching=assert_given(self._settings.enable_prompt_caching),
|
enable_prompt_caching=assert_given(self._settings.enable_prompt_caching),
|
||||||
system_instruction=assert_given(self._settings.system_instruction),
|
system_instruction=assert_given(self._settings.system_instruction),
|
||||||
|
|||||||
@@ -233,10 +233,11 @@ class AssemblyAISTTService(WebsocketSTTService):
|
|||||||
sample_rate = connection_params.sample_rate
|
sample_rate = connection_params.sample_rate
|
||||||
encoding = connection_params.encoding
|
encoding = connection_params.encoding
|
||||||
default_settings.model = connection_params.speech_model
|
default_settings.model = connection_params.speech_model
|
||||||
default_settings.formatted_finals = connection_params.formatted_finals
|
# Note: `formatted_finals` and `word_finalization_max_wait_time`
|
||||||
default_settings.word_finalization_max_wait_time = (
|
# were added to Settings after this deprecated input model
|
||||||
connection_params.word_finalization_max_wait_time
|
# was frozen and have no equivalent on
|
||||||
)
|
# AssemblyAIConnectionParams; they are only configurable via
|
||||||
|
# the canonical `settings=...` API.
|
||||||
default_settings.end_of_turn_confidence_threshold = (
|
default_settings.end_of_turn_confidence_threshold = (
|
||||||
connection_params.end_of_turn_confidence_threshold
|
connection_params.end_of_turn_confidence_threshold
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -42,14 +42,17 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_async_language(language: Language) -> str | None:
|
def language_to_async_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Async language code.
|
"""Convert a Language enum to Async language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Async language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.EN: "en",
|
Language.EN: "en",
|
||||||
|
|||||||
@@ -201,7 +201,11 @@ class AWSAgentCoreProcessor(FrameProcessor):
|
|||||||
if not payload:
|
if not payload:
|
||||||
return
|
return
|
||||||
|
|
||||||
async with self._aws_session.client("bedrock-agentcore", **self._aws_params) as client:
|
# aioboto3's `client()` is an async context manager but its stubs don't
|
||||||
|
# advertise `__aenter__` / `__aexit__` in a way pyright can see.
|
||||||
|
async with self._aws_session.client( # pyright: ignore[reportGeneralTypeIssues]
|
||||||
|
"bedrock-agentcore", **self._aws_params
|
||||||
|
) as client:
|
||||||
# Invoke the AgentCore agent
|
# Invoke the AgentCore agent
|
||||||
response = await client.invoke_agent_runtime(
|
response = await client.invoke_agent_runtime(
|
||||||
agentRuntimeArn=self._agentArn, payload=payload.encode()
|
agentRuntimeArn=self._agentArn, payload=payload.encode()
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ class AWSBedrockLLMSettings(LLMSettings):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AWSBedrockLLMService(LLMService):
|
class AWSBedrockLLMService(LLMService[AWSBedrockLLMAdapter]):
|
||||||
"""AWS Bedrock Large Language Model service implementation.
|
"""AWS Bedrock Large Language Model service implementation.
|
||||||
|
|
||||||
Provides inference capabilities for AWS Bedrock models including Amazon Nova
|
Provides inference capabilities for AWS Bedrock models including Amazon Nova
|
||||||
@@ -282,8 +282,8 @@ class AWSBedrockLLMService(LLMService):
|
|||||||
effective_instruction = system_instruction or assert_given(
|
effective_instruction = system_instruction or assert_given(
|
||||||
self._settings.system_instruction
|
self._settings.system_instruction
|
||||||
)
|
)
|
||||||
adapter: AWSBedrockLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params(
|
params = adapter.get_llm_invocation_params(
|
||||||
context, system_instruction=effective_instruction
|
context, system_instruction=effective_instruction
|
||||||
)
|
)
|
||||||
messages = params["messages"]
|
messages = params["messages"]
|
||||||
@@ -371,8 +371,8 @@ class AWSBedrockLLMService(LLMService):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def _get_llm_invocation_params(self, context: LLMContext) -> AWSBedrockLLMInvocationParams:
|
def _get_llm_invocation_params(self, context: LLMContext) -> AWSBedrockLLMInvocationParams:
|
||||||
adapter: AWSBedrockLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params(
|
params = adapter.get_llm_invocation_params(
|
||||||
context, system_instruction=assert_given(self._settings.system_instruction)
|
context, system_instruction=assert_given(self._settings.system_instruction)
|
||||||
)
|
)
|
||||||
return params
|
return params
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ from pipecat.frames.frames import (
|
|||||||
UserStartedSpeakingFrame,
|
UserStartedSpeakingFrame,
|
||||||
UserStoppedSpeakingFrame,
|
UserStoppedSpeakingFrame,
|
||||||
)
|
)
|
||||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
|
||||||
from pipecat.processors.frame_processor import FrameDirection
|
from pipecat.processors.frame_processor import FrameDirection
|
||||||
from pipecat.services.aws.nova_sonic.session_continuation import (
|
from pipecat.services.aws.nova_sonic.session_continuation import (
|
||||||
SessionContinuationHelper,
|
SessionContinuationHelper,
|
||||||
@@ -235,7 +235,7 @@ class AWSNovaSonicLLMSettings(LLMSettings):
|
|||||||
endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
|
endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
|
||||||
|
|
||||||
|
|
||||||
class AWSNovaSonicLLMService(LLMService):
|
class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]):
|
||||||
"""AWS Nova Sonic speech-to-speech LLM service.
|
"""AWS Nova Sonic speech-to-speech LLM service.
|
||||||
|
|
||||||
Provides bidirectional audio streaming, real-time transcription, text generation,
|
Provides bidirectional audio streaming, real-time transcription, text generation,
|
||||||
@@ -501,12 +501,18 @@ class AWSNovaSonicLLMService(LLMService):
|
|||||||
service, and reconnects with the preserved context.
|
service, and reconnects with the preserved context.
|
||||||
"""
|
"""
|
||||||
logger.debug("Resetting conversation")
|
logger.debug("Resetting conversation")
|
||||||
if self._assistant_is_responding:
|
|
||||||
self._assistant_is_responding = False
|
|
||||||
await self._report_assistant_response_ended()
|
|
||||||
|
|
||||||
# Grab context to carry through disconnect/reconnect
|
# Grab context to carry through disconnect/reconnect
|
||||||
context = self._context
|
context = self._context
|
||||||
|
if context is None:
|
||||||
|
logger.warning(
|
||||||
|
"reset_conversation called before an initial context was received; nothing to reset"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
if self._assistant_is_responding:
|
||||||
|
self._assistant_is_responding = False
|
||||||
|
await self._report_assistant_response_ended()
|
||||||
|
|
||||||
await self._disconnect()
|
await self._disconnect()
|
||||||
await self._start_connecting()
|
await self._start_connecting()
|
||||||
@@ -606,9 +612,18 @@ class AWSNovaSonicLLMService(LLMService):
|
|||||||
await self._disconnect()
|
await self._disconnect()
|
||||||
|
|
||||||
async def _process_completed_function_calls(self, send_new_results: bool):
|
async def _process_completed_function_calls(self, send_new_results: bool):
|
||||||
|
if not self._context: # should never happen
|
||||||
|
return
|
||||||
# Check for set of completed function calls in the context
|
# Check for set of completed function calls in the context
|
||||||
for message in self._context.get_messages():
|
for message in self._context.get_messages():
|
||||||
if message.get("role") and message.get("content") not in ["IN_PROGRESS", "CANCELLED"]:
|
# LLMSpecificMessages are opaque provider-specific payloads, not
|
||||||
|
# standard tool-result messages — skip them.
|
||||||
|
if isinstance(message, LLMSpecificMessage):
|
||||||
|
continue
|
||||||
|
if message.get("role") == "tool" and message.get("content") not in [
|
||||||
|
"IN_PROGRESS",
|
||||||
|
"CANCELLED",
|
||||||
|
]:
|
||||||
tool_call_id = message.get("tool_call_id")
|
tool_call_id = message.get("tool_call_id")
|
||||||
if tool_call_id and tool_call_id not in self._completed_tool_calls:
|
if tool_call_id and tool_call_id not in self._completed_tool_calls:
|
||||||
# Found a newly-completed function call - send the result to the service
|
# Found a newly-completed function call - send the result to the service
|
||||||
@@ -629,7 +644,7 @@ class AWSNovaSonicLLMService(LLMService):
|
|||||||
await self._process_completed_function_calls(send_new_results=False)
|
await self._process_completed_function_calls(send_new_results=False)
|
||||||
|
|
||||||
# Read context
|
# Read context
|
||||||
adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
llm_connection_params = adapter.get_llm_invocation_params(
|
llm_connection_params = adapter.get_llm_invocation_params(
|
||||||
self._context, system_instruction=assert_given(self._settings.system_instruction)
|
self._context, system_instruction=assert_given(self._settings.system_instruction)
|
||||||
)
|
)
|
||||||
@@ -639,7 +654,7 @@ class AWSNovaSonicLLMService(LLMService):
|
|||||||
tools = (
|
tools = (
|
||||||
llm_connection_params["tools"]
|
llm_connection_params["tools"]
|
||||||
if llm_connection_params["tools"]
|
if llm_connection_params["tools"]
|
||||||
else adapter.from_standard_tools(self._tools)
|
else (adapter.from_standard_tools(self._tools) or [])
|
||||||
)
|
)
|
||||||
logger.debug(f"Using tools: {tools}")
|
logger.debug(f"Using tools: {tools}")
|
||||||
await self._send_prompt_start_event(tools)
|
await self._send_prompt_start_event(tools)
|
||||||
@@ -959,7 +974,9 @@ class AWSNovaSonicLLMService(LLMService):
|
|||||||
async def open_stream(self, client):
|
async def open_stream(self, client):
|
||||||
"""Open a bidirectional stream on the given client."""
|
"""Open a bidirectional stream on the given client."""
|
||||||
return await client.invoke_model_with_bidirectional_stream(
|
return await client.invoke_model_with_bidirectional_stream(
|
||||||
InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model)
|
InvokeModelWithBidirectionalStreamOperationInput(
|
||||||
|
model_id=assert_given(self._settings.model)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
async def send_event(self, event_json: str, stream):
|
async def send_event(self, event_json: str, stream):
|
||||||
@@ -1106,16 +1123,18 @@ class AWSNovaSonicLLMService(LLMService):
|
|||||||
'''
|
'''
|
||||||
await self.send_event(event_json, stream)
|
await self.send_event(event_json, stream)
|
||||||
|
|
||||||
def get_setup_params(self):
|
def get_setup_params(self) -> tuple[str | None, list]:
|
||||||
"""Return ``(system_instruction, tools)`` for the next session setup."""
|
"""Return ``(system_instruction, tools)`` for the next session setup."""
|
||||||
if not self._context:
|
if not self._context:
|
||||||
return None, []
|
return None, []
|
||||||
adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
llm_params = adapter.get_llm_invocation_params(
|
llm_params = adapter.get_llm_invocation_params(
|
||||||
self._context, system_instruction=self._settings.system_instruction
|
self._context, system_instruction=assert_given(self._settings.system_instruction)
|
||||||
)
|
)
|
||||||
tools = (
|
tools = (
|
||||||
llm_params["tools"] if llm_params["tools"] else adapter.from_standard_tools(self._tools)
|
llm_params["tools"]
|
||||||
|
if llm_params["tools"]
|
||||||
|
else (adapter.from_standard_tools(self._tools) or [])
|
||||||
)
|
)
|
||||||
return llm_params["system_instruction"], tools
|
return llm_params["system_instruction"], tools
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import random
|
|||||||
import string
|
import string
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -38,6 +38,7 @@ from pipecat.utils.time import time_now_iso8601
|
|||||||
from pipecat.utils.tracing.service_decorators import traced_stt
|
from pipecat.utils.tracing.service_decorators import traced_stt
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
from websockets import Subprotocol
|
||||||
from websockets.asyncio.client import connect as websocket_connect
|
from websockets.asyncio.client import connect as websocket_connect
|
||||||
from websockets.protocol import State
|
from websockets.protocol import State
|
||||||
except ModuleNotFoundError as e:
|
except ModuleNotFoundError as e:
|
||||||
@@ -314,7 +315,7 @@ class AWSTranscribeSTTService(WebsocketSTTService):
|
|||||||
self._websocket = await websocket_connect(
|
self._websocket = await websocket_connect(
|
||||||
presigned_url,
|
presigned_url,
|
||||||
additional_headers=additional_headers,
|
additional_headers=additional_headers,
|
||||||
subprotocols=["mqtt"],
|
subprotocols=[Subprotocol("mqtt")],
|
||||||
ping_interval=None,
|
ping_interval=None,
|
||||||
ping_timeout=None,
|
ping_timeout=None,
|
||||||
compression=None,
|
compression=None,
|
||||||
@@ -534,7 +535,11 @@ class AWSTranscribeSTTService(WebsocketSTTService):
|
|||||||
is_final = not result.get("IsPartial", True)
|
is_final = not result.get("IsPartial", True)
|
||||||
|
|
||||||
if transcript:
|
if transcript:
|
||||||
language = assert_given(self._settings.language)
|
# Technically `_settings.language` could be a raw string, but
|
||||||
|
# Language is a StrEnum so downstream handles either.
|
||||||
|
language = cast(
|
||||||
|
"Language | None", assert_given(self._settings.language)
|
||||||
|
)
|
||||||
if is_final:
|
if is_final:
|
||||||
await self.push_frame(
|
await self.push_frame(
|
||||||
TranscriptionFrame(
|
TranscriptionFrame(
|
||||||
|
|||||||
@@ -37,14 +37,16 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_aws_language(language: Language) -> str | None:
|
def language_to_aws_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to AWS Polly language code.
|
"""Convert a Language enum to AWS Polly language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding AWS Polly language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Arabic
|
# Arabic
|
||||||
@@ -343,7 +345,11 @@ class AWSPollyTTSService(TTSService):
|
|||||||
# Filter out None values
|
# Filter out None values
|
||||||
filtered_params = {k: v for k, v in params.items() if v is not None}
|
filtered_params = {k: v for k, v in params.items() if v is not None}
|
||||||
|
|
||||||
async with self._aws_session.client("polly", **self._aws_params) as polly:
|
# aioboto3's `client()` is an async context manager but its stubs
|
||||||
|
# don't advertise `__aenter__` / `__aexit__` to pyright.
|
||||||
|
async with self._aws_session.client( # pyright: ignore[reportGeneralTypeIssues]
|
||||||
|
"polly", **self._aws_params
|
||||||
|
) as polly:
|
||||||
response = await polly.synthesize_speech(**filtered_params)
|
response = await polly.synthesize_speech(**filtered_params)
|
||||||
if "AudioStream" in response:
|
if "AudioStream" in response:
|
||||||
# Get the streaming body and read it
|
# Get the streaming body and read it
|
||||||
@@ -351,7 +357,7 @@ class AWSPollyTTSService(TTSService):
|
|||||||
audio_data = await stream.read()
|
audio_data = await stream.read()
|
||||||
else:
|
else:
|
||||||
logger.error(f"{self} No audio stream in response")
|
logger.error(f"{self} No audio stream in response")
|
||||||
audio_data = None
|
return
|
||||||
|
|
||||||
audio_data = await self._resampler.resample(audio_data, 16000, self.sample_rate)
|
audio_data = await self._resampler.resample(audio_data, 16000, self.sample_rate)
|
||||||
|
|
||||||
|
|||||||
@@ -92,14 +92,18 @@ class AWSTranscribePresignedURL:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1"
|
self,
|
||||||
|
access_key: str,
|
||||||
|
secret_key: str,
|
||||||
|
session_token: str | None,
|
||||||
|
region: str = "us-east-1",
|
||||||
):
|
):
|
||||||
"""Initialize the presigned URL generator.
|
"""Initialize the presigned URL generator.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
access_key: AWS access key ID.
|
access_key: AWS access key ID.
|
||||||
secret_key: AWS secret access key.
|
secret_key: AWS secret access key.
|
||||||
session_token: AWS session token for temporary credentials.
|
session_token: AWS session token for temporary credentials (optional).
|
||||||
region: AWS region for the service. Defaults to "us-east-1".
|
region: AWS region for the service. Defaults to "us-east-1".
|
||||||
"""
|
"""
|
||||||
self.access_key = access_key
|
self.access_key = access_key
|
||||||
@@ -129,8 +133,8 @@ class AWSTranscribePresignedURL:
|
|||||||
sample_rate: int,
|
sample_rate: int,
|
||||||
language_code: str = "",
|
language_code: str = "",
|
||||||
media_encoding: str = "pcm",
|
media_encoding: str = "pcm",
|
||||||
vocabulary_name: str = "",
|
vocabulary_name: str | None = None,
|
||||||
vocabulary_filter_name: str = "",
|
vocabulary_filter_name: str | None = None,
|
||||||
show_speaker_label: bool = False,
|
show_speaker_label: bool = False,
|
||||||
enable_channel_identification: bool = False,
|
enable_channel_identification: bool = False,
|
||||||
number_of_channels: int = 1,
|
number_of_channels: int = 1,
|
||||||
|
|||||||
@@ -9,14 +9,16 @@
|
|||||||
from pipecat.transcriptions.language import Language, resolve_language
|
from pipecat.transcriptions.language import Language, resolve_language
|
||||||
|
|
||||||
|
|
||||||
def language_to_azure_language(language: Language) -> str | None:
|
def language_to_azure_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Azure language code.
|
"""Convert a Language enum to Azure language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Azure language code, or None if not supported.
|
The corresponding Azure language code. If ``language`` is not in the
|
||||||
|
verified mapping, falls back to the full language code string and logs
|
||||||
|
a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Afrikaans
|
# Afrikaans
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ Speech SDK for real-time audio transcription.
|
|||||||
import asyncio
|
import asyncio
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -182,9 +182,9 @@ class AzureSTTService(STTService):
|
|||||||
changed = await super()._update_settings(delta)
|
changed = await super()._update_settings(delta)
|
||||||
|
|
||||||
if "language" in changed:
|
if "language" in changed:
|
||||||
self._speech_config.speech_recognition_language = (
|
self._speech_config.speech_recognition_language = assert_given(
|
||||||
self._settings.language or language_to_azure_language(Language.EN_US)
|
self._settings.language
|
||||||
)
|
) or language_to_azure_language(Language.EN_US)
|
||||||
if self._audio_stream:
|
if self._audio_stream:
|
||||||
await self._disconnect()
|
await self._disconnect()
|
||||||
await self._connect()
|
await self._connect()
|
||||||
@@ -280,8 +280,11 @@ class AzureSTTService(STTService):
|
|||||||
|
|
||||||
def _on_handle_recognized(self, event):
|
def _on_handle_recognized(self, event):
|
||||||
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
|
if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
|
||||||
language = getattr(event.result, "language", None) or assert_given(
|
# Technically either source could be a raw string, but Language is
|
||||||
self._settings.language
|
# a StrEnum so downstream handles either.
|
||||||
|
language = cast(
|
||||||
|
"Language | None",
|
||||||
|
getattr(event.result, "language", None) or assert_given(self._settings.language),
|
||||||
)
|
)
|
||||||
frame = TranscriptionFrame(
|
frame = TranscriptionFrame(
|
||||||
event.result.text,
|
event.result.text,
|
||||||
@@ -297,8 +300,11 @@ class AzureSTTService(STTService):
|
|||||||
|
|
||||||
def _on_handle_recognizing(self, event):
|
def _on_handle_recognizing(self, event):
|
||||||
if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0:
|
if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0:
|
||||||
language = getattr(event.result, "language", None) or assert_given(
|
# Technically either source could be a raw string, but Language is
|
||||||
self._settings.language
|
# a StrEnum so downstream handles either.
|
||||||
|
language = cast(
|
||||||
|
"Language | None",
|
||||||
|
getattr(event.result, "language", None) or assert_given(self._settings.language),
|
||||||
)
|
)
|
||||||
frame = InterimTranscriptionFrame(
|
frame = InterimTranscriptionFrame(
|
||||||
event.result.text,
|
event.result.text,
|
||||||
|
|||||||
@@ -44,14 +44,17 @@ MODEL_SAMPLE_RATES: dict[str, int] = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def language_to_camb_language(language: Language) -> str | None:
|
def language_to_camb_language(language: Language) -> str:
|
||||||
"""Convert a Pipecat Language enum to Camb.ai language code.
|
"""Convert a Pipecat Language enum to Camb.ai language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Camb.ai language code (BCP-47 format), or None if not supported.
|
The corresponding Camb.ai language code (BCP-47 format). If
|
||||||
|
``language`` is not in the verified mapping, falls back to the base
|
||||||
|
language code (e.g., ``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.EN: "en-us",
|
Language.EN: "en-us",
|
||||||
|
|||||||
@@ -290,6 +290,11 @@ class CartesiaSTTService(WebsocketSTTService):
|
|||||||
if not self._websocket or self._websocket.state is not State.OPEN:
|
if not self._websocket or self._websocket.state is not State.OPEN:
|
||||||
await self._connect()
|
await self._connect()
|
||||||
|
|
||||||
|
if self._websocket is None:
|
||||||
|
logger.warning(f"{self}: websocket unavailable after reconnect, dropping audio")
|
||||||
|
yield None
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await self._websocket.send(audio)
|
await self._websocket.send(audio)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -62,14 +62,17 @@ class GenerationConfig(BaseModel):
|
|||||||
emotion: str | None = None
|
emotion: str | None = None
|
||||||
|
|
||||||
|
|
||||||
def language_to_cartesia_language(language: Language) -> str | None:
|
def language_to_cartesia_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Cartesia language code.
|
"""Convert a Language enum to Cartesia language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Cartesia language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AR: "ar",
|
Language.AR: "ar",
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ from pipecat.utils.time import time_now_iso8601
|
|||||||
from pipecat.utils.tracing.service_decorators import traced_stt
|
from pipecat.utils.tracing.service_decorators import traced_stt
|
||||||
|
|
||||||
|
|
||||||
def language_to_deepgram_flux_language(language: Language) -> str | None:
|
def language_to_deepgram_flux_language(language: Language) -> str:
|
||||||
"""Convert a Pipecat Language to a Deepgram Flux language code.
|
"""Convert a Pipecat Language to a Deepgram Flux language code.
|
||||||
|
|
||||||
Only honored by the ``flux-general-multi`` model. Locale variants
|
Only honored by the ``flux-general-multi`` model. Locale variants
|
||||||
@@ -253,7 +253,7 @@ class DeepgramFluxSTTBase(STTService):
|
|||||||
params.append(f"mip_opt_out={str(self._mip_opt_out).lower()}")
|
params.append(f"mip_opt_out={str(self._mip_opt_out).lower()}")
|
||||||
|
|
||||||
# Add keyterm parameters (can have multiple)
|
# Add keyterm parameters (can have multiple)
|
||||||
for keyterm in self._settings.keyterm:
|
for keyterm in assert_given(self._settings.keyterm):
|
||||||
params.append(urlencode({"keyterm": keyterm}))
|
params.append(urlencode({"keyterm": keyterm}))
|
||||||
|
|
||||||
# Add tag parameters (can have multiple)
|
# Add tag parameters (can have multiple)
|
||||||
@@ -536,6 +536,10 @@ class DeepgramFluxSTTBase(STTService):
|
|||||||
event = data.get("event")
|
event = data.get("event")
|
||||||
transcript = data.get("transcript", "")
|
transcript = data.get("transcript", "")
|
||||||
|
|
||||||
|
if not isinstance(event, str):
|
||||||
|
logger.debug(f"Unhandled TurnInfo event (not a string): {event}")
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
flux_event_type = FluxEventType(event)
|
flux_event_type = FluxEventType(event)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
@@ -648,7 +652,11 @@ class DeepgramFluxSTTBase(STTService):
|
|||||||
detected_language = self._primary_detected_language(data)
|
detected_language = self._primary_detected_language(data)
|
||||||
|
|
||||||
min_confidence = assert_given(self._settings.min_confidence)
|
min_confidence = assert_given(self._settings.min_confidence)
|
||||||
if not min_confidence or average_confidence > min_confidence:
|
# No threshold (None or 0.0) → accept. Otherwise require confidence
|
||||||
|
# data and compare; drop if data is missing.
|
||||||
|
if not min_confidence or (
|
||||||
|
average_confidence is not None and average_confidence > min_confidence
|
||||||
|
):
|
||||||
# EndOfTurn means Flux has determined the turn is complete,
|
# EndOfTurn means Flux has determined the turn is complete,
|
||||||
# so this TranscriptionFrame is always finalized
|
# so this TranscriptionFrame is always finalized
|
||||||
await self.push_frame(
|
await self.push_frame(
|
||||||
|
|||||||
@@ -145,9 +145,17 @@ class DeepgramFluxSageMakerSTTService(DeepgramFluxSTTBase):
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
async def _transport_send_audio(self, audio: bytes):
|
async def _transport_send_audio(self, audio: bytes):
|
||||||
|
if (
|
||||||
|
self._client is None
|
||||||
|
): # should never happen — caller should gate on _transport_is_active()
|
||||||
|
return
|
||||||
await self._client.send_audio_chunk(audio)
|
await self._client.send_audio_chunk(audio)
|
||||||
|
|
||||||
async def _transport_send_json(self, message: dict):
|
async def _transport_send_json(self, message: dict):
|
||||||
|
if (
|
||||||
|
self._client is None
|
||||||
|
): # should never happen — caller should gate on _transport_is_active()
|
||||||
|
return
|
||||||
await self._client.send_json(message)
|
await self._client.send_json(message)
|
||||||
|
|
||||||
def _transport_is_active(self) -> bool:
|
def _transport_is_active(self) -> bool:
|
||||||
|
|||||||
@@ -240,9 +240,17 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService):
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
async def _transport_send_audio(self, audio: bytes):
|
async def _transport_send_audio(self, audio: bytes):
|
||||||
|
if (
|
||||||
|
self._websocket is None
|
||||||
|
): # should never happen — caller should gate on _transport_is_active()
|
||||||
|
return
|
||||||
await self._websocket.send(audio)
|
await self._websocket.send(audio)
|
||||||
|
|
||||||
async def _transport_send_json(self, message: dict):
|
async def _transport_send_json(self, message: dict):
|
||||||
|
if (
|
||||||
|
self._websocket is None
|
||||||
|
): # should never happen — caller should gate on _transport_is_active()
|
||||||
|
return
|
||||||
await self._websocket.send(json.dumps(message))
|
await self._websocket.send(json.dumps(message))
|
||||||
|
|
||||||
def _transport_is_active(self) -> bool:
|
def _transport_is_active(self) -> bool:
|
||||||
@@ -291,14 +299,19 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService):
|
|||||||
|
|
||||||
self._connection_established_event.clear()
|
self._connection_established_event.clear()
|
||||||
self._user_is_speaking = False
|
self._user_is_speaking = False
|
||||||
self._websocket = await websocket_connect(
|
# `_connect` sets `_websocket_url` before calling us; the assert
|
||||||
|
# narrows for pyright.
|
||||||
|
assert self._websocket_url is not None
|
||||||
|
websocket = await websocket_connect(
|
||||||
self._websocket_url,
|
self._websocket_url,
|
||||||
additional_headers={"Authorization": f"Token {self._api_key}"},
|
additional_headers={"Authorization": f"Token {self._api_key}"},
|
||||||
)
|
)
|
||||||
|
self._websocket = websocket
|
||||||
|
|
||||||
headers = {
|
# `response` is populated after the handshake completes (which it
|
||||||
k: v for k, v in self._websocket.response.headers.items() if k.startswith("dg-")
|
# has, since `websocket_connect` already returned).
|
||||||
}
|
response_headers = websocket.response.headers if websocket.response else {}
|
||||||
|
headers = {k: v for k, v in response_headers.items() if k.startswith("dg-")}
|
||||||
logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}')
|
logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}')
|
||||||
|
|
||||||
# Creating the receiver task
|
# Creating the receiver task
|
||||||
|
|||||||
@@ -337,6 +337,10 @@ class DeepgramSageMakerTTSService(TTSService):
|
|||||||
the response processor).
|
the response processor).
|
||||||
"""
|
"""
|
||||||
logger.debug(f"{self}: Generating TTS [{text}]")
|
logger.debug(f"{self}: Generating TTS [{text}]")
|
||||||
|
if self._client is None:
|
||||||
|
logger.warning(f"{self}: client unavailable, skipping TTS")
|
||||||
|
yield ErrorFrame(error="client unavailable")
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
await self._client.send_json({"type": "Speak", "text": text})
|
await self._client.send_json({"type": "Speak", "text": text})
|
||||||
yield None
|
yield None
|
||||||
|
|||||||
@@ -232,17 +232,19 @@ class DeepgramTTSService(WebsocketTTSService):
|
|||||||
|
|
||||||
headers = {"Authorization": f"Token {self._api_key}"}
|
headers = {"Authorization": f"Token {self._api_key}"}
|
||||||
|
|
||||||
self._websocket = await websocket_connect(url, additional_headers=headers)
|
websocket = await websocket_connect(url, additional_headers=headers)
|
||||||
|
self._websocket = websocket
|
||||||
|
|
||||||
headers = {
|
# `response` is populated after the handshake completes (which it
|
||||||
k: v for k, v in self._websocket.response.headers.items() if k.startswith("dg-")
|
# has, since `websocket_connect` already returned).
|
||||||
}
|
response_headers = websocket.response.headers if websocket.response else {}
|
||||||
|
headers = {k: v for k, v in response_headers.items() if k.startswith("dg-")}
|
||||||
logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}')
|
logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}')
|
||||||
|
|
||||||
await self._call_event_handler("on_connected")
|
await self._call_event_handler("on_connected")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"{self} exception: {e}")
|
logger.error(f"{self} exception: {e}")
|
||||||
await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
|
await self.push_error_frame(ErrorFrame(error=f"{self} error: {e}"))
|
||||||
self._websocket = None
|
self._websocket = None
|
||||||
await self._call_event_handler("on_connection_error", f"{e}")
|
await self._call_event_handler("on_connection_error", f"{e}")
|
||||||
|
|
||||||
@@ -258,7 +260,7 @@ class DeepgramTTSService(WebsocketTTSService):
|
|||||||
await self._websocket.close()
|
await self._websocket.close()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"{self} exception: {e}")
|
logger.error(f"{self} exception: {e}")
|
||||||
await self.push_error(ErrorFrame(error=f"{self} error: {e}"))
|
await self.push_error_frame(ErrorFrame(error=f"{self} error: {e}"))
|
||||||
finally:
|
finally:
|
||||||
self._websocket = None
|
self._websocket = None
|
||||||
await self._call_event_handler("on_disconnected")
|
await self._call_event_handler("on_disconnected")
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_elevenlabs_language(language: Language) -> str | None:
|
def language_to_elevenlabs_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to ElevenLabs language code.
|
"""Convert a Language enum to ElevenLabs language code.
|
||||||
|
|
||||||
Source:
|
Source:
|
||||||
@@ -64,7 +64,9 @@ def language_to_elevenlabs_language(language: Language) -> str | None:
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding ElevenLabs language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AF: "afr", # Afrikaans
|
Language.AF: "afr", # Afrikaans
|
||||||
@@ -739,6 +741,10 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
|
|||||||
Args:
|
Args:
|
||||||
silence: Silent 16-bit mono PCM audio bytes.
|
silence: Silent 16-bit mono PCM audio bytes.
|
||||||
"""
|
"""
|
||||||
|
if (
|
||||||
|
self._websocket is None
|
||||||
|
): # should never happen — caller should gate on _is_keepalive_ready()
|
||||||
|
return
|
||||||
audio_base64 = base64.b64encode(silence).decode("utf-8")
|
audio_base64 = base64.b64encode(silence).decode("utf-8")
|
||||||
message = {
|
message = {
|
||||||
"message_type": "input_audio_chunk",
|
"message_type": "input_audio_chunk",
|
||||||
|
|||||||
@@ -69,14 +69,17 @@ ELEVENLABS_MULTILINGUAL_MODELS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def language_to_elevenlabs_language(language: Language) -> str | None:
|
def language_to_elevenlabs_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to ElevenLabs language code.
|
"""Convert a Language enum to ElevenLabs language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding ElevenLabs language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AR: "ar",
|
Language.AR: "ar",
|
||||||
@@ -905,6 +908,11 @@ class ElevenLabsTTSService(WebsocketTTSService):
|
|||||||
if not self._websocket or self._websocket.state is State.CLOSED:
|
if not self._websocket or self._websocket.state is State.CLOSED:
|
||||||
await self._connect()
|
await self._connect()
|
||||||
|
|
||||||
|
if self._websocket is None:
|
||||||
|
logger.warning(f"{self}: websocket unavailable after reconnect, skipping TTS")
|
||||||
|
yield ErrorFrame(error="websocket unavailable")
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not self.audio_context_available(context_id):
|
if not self.audio_context_available(context_id):
|
||||||
await self.create_audio_context(context_id)
|
await self.create_audio_context(context_id)
|
||||||
@@ -915,7 +923,7 @@ class ElevenLabsTTSService(WebsocketTTSService):
|
|||||||
self._partial_word_start_time = 0.0
|
self._partial_word_start_time = 0.0
|
||||||
|
|
||||||
# Initialize context with voice settings and pronunciation dictionaries
|
# Initialize context with voice settings and pronunciation dictionaries
|
||||||
msg = {"text": " ", "context_id": context_id}
|
msg: dict[str, Any] = {"text": " ", "context_id": context_id}
|
||||||
if self._voice_settings:
|
if self._voice_settings:
|
||||||
msg["voice_settings"] = self._voice_settings
|
msg["voice_settings"] = self._voice_settings
|
||||||
if self._pronunciation_dictionary_locators:
|
if self._pronunciation_dictionary_locators:
|
||||||
@@ -1260,7 +1268,7 @@ class ElevenLabsHttpTTSService(TTSService):
|
|||||||
url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps"
|
url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps"
|
||||||
|
|
||||||
model_id = assert_given(self._settings.model)
|
model_id = assert_given(self._settings.model)
|
||||||
payload: dict[str, str | dict[str, float | bool]] = {
|
payload: dict[str, Any] = {
|
||||||
"text": text,
|
"text": text,
|
||||||
"model_id": model_id,
|
"model_id": model_id,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,14 +28,17 @@ from pipecat.utils.time import time_now_iso8601
|
|||||||
from pipecat.utils.tracing.service_decorators import traced_stt
|
from pipecat.utils.tracing.service_decorators import traced_stt
|
||||||
|
|
||||||
|
|
||||||
def language_to_fal_language(language: Language) -> str | None:
|
def language_to_fal_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Fal's Wizper language code.
|
"""Convert a Language enum to Fal's Wizper language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Fal Wizper language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AF: "af",
|
Language.AF: "af",
|
||||||
|
|||||||
@@ -281,7 +281,8 @@ class FishAudioTTSService(InterruptibleTTSService):
|
|||||||
model = assert_given(self._settings.model)
|
model = assert_given(self._settings.model)
|
||||||
if model is not None:
|
if model is not None:
|
||||||
headers["model"] = model
|
headers["model"] = model
|
||||||
self._websocket = await websocket_connect(self._base_url, additional_headers=headers)
|
websocket = await websocket_connect(self._base_url, additional_headers=headers)
|
||||||
|
self._websocket = websocket
|
||||||
|
|
||||||
# Send initial start message with ormsgpack
|
# Send initial start message with ormsgpack
|
||||||
request_settings = {
|
request_settings = {
|
||||||
@@ -300,7 +301,7 @@ class FishAudioTTSService(InterruptibleTTSService):
|
|||||||
if self._settings.top_p is not None:
|
if self._settings.top_p is not None:
|
||||||
request_settings["top_p"] = self._settings.top_p
|
request_settings["top_p"] = self._settings.top_p
|
||||||
start_message = {"event": "start", "request": {"text": "", **request_settings}}
|
start_message = {"event": "start", "request": {"text": "", **request_settings}}
|
||||||
await self._websocket.send(ormsgpack.packb(start_message))
|
await websocket.send(ormsgpack.packb(start_message))
|
||||||
logger.debug("Sent start message to Fish Audio")
|
logger.debug("Sent start message to Fish Audio")
|
||||||
|
|
||||||
await self._call_event_handler("on_connected")
|
await self._call_event_handler("on_connected")
|
||||||
|
|||||||
@@ -56,14 +56,17 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_gladia_language(language: Language) -> str | None:
|
def language_to_gladia_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Gladia's language code format.
|
"""Convert a Language enum to Gladia's language code format.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The Gladia language code string or None if not supported.
|
The corresponding Gladia language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AF: "af",
|
Language.AF: "af",
|
||||||
@@ -361,7 +364,7 @@ class GladiaSTTService(WebsocketSTTService):
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The Gladia language code string or None if not supported.
|
The Gladia language code string, or None if not supported.
|
||||||
"""
|
"""
|
||||||
return language_to_gladia_language(language)
|
return language_to_gladia_language(language)
|
||||||
|
|
||||||
@@ -539,6 +542,8 @@ class GladiaSTTService(WebsocketSTTService):
|
|||||||
|
|
||||||
logger.debug(f"{self}Connecting to Gladia WebSocket")
|
logger.debug(f"{self}Connecting to Gladia WebSocket")
|
||||||
|
|
||||||
|
if self._session_url is None:
|
||||||
|
raise RuntimeError(f"{self} session URL is not initialized")
|
||||||
self._websocket = await websocket_connect(self._session_url)
|
self._websocket = await websocket_connect(self._session_url)
|
||||||
self._connection_active = True
|
self._connection_active = True
|
||||||
|
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ MAX_CONSECUTIVE_FAILURES = 3
|
|||||||
CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds
|
CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds
|
||||||
|
|
||||||
|
|
||||||
def language_to_gemini_language(language: Language) -> str | None:
|
def language_to_gemini_language(language: Language) -> str:
|
||||||
"""Maps a Language enum value to a Gemini Live supported language code.
|
"""Maps a Language enum value to a Gemini Live supported language code.
|
||||||
|
|
||||||
Source:
|
Source:
|
||||||
@@ -121,7 +121,9 @@ def language_to_gemini_language(language: Language) -> str | None:
|
|||||||
language: The language enum value to convert.
|
language: The language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The Gemini language code string, or None if the language is not supported.
|
The Gemini language code string. If ``language`` is not in the
|
||||||
|
verified mapping, falls back to the full language code string and logs
|
||||||
|
a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Arabic
|
# Arabic
|
||||||
@@ -351,7 +353,7 @@ class GeminiLiveLLMSettings(LLMSettings):
|
|||||||
proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
|
proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
|
||||||
|
|
||||||
|
|
||||||
class GeminiLiveLLMService(LLMService):
|
class GeminiLiveLLMService(LLMService[GeminiLLMAdapter]):
|
||||||
"""Provides access to Google's Gemini Live API.
|
"""Provides access to Google's Gemini Live API.
|
||||||
|
|
||||||
This service enables real-time conversations with Gemini, supporting both
|
This service enables real-time conversations with Gemini, supporting both
|
||||||
@@ -778,7 +780,7 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
# init-provided values). Note that the determination of "effective"
|
# init-provided values). Note that the determination of "effective"
|
||||||
# system instruction is delegated to the adapter, which still
|
# system instruction is delegated to the adapter, which still
|
||||||
# chooses the init-provided value if there is one.
|
# chooses the init-provided value if there is one.
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
params = adapter.get_llm_invocation_params(
|
params = adapter.get_llm_invocation_params(
|
||||||
self._context, system_instruction=assert_given(self._system_instruction_from_init)
|
self._context, system_instruction=assert_given(self._system_instruction_from_init)
|
||||||
)
|
)
|
||||||
@@ -840,7 +842,7 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
|
|
||||||
async def _process_completed_function_calls(self, send_new_results: bool):
|
async def _process_completed_function_calls(self, send_new_results: bool):
|
||||||
# Check for set of completed function calls in the context
|
# Check for set of completed function calls in the context
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
messages = adapter.get_llm_invocation_params(self._context).get("messages", [])
|
messages = adapter.get_llm_invocation_params(self._context).get("messages", [])
|
||||||
for message in messages:
|
for message in messages:
|
||||||
if message.parts:
|
if message.parts:
|
||||||
@@ -1027,7 +1029,7 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
# Add system instruction and tools to configuration, if provided.
|
# Add system instruction and tools to configuration, if provided.
|
||||||
# These settings from the context take precedence over the ones
|
# These settings from the context take precedence over the ones
|
||||||
# provided at initialization time.
|
# provided at initialization time.
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
system_instruction = None
|
system_instruction = None
|
||||||
tools = None
|
tools = None
|
||||||
if self._context:
|
if self._context:
|
||||||
@@ -1333,7 +1335,7 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
self._run_llm_when_session_ready = True
|
self._run_llm_when_session_ready = True
|
||||||
return
|
return
|
||||||
|
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
messages = adapter.get_llm_invocation_params(self._context).get("messages", [])
|
messages = adapter.get_llm_invocation_params(self._context).get("messages", [])
|
||||||
if not messages:
|
if not messages:
|
||||||
# No messages to seed convo with, so we're ready for realtime input right away
|
# No messages to seed convo with, so we're ready for realtime input right away
|
||||||
@@ -1392,7 +1394,7 @@ class GeminiLiveLLMService(LLMService):
|
|||||||
# Create a throwaway context just for the purpose of getting messages
|
# Create a throwaway context just for the purpose of getting messages
|
||||||
# in the right format
|
# in the right format
|
||||||
context = LLMContext(messages=messages_list)
|
context = LLMContext(messages=messages_list)
|
||||||
adapter: GeminiLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
messages = adapter.get_llm_invocation_params(context).get("messages", [])
|
messages = adapter.get_llm_invocation_params(context).get("messages", [])
|
||||||
|
|
||||||
if not messages:
|
if not messages:
|
||||||
|
|||||||
@@ -174,11 +174,17 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService):
|
|||||||
default_settings.temperature = params.temperature
|
default_settings.temperature = params.temperature
|
||||||
default_settings.top_k = params.top_k
|
default_settings.top_k = params.top_k
|
||||||
default_settings.top_p = params.top_p
|
default_settings.top_p = params.top_p
|
||||||
default_settings.modalities = params.modalities
|
# `params.modalities` and `params.media_resolution` are typed
|
||||||
|
# `<Enum> | None` on the deprecated InputParams, but None isn't
|
||||||
|
# a valid setting value (downstream uses call `.value` on
|
||||||
|
# them). Fall back to the canonical defaults.
|
||||||
|
default_settings.modalities = params.modalities or GeminiModalities.AUDIO
|
||||||
default_settings.language = (
|
default_settings.language = (
|
||||||
language_to_gemini_language(params.language) if params.language else "en-US"
|
language_to_gemini_language(params.language) if params.language else "en-US"
|
||||||
)
|
)
|
||||||
default_settings.media_resolution = params.media_resolution
|
default_settings.media_resolution = (
|
||||||
|
params.media_resolution or GeminiMediaResolution.UNSPECIFIED
|
||||||
|
)
|
||||||
default_settings.vad = params.vad
|
default_settings.vad = params.vad
|
||||||
default_settings.context_window_compression = (
|
default_settings.context_window_compression = (
|
||||||
params.context_window_compression.model_dump()
|
params.context_window_compression.model_dump()
|
||||||
@@ -233,7 +239,9 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_credentials(credentials: str | None, credentials_path: str | None) -> str:
|
def _get_credentials(
|
||||||
|
credentials: str | None, credentials_path: str | None
|
||||||
|
) -> service_account.Credentials:
|
||||||
"""Retrieve Credentials using Google service account credentials JSON.
|
"""Retrieve Credentials using Google service account credentials JSON.
|
||||||
|
|
||||||
Supports multiple authentication methods:
|
Supports multiple authentication methods:
|
||||||
@@ -246,7 +254,8 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService):
|
|||||||
credentials_path: Path to the service account JSON file.
|
credentials_path: Path to the service account JSON file.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
OAuth token for API authentication.
|
A service-account ``Credentials`` object suitable for the Vertex
|
||||||
|
AI client (with its access token refreshed).
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If no valid credentials are provided or found.
|
ValueError: If no valid credentials are provided or found.
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ from pipecat.services.image_service import ImageGenService
|
|||||||
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, assert_given
|
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, assert_given
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from google import genai
|
import google.genai as genai
|
||||||
from google.genai import types
|
from google.genai import types
|
||||||
except ModuleNotFoundError as e:
|
except ModuleNotFoundError as e:
|
||||||
logger.error(f"Exception: {e}")
|
logger.error(f"Exception: {e}")
|
||||||
@@ -153,8 +153,12 @@ class GoogleImageGenService(ImageGenService):
|
|||||||
await self.start_ttfb_metrics()
|
await self.start_ttfb_metrics()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
model = assert_given(self._settings.model)
|
||||||
|
if model is None:
|
||||||
|
yield ErrorFrame("Google image generation model must be specified")
|
||||||
|
return
|
||||||
response = await self._client.aio.models.generate_images(
|
response = await self._client.aio.models.generate_images(
|
||||||
model=self._settings.model,
|
model=model,
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
config=types.GenerateImagesConfig(
|
config=types.GenerateImagesConfig(
|
||||||
number_of_images=assert_given(self._settings.number_of_images),
|
number_of_images=assert_given(self._settings.number_of_images),
|
||||||
@@ -169,8 +173,9 @@ class GoogleImageGenService(ImageGenService):
|
|||||||
|
|
||||||
for img_response in response.generated_images:
|
for img_response in response.generated_images:
|
||||||
# Google returns the image data directly
|
# Google returns the image data directly
|
||||||
image_bytes = img_response.image.image_bytes
|
if img_response.image is None or img_response.image.image_bytes is None:
|
||||||
image = Image.open(io.BytesIO(image_bytes))
|
continue
|
||||||
|
image = Image.open(io.BytesIO(img_response.image.image_bytes))
|
||||||
|
|
||||||
frame = URLImageRawFrame(
|
frame = URLImageRawFrame(
|
||||||
url=None, # Google doesn't provide URLs, only image data
|
url=None, # Google doesn't provide URLs, only image data
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from loguru import logger
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter, GeminiLLMInvocationParams
|
from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
AssistantImageRawFrame,
|
AssistantImageRawFrame,
|
||||||
Frame,
|
Frame,
|
||||||
@@ -52,7 +52,7 @@ from pipecat.utils.tracing.service_decorators import traced_llm
|
|||||||
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
|
os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from google import genai
|
import google.genai as genai
|
||||||
from google.api_core.exceptions import DeadlineExceeded
|
from google.api_core.exceptions import DeadlineExceeded
|
||||||
from google.genai.types import (
|
from google.genai.types import (
|
||||||
GenerateContentConfig,
|
GenerateContentConfig,
|
||||||
@@ -124,7 +124,7 @@ class GoogleLLMSettings(LLMSettings):
|
|||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
|
||||||
class GoogleLLMService(LLMService):
|
class GoogleLLMService(LLMService[GeminiLLMAdapter]):
|
||||||
"""Google AI (Gemini) LLM service implementation.
|
"""Google AI (Gemini) LLM service implementation.
|
||||||
|
|
||||||
This class implements inference with Google's AI models, translating internally
|
This class implements inference with Google's AI models, translating internally
|
||||||
@@ -292,7 +292,7 @@ class GoogleLLMService(LLMService):
|
|||||||
tools = []
|
tools = []
|
||||||
effective_instruction = system_instruction or self._settings.system_instruction
|
effective_instruction = system_instruction or self._settings.system_instruction
|
||||||
adapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
params: GeminiLLMInvocationParams = adapter.get_llm_invocation_params(
|
params = adapter.get_llm_invocation_params(
|
||||||
context, system_instruction=effective_instruction
|
context, system_instruction=effective_instruction
|
||||||
)
|
)
|
||||||
messages = params["messages"]
|
messages = params["messages"]
|
||||||
@@ -387,7 +387,7 @@ class GoogleLLMService(LLMService):
|
|||||||
|
|
||||||
async def _stream_content(self, context: LLMContext) -> AsyncIterator[GenerateContentResponse]:
|
async def _stream_content(self, context: LLMContext) -> AsyncIterator[GenerateContentResponse]:
|
||||||
adapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
params: GeminiLLMInvocationParams = adapter.get_llm_invocation_params(
|
params = adapter.get_llm_invocation_params(
|
||||||
context, system_instruction=assert_given(self._settings.system_instruction)
|
context, system_instruction=assert_given(self._settings.system_instruction)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -60,14 +60,17 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_google_stt_language(language: Language) -> str | None:
|
def language_to_google_stt_language(language: Language) -> str:
|
||||||
"""Maps Language enum to Google Speech-to-Text V2 language codes.
|
"""Maps Language enum to Google Speech-to-Text V2 language codes.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: Language enum value.
|
language: Language enum value.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Optional[str]: Google STT language code or None if not supported.
|
The corresponding Google STT language code. If ``language`` is not
|
||||||
|
in the verified mapping, falls back to the full language code string
|
||||||
|
and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Afrikaans
|
# Afrikaans
|
||||||
@@ -617,17 +620,20 @@ class GoogleSTTService(STTService):
|
|||||||
"""
|
"""
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def language_to_service_language(self, language: Language | list[Language]) -> str | list[str]:
|
def language_to_service_language(self, language: Language) -> str:
|
||||||
"""Convert Language enum(s) to Google STT language code(s).
|
"""Convert a Language enum to a Google STT language code.
|
||||||
|
|
||||||
|
Narrower return type than the base class's ``str | None``: this
|
||||||
|
override always returns a string, falling back to ``"en-US"`` for
|
||||||
|
languages not in the verified mapping (see
|
||||||
|
:func:`language_to_google_stt_language`).
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: Single Language enum or list of Language enums.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str | List[str]: Google STT language code(s).
|
The Google STT language code.
|
||||||
"""
|
"""
|
||||||
if isinstance(language, list):
|
|
||||||
return [language_to_google_stt_language(lang) or "en-US" for lang in language]
|
|
||||||
return language_to_google_stt_language(language) or "en-US"
|
return language_to_google_stt_language(language) or "en-US"
|
||||||
|
|
||||||
def _get_language_codes(self) -> list[str]:
|
def _get_language_codes(self) -> list[str]:
|
||||||
@@ -639,8 +645,9 @@ class GoogleSTTService(STTService):
|
|||||||
Returns:
|
Returns:
|
||||||
List[str]: Google STT language code strings.
|
List[str]: Google STT language code strings.
|
||||||
"""
|
"""
|
||||||
if self._settings.languages:
|
languages = assert_given(self._settings.languages)
|
||||||
return [self.language_to_service_language(lang) for lang in self._settings.languages]
|
if languages:
|
||||||
|
return [self.language_to_service_language(lang) for lang in languages]
|
||||||
language_codes = assert_given(self._settings.language_codes)
|
language_codes = assert_given(self._settings.language_codes)
|
||||||
if language_codes:
|
if language_codes:
|
||||||
return list(language_codes)
|
return list(language_codes)
|
||||||
|
|||||||
@@ -60,7 +60,7 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_google_tts_language(language: Language) -> str | None:
|
def language_to_google_tts_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Google TTS language code.
|
"""Convert a Language enum to Google TTS language code.
|
||||||
|
|
||||||
Source:
|
Source:
|
||||||
@@ -70,7 +70,9 @@ def language_to_google_tts_language(language: Language) -> str | None:
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Google TTS language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Arabic
|
# Arabic
|
||||||
@@ -219,7 +221,7 @@ def language_to_google_tts_language(language: Language) -> str | None:
|
|||||||
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
|
return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
|
||||||
|
|
||||||
|
|
||||||
def language_to_gemini_tts_language(language: Language) -> str | None:
|
def language_to_gemini_tts_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Gemini TTS language code.
|
"""Convert a Language enum to Gemini TTS language code.
|
||||||
|
|
||||||
Source:
|
Source:
|
||||||
@@ -229,7 +231,9 @@ def language_to_gemini_tts_language(language: Language) -> str | None:
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Gemini TTS language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Afrikaans (Preview)
|
# Afrikaans (Preview)
|
||||||
@@ -1413,7 +1417,7 @@ class GeminiTTSService(GoogleBaseTTSService):
|
|||||||
if self._settings.multi_speaker and self._settings.speaker_configs:
|
if self._settings.multi_speaker and self._settings.speaker_configs:
|
||||||
# Multi-speaker mode
|
# Multi-speaker mode
|
||||||
speaker_voice_configs = []
|
speaker_voice_configs = []
|
||||||
for speaker_config in self._settings.speaker_configs:
|
for speaker_config in assert_given(self._settings.speaker_configs):
|
||||||
speaker_voice_configs.append(
|
speaker_voice_configs.append(
|
||||||
texttospeech_v1.MultispeakerPrebuiltVoice(
|
texttospeech_v1.MultispeakerPrebuiltVoice(
|
||||||
speaker_alias=speaker_config["speaker_alias"],
|
speaker_alias=speaker_config["speaker_alias"],
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import base64
|
|||||||
import json
|
import json
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
@@ -79,14 +79,17 @@ def _input_format_from_encoding(encoding: str, sample_rate: int) -> str:
|
|||||||
return encoding
|
return encoding
|
||||||
|
|
||||||
|
|
||||||
def language_to_gradium_language(language: Language) -> str | None:
|
def language_to_gradium_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to Gradium's language code format.
|
"""Convert a Language enum to Gradium's language code format.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The Gradium language code string or None if not supported.
|
The corresponding Gradium language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.DE: "de",
|
Language.DE: "de",
|
||||||
@@ -383,10 +386,11 @@ class GradiumSTTService(WebsocketSTTService):
|
|||||||
"x-api-key": self._api_key,
|
"x-api-key": self._api_key,
|
||||||
"x-api-source": "pipecat",
|
"x-api-source": "pipecat",
|
||||||
}
|
}
|
||||||
self._websocket = await websocket_connect(
|
websocket = await websocket_connect(
|
||||||
ws_url,
|
ws_url,
|
||||||
additional_headers=headers,
|
additional_headers=headers,
|
||||||
)
|
)
|
||||||
|
self._websocket = websocket
|
||||||
await self._call_event_handler("on_connected")
|
await self._call_event_handler("on_connected")
|
||||||
setup_msg = {
|
setup_msg = {
|
||||||
"type": "setup",
|
"type": "setup",
|
||||||
@@ -397,8 +401,10 @@ class GradiumSTTService(WebsocketSTTService):
|
|||||||
json_config = {}
|
json_config = {}
|
||||||
if self._json_config:
|
if self._json_config:
|
||||||
json_config = json.loads(self._json_config)
|
json_config = json.loads(self._json_config)
|
||||||
language = assert_given(self._settings.language)
|
# Technically `_settings.language` could be a raw string, but
|
||||||
if language:
|
# Language is a StrEnum so downstream handles either.
|
||||||
|
language = cast("Language | None", assert_given(self._settings.language))
|
||||||
|
if language is not None:
|
||||||
gradium_language = language_to_gradium_language(language)
|
gradium_language = language_to_gradium_language(language)
|
||||||
if gradium_language:
|
if gradium_language:
|
||||||
json_config["language"] = gradium_language
|
json_config["language"] = gradium_language
|
||||||
@@ -406,8 +412,8 @@ class GradiumSTTService(WebsocketSTTService):
|
|||||||
json_config["delay_in_frames"] = self._settings.delay_in_frames
|
json_config["delay_in_frames"] = self._settings.delay_in_frames
|
||||||
if json_config:
|
if json_config:
|
||||||
setup_msg["json_config"] = json_config
|
setup_msg["json_config"] = json_config
|
||||||
await self._websocket.send(json.dumps(setup_msg))
|
await websocket.send(json.dumps(setup_msg))
|
||||||
ready_msg = await self._websocket.recv()
|
ready_msg = await websocket.recv()
|
||||||
ready_msg = json.loads(ready_msg)
|
ready_msg = json.loads(ready_msg)
|
||||||
if ready_msg["type"] == "error":
|
if ready_msg["type"] == "error":
|
||||||
raise Exception(f"received error {ready_msg['message']}")
|
raise Exception(f"received error {ready_msg['message']}")
|
||||||
@@ -478,12 +484,14 @@ class GradiumSTTService(WebsocketSTTService):
|
|||||||
"""
|
"""
|
||||||
self._accumulated_text.append(text)
|
self._accumulated_text.append(text)
|
||||||
accumulated = " ".join(self._accumulated_text)
|
accumulated = " ".join(self._accumulated_text)
|
||||||
|
# Technically `_settings.language` could be a raw string, but Language
|
||||||
|
# is a StrEnum so downstream handles either.
|
||||||
await self.push_frame(
|
await self.push_frame(
|
||||||
InterimTranscriptionFrame(
|
InterimTranscriptionFrame(
|
||||||
text=accumulated,
|
text=accumulated,
|
||||||
user_id=self._user_id,
|
user_id=self._user_id,
|
||||||
timestamp=time_now_iso8601(),
|
timestamp=time_now_iso8601(),
|
||||||
language=assert_given(self._settings.language),
|
language=cast("Language | None", assert_given(self._settings.language)),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
await self.stop_processing_metrics()
|
await self.stop_processing_metrics()
|
||||||
@@ -515,7 +523,9 @@ class GradiumSTTService(WebsocketSTTService):
|
|||||||
text = " ".join(self._accumulated_text)
|
text = " ".join(self._accumulated_text)
|
||||||
self._accumulated_text.clear()
|
self._accumulated_text.clear()
|
||||||
logger.debug(f"Final transcription: [{text}]")
|
logger.debug(f"Final transcription: [{text}]")
|
||||||
language = assert_given(self._settings.language)
|
# Technically `_settings.language` could be a raw string, but Language
|
||||||
|
# is a StrEnum so downstream handles either.
|
||||||
|
language = cast("Language | None", assert_given(self._settings.language))
|
||||||
await self.push_frame(
|
await self.push_frame(
|
||||||
TranscriptionFrame(
|
TranscriptionFrame(
|
||||||
text,
|
text,
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import io
|
|||||||
import wave
|
import wave
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Literal, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
@@ -31,6 +32,19 @@ except ModuleNotFoundError as e:
|
|||||||
logger.error("In order to use Groq, you need to `pip install pipecat-ai[groq]`.")
|
logger.error("In order to use Groq, you need to `pip install pipecat-ai[groq]`.")
|
||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
# Hint set for `output_format`. The values mirror the Literal that
|
||||||
|
# `groq.resources.audio.speech.AsyncSpeech.create` accepts on its
|
||||||
|
# `response_format` parameter (also visible as the `response_format` field of
|
||||||
|
# `groq.types.audio.SpeechCreateParams`). The groq SDK does not export this as
|
||||||
|
# a named alias, so we redeclare it here.
|
||||||
|
#
|
||||||
|
# This alias is used in unions like `GroqAudioFormat | str`, so pyright shows
|
||||||
|
# these values as completion hints without rejecting other strings. If groq
|
||||||
|
# adds a new format before this list is updated, callers can still pass it and
|
||||||
|
# we forward it through (with a cast at the API boundary). Keep in sync on a
|
||||||
|
# best-effort basis when bumping the groq dep.
|
||||||
|
GroqAudioFormat = Literal["flac", "mp3", "mulaw", "ogg", "wav"]
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class GroqTTSSettings(TTSSettings):
|
class GroqTTSSettings(TTSSettings):
|
||||||
@@ -74,7 +88,7 @@ class GroqTTSService(TTSService):
|
|||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
api_key: str,
|
api_key: str,
|
||||||
output_format: str = "wav",
|
output_format: GroqAudioFormat | str = "wav",
|
||||||
params: InputParams | None = None,
|
params: InputParams | None = None,
|
||||||
model_name: str | None = None,
|
model_name: str | None = None,
|
||||||
voice_id: str | None = None,
|
voice_id: str | None = None,
|
||||||
@@ -147,7 +161,7 @@ class GroqTTSService(TTSService):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self._api_key = api_key
|
self._api_key = api_key
|
||||||
self._output_format = output_format
|
self._output_format: str = output_format
|
||||||
|
|
||||||
self._client = AsyncGroq(api_key=self._api_key)
|
self._client = AsyncGroq(api_key=self._api_key)
|
||||||
|
|
||||||
@@ -178,12 +192,18 @@ class GroqTTSService(TTSService):
|
|||||||
speed = assert_given(self._settings.speed)
|
speed = assert_given(self._settings.speed)
|
||||||
if model is None:
|
if model is None:
|
||||||
raise ValueError("Groq TTS model must be specified")
|
raise ValueError("Groq TTS model must be specified")
|
||||||
|
if voice is None:
|
||||||
|
raise ValueError("Groq TTS voice must be specified")
|
||||||
if speed is None:
|
if speed is None:
|
||||||
raise ValueError("Groq TTS speed must be specified")
|
raise ValueError("Groq TTS speed must be specified")
|
||||||
response = await self._client.audio.speech.create(
|
response = await self._client.audio.speech.create(
|
||||||
model=model,
|
model=model,
|
||||||
voice=voice,
|
voice=voice,
|
||||||
response_format=self._output_format,
|
# Cast satisfies groq's stricter Literal typing while letting
|
||||||
|
# callers pass any string (e.g. a newer groq format we haven't
|
||||||
|
# yet added to GroqAudioFormat). If the value is unsupported,
|
||||||
|
# groq's API will surface a runtime error with a clear message.
|
||||||
|
response_format=cast(GroqAudioFormat, self._output_format),
|
||||||
# Note: as of 2026-02-25, only a speed of 1.0 is supported, but
|
# Note: as of 2026-02-25, only a speed of 1.0 is supported, but
|
||||||
# here we pass it for completeness and future-proofing
|
# here we pass it for completeness and future-proofing
|
||||||
speed=speed,
|
speed=speed,
|
||||||
|
|||||||
@@ -210,11 +210,11 @@ class HeyGenApi(BaseAvatarApi):
|
|||||||
"quality": request_data.quality,
|
"quality": request_data.quality,
|
||||||
"avatar_id": request_data.avatar_id,
|
"avatar_id": request_data.avatar_id,
|
||||||
"voice": {
|
"voice": {
|
||||||
"voice_id": request_data.voice.voiceId if request_data.voice else None,
|
"voice_id": request_data.voice.voice_id if request_data.voice else None,
|
||||||
"rate": request_data.voice.rate if request_data.voice else None,
|
"rate": request_data.voice.rate if request_data.voice else None,
|
||||||
"emotion": request_data.voice.emotion if request_data.voice else None,
|
"emotion": request_data.voice.emotion if request_data.voice else None,
|
||||||
"elevenlabs_settings": (
|
"elevenlabs_settings": (
|
||||||
request_data.voice.elevenlabsSettings if request_data.voice else None
|
request_data.voice.elevenlabs_settings if request_data.voice else None
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
"knowledge_id": request_data.knowledge_id,
|
"knowledge_id": request_data.knowledge_id,
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ class StandardSessionResponse(BaseModel):
|
|||||||
access_token: str
|
access_token: str
|
||||||
livekit_agent_token: str
|
livekit_agent_token: str
|
||||||
|
|
||||||
livekit_url: str = None
|
livekit_url: str
|
||||||
ws_url: str = None
|
ws_url: str
|
||||||
|
|
||||||
raw_response: Any
|
raw_response: Any
|
||||||
|
|
||||||
|
|||||||
@@ -291,6 +291,8 @@ class HeyGenClient:
|
|||||||
"""Handle incoming WebSocket messages."""
|
"""Handle incoming WebSocket messages."""
|
||||||
try:
|
try:
|
||||||
while self._connected:
|
while self._connected:
|
||||||
|
if self._websocket is None: # should never happen while _connected is True
|
||||||
|
break
|
||||||
try:
|
try:
|
||||||
message = await self._websocket.recv()
|
message = await self._websocket.recv()
|
||||||
parsed_message = json.loads(message)
|
parsed_message = json.loads(message)
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ from pipecat import version as pipecat_version
|
|||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
CancelFrame,
|
CancelFrame,
|
||||||
EndFrame,
|
EndFrame,
|
||||||
|
ErrorFrame,
|
||||||
Frame,
|
Frame,
|
||||||
InterruptionFrame,
|
InterruptionFrame,
|
||||||
StartFrame,
|
StartFrame,
|
||||||
@@ -289,10 +290,14 @@ class HumeTTSService(TTSService):
|
|||||||
"""
|
"""
|
||||||
logger.debug(f"{self}: Generating Hume TTS: [{text}]")
|
logger.debug(f"{self}: Generating Hume TTS: [{text}]")
|
||||||
|
|
||||||
|
voice_id = assert_given(self._settings.voice)
|
||||||
|
if voice_id is None:
|
||||||
|
yield ErrorFrame(error="Hume TTS voice must be specified")
|
||||||
|
return
|
||||||
# Build the request payload
|
# Build the request payload
|
||||||
utterance_kwargs: dict[str, Any] = {
|
utterance_kwargs: dict[str, Any] = {
|
||||||
"text": text,
|
"text": text,
|
||||||
"voice": PostedUtteranceVoiceWithId(id=assert_given(self._settings.voice)),
|
"voice": PostedUtteranceVoiceWithId(id=voice_id),
|
||||||
}
|
}
|
||||||
if self._settings.description is not None:
|
if self._settings.description is not None:
|
||||||
utterance_kwargs["description"] = self._settings.description
|
utterance_kwargs["description"] = self._settings.description
|
||||||
|
|||||||
@@ -189,7 +189,7 @@ _NON_FATAL_ERROR_CODES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class InworldRealtimeLLMService(LLMService):
|
class InworldRealtimeLLMService(LLMService[InworldRealtimeLLMAdapter]):
|
||||||
"""Inworld Realtime LLM service for real-time audio and text communication.
|
"""Inworld Realtime LLM service for real-time audio and text communication.
|
||||||
|
|
||||||
Implements the Inworld Realtime API with WebSocket communication for
|
Implements the Inworld Realtime API with WebSocket communication for
|
||||||
@@ -664,7 +664,7 @@ class InworldRealtimeLLMService(LLMService):
|
|||||||
async def _send_session_update(self):
|
async def _send_session_update(self):
|
||||||
"""Update session settings on the server."""
|
"""Update session settings on the server."""
|
||||||
settings = assert_given(self._settings.session_properties)
|
settings = assert_given(self._settings.session_properties)
|
||||||
adapter: InworldRealtimeLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
|
|
||||||
if self._context:
|
if self._context:
|
||||||
llm_invocation_params = adapter.get_llm_invocation_params(
|
llm_invocation_params = adapter.get_llm_invocation_params(
|
||||||
@@ -963,7 +963,7 @@ class InworldRealtimeLLMService(LLMService):
|
|||||||
self._run_llm_when_api_session_ready = True
|
self._run_llm_when_api_session_ready = True
|
||||||
return
|
return
|
||||||
|
|
||||||
adapter: InworldRealtimeLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
|
|
||||||
if self._llm_needs_conversation_setup:
|
if self._llm_needs_conversation_setup:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|||||||
@@ -794,8 +794,10 @@ class InworldTTSService(WebsocketTTSService):
|
|||||||
|
|
||||||
return word_times
|
return word_times
|
||||||
|
|
||||||
async def _close_context(self, context_id: str):
|
async def _close_context(self, context_id: str | None):
|
||||||
if context_id and self._websocket:
|
if not context_id:
|
||||||
|
return
|
||||||
|
if self._websocket:
|
||||||
logger.info(f"{self}: Closing context {context_id} due to interruption or completion")
|
logger.info(f"{self}: Closing context {context_id} due to interruption or completion")
|
||||||
try:
|
try:
|
||||||
await self._send_close_context(context_id)
|
await self._send_close_context(context_id)
|
||||||
|
|||||||
@@ -216,6 +216,8 @@ class KokoroTTSService(TTSService):
|
|||||||
await self.start_tts_usage_metrics(text)
|
await self.start_tts_usage_metrics(text)
|
||||||
|
|
||||||
voice = assert_given(self._settings.voice)
|
voice = assert_given(self._settings.voice)
|
||||||
|
if voice is None:
|
||||||
|
raise ValueError("Kokoro TTS voice must be specified")
|
||||||
lang = assert_given(self._settings.language)
|
lang = assert_given(self._settings.language)
|
||||||
if lang is None:
|
if lang is None:
|
||||||
raise ValueError("Kokoro TTS language must be specified")
|
raise ValueError("Kokoro TTS language must be specified")
|
||||||
|
|||||||
@@ -16,10 +16,13 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import (
|
from typing import (
|
||||||
Any,
|
Any,
|
||||||
|
Generic,
|
||||||
Protocol,
|
Protocol,
|
||||||
|
cast,
|
||||||
)
|
)
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
from typing_extensions import TypeVar
|
||||||
from websockets.exceptions import ConnectionClosed
|
from websockets.exceptions import ConnectionClosed
|
||||||
from websockets.protocol import State
|
from websockets.protocol import State
|
||||||
|
|
||||||
@@ -116,7 +119,12 @@ class FunctionCallParams:
|
|||||||
function_name: str
|
function_name: str
|
||||||
tool_call_id: str
|
tool_call_id: str
|
||||||
arguments: Mapping[str, Any]
|
arguments: Mapping[str, Any]
|
||||||
llm: LLMService
|
# `LLMService[Any]` so any concrete subclass (regardless of how — or
|
||||||
|
# whether — it parameterizes the adapter type) can be assigned here.
|
||||||
|
# Plain `LLMService` would invoke the TypeVar default and pyright would
|
||||||
|
# treat it invariantly, rejecting `LLMService[XAdapter]` at the call
|
||||||
|
# sites that build FunctionCallParams.
|
||||||
|
llm: LLMService[Any]
|
||||||
context: LLMContext
|
context: LLMContext
|
||||||
result_callback: FunctionCallResultCallback
|
result_callback: FunctionCallResultCallback
|
||||||
app_resources: Any = None
|
app_resources: Any = None
|
||||||
@@ -190,7 +198,14 @@ class FunctionCallRunnerItem:
|
|||||||
group_id: str | None = None
|
group_id: str | None = None
|
||||||
|
|
||||||
|
|
||||||
class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
|
# `default=BaseLLMAdapter` (PEP 696) so that unparameterized subclasses
|
||||||
|
# (e.g. third-party `class MyService(LLMService):` with no bracket) get
|
||||||
|
# `TAdapter = BaseLLMAdapter` instead of `Unknown` at type-check time —
|
||||||
|
# matching the pre-generic behavior of `get_llm_adapter()`.
|
||||||
|
TAdapter = TypeVar("TAdapter", bound=BaseLLMAdapter, default=BaseLLMAdapter)
|
||||||
|
|
||||||
|
|
||||||
|
class LLMService(UserTurnCompletionLLMServiceMixin, AIService, Generic[TAdapter]):
|
||||||
"""Base class for all LLM services.
|
"""Base class for all LLM services.
|
||||||
|
|
||||||
Handles function calling registration and execution with support for both
|
Handles function calling registration and execution with support for both
|
||||||
@@ -222,6 +237,7 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
_settings: LLMSettings
|
_settings: LLMSettings
|
||||||
|
_adapter: TAdapter
|
||||||
|
|
||||||
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
|
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
|
||||||
# However, subclasses should override this with a more specific adapter when necessary.
|
# However, subclasses should override this with a more specific adapter when necessary.
|
||||||
@@ -269,7 +285,12 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
|
|||||||
self._filter_incomplete_user_turns: bool = False
|
self._filter_incomplete_user_turns: bool = False
|
||||||
self._async_tool_cancellation_enabled: bool = False
|
self._async_tool_cancellation_enabled: bool = False
|
||||||
self._base_system_instruction: str | None = None
|
self._base_system_instruction: str | None = None
|
||||||
self._adapter = self.adapter_class()
|
# `adapter_class` is typed as `type[BaseLLMAdapter]` so subclasses
|
||||||
|
# don't need to spell out the generic parameter just to subclass
|
||||||
|
# (backward compatibility for 3rd-party providers outside this repo).
|
||||||
|
# Cast to TAdapter to keep `_adapter` and `get_llm_adapter()` precisely
|
||||||
|
# typed for callers that opt into `LLMService[XAdapter]`.
|
||||||
|
self._adapter = cast(TAdapter, self.adapter_class())
|
||||||
self._functions: dict[str | None, FunctionCallRegistryItem] = {}
|
self._functions: dict[str | None, FunctionCallRegistryItem] = {}
|
||||||
self._function_call_tasks: dict[asyncio.Task | None, FunctionCallRunnerItem] = {}
|
self._function_call_tasks: dict[asyncio.Task | None, FunctionCallRunnerItem] = {}
|
||||||
self._sequential_runner_task: asyncio.Task | None = None
|
self._sequential_runner_task: asyncio.Task | None = None
|
||||||
@@ -280,7 +301,7 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService):
|
|||||||
self._register_event_handler("on_function_calls_cancelled")
|
self._register_event_handler("on_function_calls_cancelled")
|
||||||
self._register_event_handler("on_completion_timeout")
|
self._register_event_handler("on_completion_timeout")
|
||||||
|
|
||||||
def get_llm_adapter(self) -> BaseLLMAdapter:
|
def get_llm_adapter(self) -> TAdapter:
|
||||||
"""Get the LLM adapter instance.
|
"""Get the LLM adapter instance.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -1112,7 +1133,7 @@ class WebsocketReconnectedError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class WebsocketLLMService(LLMService, WebsocketService):
|
class WebsocketLLMService(LLMService[TAdapter], WebsocketService, Generic[TAdapter]):
|
||||||
"""Base class for websocket-based LLM services.
|
"""Base class for websocket-based LLM services.
|
||||||
|
|
||||||
Each LLM inference is a discrete request/response exchange: send one
|
Each LLM inference is a discrete request/response exchange: send one
|
||||||
@@ -1160,7 +1181,11 @@ class WebsocketLLMService(LLMService, WebsocketService):
|
|||||||
reconnect_on_error: Whether to automatically reconnect on websocket errors.
|
reconnect_on_error: Whether to automatically reconnect on websocket errors.
|
||||||
**kwargs: Additional arguments passed to parent classes.
|
**kwargs: Additional arguments passed to parent classes.
|
||||||
"""
|
"""
|
||||||
LLMService.__init__(self, **kwargs)
|
# pyright stumbles here because the TypeVar default makes
|
||||||
|
# `LLMService` resolve to `LLMService[BaseLLMAdapter]` invariantly,
|
||||||
|
# while `self` is `WebsocketLLMService[TAdapter]` for an arbitrary
|
||||||
|
# TAdapter. The runtime call is fine — generics are erased.
|
||||||
|
LLMService.__init__(self, **kwargs) # pyright: ignore[reportArgumentType]
|
||||||
WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
|
WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs)
|
||||||
self._register_event_handler("on_connection_error")
|
self._register_event_handler("on_connection_error")
|
||||||
|
|
||||||
@@ -1240,6 +1265,11 @@ class WebsocketLLMService(LLMService, WebsocketService):
|
|||||||
Returns:
|
Returns:
|
||||||
The parsed JSON message as a dict.
|
The parsed JSON message as a dict.
|
||||||
"""
|
"""
|
||||||
|
# Should never happen — `_ensure_connected` (which callers must invoke
|
||||||
|
# first) raises ConnectionError if it can't establish a websocket.
|
||||||
|
# Match that contract here.
|
||||||
|
if self._websocket is None:
|
||||||
|
raise ConnectionError(f"{self} _ws_recv called without a websocket")
|
||||||
try:
|
try:
|
||||||
raw = await self._websocket.recv()
|
raw = await self._websocket.recv()
|
||||||
return json.loads(raw)
|
return json.loads(raw)
|
||||||
|
|||||||
@@ -37,14 +37,17 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_lmnt_language(language: Language) -> str | None:
|
def language_to_lmnt_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to LMNT language code.
|
"""Convert a Language enum to LMNT language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding LMNT language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AR: "ar",
|
Language.AR: "ar",
|
||||||
@@ -267,10 +270,11 @@ class LmntTTSService(InterruptibleTTSService):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Connect to LMNT's websocket directly
|
# Connect to LMNT's websocket directly
|
||||||
self._websocket = await websocket_connect("wss://api.lmnt.com/v1/ai/speech/stream")
|
websocket = await websocket_connect("wss://api.lmnt.com/v1/ai/speech/stream")
|
||||||
|
self._websocket = websocket
|
||||||
|
|
||||||
# Send initialization message
|
# Send initialization message
|
||||||
await self._websocket.send(json.dumps(init_msg))
|
await websocket.send(json.dumps(init_msg))
|
||||||
|
|
||||||
await self._call_event_handler("on_connected")
|
await self._call_event_handler("on_connected")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -31,14 +31,16 @@ from pipecat.transcriptions.language import Language, resolve_language
|
|||||||
from pipecat.utils.tracing.service_decorators import traced_tts
|
from pipecat.utils.tracing.service_decorators import traced_tts
|
||||||
|
|
||||||
|
|
||||||
def language_to_minimax_language(language: Language) -> str | None:
|
def language_to_minimax_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to MiniMax language format.
|
"""Convert a Language enum to MiniMax language format.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding MiniMax language name, or None if not supported.
|
The corresponding MiniMax language name. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AF: "Afrikaans",
|
Language.AF: "Afrikaans",
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ Voxtral Realtime transcription API using the Mistral SDK's RealtimeConnection.
|
|||||||
|
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -30,6 +30,7 @@ from pipecat.processors.frame_processor import FrameDirection
|
|||||||
from pipecat.services.settings import STTSettings, assert_given
|
from pipecat.services.settings import STTSettings, assert_given
|
||||||
from pipecat.services.stt_latency import MISTRAL_TTFS_P99
|
from pipecat.services.stt_latency import MISTRAL_TTFS_P99
|
||||||
from pipecat.services.stt_service import STTService
|
from pipecat.services.stt_service import STTService
|
||||||
|
from pipecat.transcriptions.language import Language
|
||||||
from pipecat.utils.time import time_now_iso8601
|
from pipecat.utils.time import time_now_iso8601
|
||||||
from pipecat.utils.tracing.service_decorators import traced_stt
|
from pipecat.utils.tracing.service_decorators import traced_stt
|
||||||
|
|
||||||
@@ -132,7 +133,7 @@ class MistralSTTService(STTService):
|
|||||||
self._connection: RealtimeConnection | None = None
|
self._connection: RealtimeConnection | None = None
|
||||||
self._receive_task = None
|
self._receive_task = None
|
||||||
self._accumulated_text = ""
|
self._accumulated_text = ""
|
||||||
self._detected_language: str | None = None
|
self._detected_language: Language | None = None
|
||||||
|
|
||||||
def can_generate_metrics(self) -> bool:
|
def can_generate_metrics(self) -> bool:
|
||||||
"""Check if the service can generate processing metrics.
|
"""Check if the service can generate processing metrics.
|
||||||
@@ -197,6 +198,13 @@ class MistralSTTService(STTService):
|
|||||||
if not self._connection or self._connection.is_closed:
|
if not self._connection or self._connection.is_closed:
|
||||||
await self._connect()
|
await self._connect()
|
||||||
|
|
||||||
|
# `_connect` swallows exceptions and may leave `_connection` unset;
|
||||||
|
# drop the audio chunk rather than crashing if reconnect failed.
|
||||||
|
if self._connection is None:
|
||||||
|
logger.warning(f"{self}: dropping audio chunk — Mistral STT not connected")
|
||||||
|
yield None
|
||||||
|
return
|
||||||
|
|
||||||
await self._connection.send_audio(audio)
|
await self._connection.send_audio(audio)
|
||||||
yield None
|
yield None
|
||||||
|
|
||||||
@@ -247,8 +255,13 @@ class MistralSTTService(STTService):
|
|||||||
|
|
||||||
async def _receive_events(self):
|
async def _receive_events(self):
|
||||||
"""Background task: iterate connection events and handle them."""
|
"""Background task: iterate connection events and handle them."""
|
||||||
|
# `_connect` started this task only after assigning `_connection`,
|
||||||
|
# so it should not be None here; bail out defensively just in case.
|
||||||
|
connection = self._connection
|
||||||
|
if connection is None:
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
async for event in self._connection.events():
|
async for event in connection.events():
|
||||||
if isinstance(event, RealtimeTranscriptionSessionCreated):
|
if isinstance(event, RealtimeTranscriptionSessionCreated):
|
||||||
logger.debug(f"{self}: Session created: {event.session}")
|
logger.debug(f"{self}: Session created: {event.session}")
|
||||||
await self._call_event_handler("on_connected")
|
await self._call_event_handler("on_connected")
|
||||||
@@ -278,7 +291,9 @@ class MistralSTTService(STTService):
|
|||||||
self._accumulated_text = ""
|
self._accumulated_text = ""
|
||||||
|
|
||||||
elif isinstance(event, TranscriptionStreamLanguage):
|
elif isinstance(event, TranscriptionStreamLanguage):
|
||||||
self._detected_language = event.audio_language
|
# Technically the SDK could emit a code we haven't added yet,
|
||||||
|
# but Language is a StrEnum so downstream handles either.
|
||||||
|
self._detected_language = cast("Language | None", event.audio_language)
|
||||||
|
|
||||||
elif isinstance(event, RealtimeTranscriptionError):
|
elif isinstance(event, RealtimeTranscriptionError):
|
||||||
error_msg = event.error.message if event.error else "Unknown error"
|
error_msg = event.error.message if event.error else "Unknown error"
|
||||||
|
|||||||
@@ -153,9 +153,14 @@ class MoondreamService(VisionService):
|
|||||||
logger.debug(f"Analyzing image (bytes length: {len(frame.image)})")
|
logger.debug(f"Analyzing image (bytes length: {len(frame.image)})")
|
||||||
|
|
||||||
def get_image_description(image_bytes: bytes, text: str | None) -> str:
|
def get_image_description(image_bytes: bytes, text: str | None) -> str:
|
||||||
|
if frame.format is None:
|
||||||
|
raise ValueError("Cannot decode image bytes without a format")
|
||||||
image = Image.frombytes(frame.format, frame.size, image_bytes)
|
image = Image.frombytes(frame.format, frame.size, image_bytes)
|
||||||
image_embeds = self._model.encode_image(image)
|
# `encode_image` and `query` are custom methods provided by the
|
||||||
description = self._model.query(image_embeds, text)["answer"]
|
# moondream2 model code (via `trust_remote_code=True`) that pyright
|
||||||
|
# can't see on `AutoModelForCausalLM`'s base type.
|
||||||
|
image_embeds = self._model.encode_image(image) # pyright: ignore[reportCallIssue]
|
||||||
|
description = self._model.query(image_embeds, text)["answer"] # pyright: ignore[reportCallIssue]
|
||||||
return description
|
return description
|
||||||
|
|
||||||
description = await asyncio.to_thread(get_image_description, frame.image, frame.text)
|
description = await asyncio.to_thread(get_image_description, frame.image, frame.text)
|
||||||
|
|||||||
@@ -44,14 +44,17 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_neuphonic_lang_code(language: Language) -> str | None:
|
def language_to_neuphonic_lang_code(language: Language) -> str:
|
||||||
"""Convert a Language enum to Neuphonic language code.
|
"""Convert a Language enum to Neuphonic language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Neuphonic language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.DE: "de",
|
Language.DE: "de",
|
||||||
@@ -328,7 +331,10 @@ class NeuphonicTTSService(InterruptibleTTSService):
|
|||||||
|
|
||||||
async def _receive_messages(self):
|
async def _receive_messages(self):
|
||||||
"""Receive and process messages from Neuphonic WebSocket."""
|
"""Receive and process messages from Neuphonic WebSocket."""
|
||||||
async for message in self._websocket:
|
websocket = self._websocket
|
||||||
|
if websocket is None:
|
||||||
|
return
|
||||||
|
async for message in websocket:
|
||||||
if isinstance(message, str):
|
if isinstance(message, str):
|
||||||
msg = json.loads(message)
|
msg = json.loads(message)
|
||||||
if msg.get("data") and msg["data"].get("audio"):
|
if msg.get("data") and msg["data"].get("audio"):
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_nvidia_nemotron_speech_language(language: Language) -> str | None:
|
def language_to_nvidia_nemotron_speech_language(language: Language) -> str:
|
||||||
"""Maps Language enum to NVIDIA Nemotron Speech ASR language codes.
|
"""Maps Language enum to NVIDIA Nemotron Speech ASR language codes.
|
||||||
|
|
||||||
Source:
|
Source:
|
||||||
@@ -59,7 +59,9 @@ def language_to_nvidia_nemotron_speech_language(language: Language) -> str | Non
|
|||||||
language: Language enum value.
|
language: Language enum value.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str | None: NVIDIA Nemotron Speech language code or None if not supported.
|
The NVIDIA Nemotron Speech language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
# Arabic
|
# Arabic
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ from openai._types import NotGiven as OpenAINotGiven
|
|||||||
from openai.types.chat import ChatCompletionChunk
|
from openai.types.chat import ChatCompletionChunk
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams
|
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter, OpenAILLMInvocationParams
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
Frame,
|
Frame,
|
||||||
LLMContextFrame,
|
LLMContextFrame,
|
||||||
@@ -39,7 +39,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext
|
|||||||
from pipecat.processors.frame_processor import FrameDirection
|
from pipecat.processors.frame_processor import FrameDirection
|
||||||
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
|
from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
|
||||||
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
|
from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
|
||||||
from pipecat.services.settings import LLMSettings, _NotGiven
|
from pipecat.services.settings import LLMSettings, _NotGiven, assert_given
|
||||||
from pipecat.utils.tracing.service_decorators import traced_llm
|
from pipecat.utils.tracing.service_decorators import traced_llm
|
||||||
|
|
||||||
|
|
||||||
@@ -66,12 +66,12 @@ class OpenAILLMSettings(LLMSettings):
|
|||||||
)
|
)
|
||||||
top_p: float | None | _NotGiven | OpenAINotGiven = field(default_factory=lambda: _NOT_GIVEN)
|
top_p: float | None | _NotGiven | OpenAINotGiven = field(default_factory=lambda: _NOT_GIVEN)
|
||||||
max_tokens: int | None | _NotGiven | OpenAINotGiven = field(default_factory=lambda: _NOT_GIVEN)
|
max_tokens: int | None | _NotGiven | OpenAINotGiven = field(default_factory=lambda: _NOT_GIVEN)
|
||||||
max_completion_tokens: int | _NotGiven | OpenAINotGiven = field(
|
max_completion_tokens: int | None | _NotGiven | OpenAINotGiven = field(
|
||||||
default_factory=lambda: _NOT_GIVEN
|
default_factory=lambda: _NOT_GIVEN
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class BaseOpenAILLMService(LLMService):
|
class BaseOpenAILLMService(LLMService[OpenAILLMAdapter]):
|
||||||
"""Base class for all services that use the AsyncOpenAI client.
|
"""Base class for all services that use the AsyncOpenAI client.
|
||||||
|
|
||||||
This service consumes LLMContextFrame frames, which contain a reference to
|
This service consumes LLMContextFrame frames, which contain a reference to
|
||||||
@@ -297,9 +297,9 @@ class BaseOpenAILLMService(LLMService):
|
|||||||
f"{self}: Generating chat from context {adapter.get_messages_for_logging(context)}"
|
f"{self}: Generating chat from context {adapter.get_messages_for_logging(context)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
params_from_context: OpenAILLMInvocationParams = adapter.get_llm_invocation_params(
|
params_from_context = adapter.get_llm_invocation_params(
|
||||||
context,
|
context,
|
||||||
system_instruction=self._settings.system_instruction,
|
system_instruction=assert_given(self._settings.system_instruction),
|
||||||
convert_developer_to_user=not self.supports_developer_role,
|
convert_developer_to_user=not self.supports_developer_role,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -374,7 +374,7 @@ class BaseOpenAILLMService(LLMService):
|
|||||||
"""
|
"""
|
||||||
effective_instruction = system_instruction or self._settings.system_instruction
|
effective_instruction = system_instruction or self._settings.system_instruction
|
||||||
adapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
invocation_params: OpenAILLMInvocationParams = adapter.get_llm_invocation_params(
|
invocation_params = adapter.get_llm_invocation_params(
|
||||||
context,
|
context,
|
||||||
system_instruction=effective_instruction,
|
system_instruction=effective_instruction,
|
||||||
convert_developer_to_user=not self.supports_developer_role,
|
convert_developer_to_user=not self.supports_developer_role,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ for creating images from text prompts.
|
|||||||
import io
|
import io
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Literal
|
from typing import Literal, cast
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
@@ -28,6 +28,28 @@ from pipecat.frames.frames import (
|
|||||||
from pipecat.services.image_service import ImageGenService
|
from pipecat.services.image_service import ImageGenService
|
||||||
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, assert_given
|
from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, assert_given
|
||||||
|
|
||||||
|
# Hint set for the `size` argument to `images.generate`. The values mirror the
|
||||||
|
# Literal that `openai.resources.images.Images.generate` accepts on its `size`
|
||||||
|
# parameter (also visible as the `size` field of
|
||||||
|
# `openai.types.image_generate_params.ImageGenerateParams`). The OpenAI SDK
|
||||||
|
# does not export this as a named alias, so we redeclare it here.
|
||||||
|
#
|
||||||
|
# We cast `_settings.image_size` (a plain `str`) to this Literal at the API
|
||||||
|
# boundary so callers can still pass any size string (e.g. a newer value the
|
||||||
|
# SDK accepts before this list is updated). Invalid values surface as an
|
||||||
|
# OpenAI API error at runtime. Keep in sync on a best-effort basis when
|
||||||
|
# bumping the openai dep.
|
||||||
|
OpenAIImageSize = Literal[
|
||||||
|
"auto",
|
||||||
|
"1024x1024",
|
||||||
|
"1536x1024",
|
||||||
|
"1024x1536",
|
||||||
|
"256x256",
|
||||||
|
"512x512",
|
||||||
|
"1792x1024",
|
||||||
|
"1024x1792",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class OpenAIImageGenSettings(ImageGenSettings):
|
class OpenAIImageGenSettings(ImageGenSettings):
|
||||||
@@ -116,15 +138,19 @@ class OpenAIImageGenService(ImageGenService):
|
|||||||
"""
|
"""
|
||||||
logger.debug(f"Generating image from prompt: {prompt}")
|
logger.debug(f"Generating image from prompt: {prompt}")
|
||||||
|
|
||||||
|
size = cast(OpenAIImageSize | None, assert_given(self._settings.image_size))
|
||||||
image = await self._client.images.generate(
|
image = await self._client.images.generate(
|
||||||
prompt=prompt,
|
prompt=prompt,
|
||||||
model=assert_given(self._settings.model),
|
model=assert_given(self._settings.model),
|
||||||
n=1,
|
n=1,
|
||||||
size=assert_given(self._settings.image_size),
|
size=size,
|
||||||
)
|
)
|
||||||
|
|
||||||
image_url = image.data[0].url
|
if not image.data:
|
||||||
|
yield ErrorFrame("Image generation failed: no data returned")
|
||||||
|
return
|
||||||
|
|
||||||
|
image_url = image.data[0].url
|
||||||
if not image_url:
|
if not image_url:
|
||||||
yield ErrorFrame("Image generation failed")
|
yield ErrorFrame("Image generation failed")
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -194,7 +194,7 @@ class OpenAIRealtimeLLMSettings(LLMSettings):
|
|||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
|
||||||
class OpenAIRealtimeLLMService(LLMService):
|
class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]):
|
||||||
"""OpenAI Realtime LLM service providing real-time audio and text communication.
|
"""OpenAI Realtime LLM service providing real-time audio and text communication.
|
||||||
|
|
||||||
Implements the OpenAI Realtime API with WebSocket communication for low-latency
|
Implements the OpenAI Realtime API with WebSocket communication for low-latency
|
||||||
@@ -657,7 +657,7 @@ class OpenAIRealtimeLLMService(LLMService):
|
|||||||
|
|
||||||
async def _send_session_update(self):
|
async def _send_session_update(self):
|
||||||
settings = assert_given(self._settings.session_properties)
|
settings = assert_given(self._settings.session_properties)
|
||||||
adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
|
|
||||||
if self._context:
|
if self._context:
|
||||||
llm_invocation_params = adapter.get_llm_invocation_params(
|
llm_invocation_params = adapter.get_llm_invocation_params(
|
||||||
@@ -1002,7 +1002,7 @@ class OpenAIRealtimeLLMService(LLMService):
|
|||||||
self._run_llm_when_api_session_ready = True
|
self._run_llm_when_api_session_ready = True
|
||||||
return
|
return
|
||||||
|
|
||||||
adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
|
|
||||||
# Configure the LLM for this session if needed
|
# Configure the LLM for this session if needed
|
||||||
if self._llm_needs_conversation_setup:
|
if self._llm_needs_conversation_setup:
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ class OpenAIResponsesLLMSettings(LLMSettings):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class _BaseOpenAIResponsesLLMService(LLMService):
|
class _BaseOpenAIResponsesLLMService(LLMService[OpenAIResponsesLLMAdapter]):
|
||||||
"""Shared base for HTTP and WebSocket OpenAI Responses API services.
|
"""Shared base for HTTP and WebSocket OpenAI Responses API services.
|
||||||
|
|
||||||
Contains settings, adapter reference, HTTP client creation, parameter
|
Contains settings, adapter reference, HTTP client creation, parameter
|
||||||
@@ -294,7 +294,7 @@ class _BaseOpenAIResponsesLLMService(LLMService):
|
|||||||
Returns:
|
Returns:
|
||||||
The LLM's response as a string, or None if no response is generated.
|
The LLM's response as a string, or None if no response is generated.
|
||||||
"""
|
"""
|
||||||
adapter: OpenAIResponsesLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
effective_instruction = system_instruction or assert_given(
|
effective_instruction = system_instruction or assert_given(
|
||||||
self._settings.system_instruction
|
self._settings.system_instruction
|
||||||
)
|
)
|
||||||
@@ -353,7 +353,9 @@ class _BaseOpenAIResponsesLLMService(LLMService):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
class OpenAIResponsesLLMService(_BaseOpenAIResponsesLLMService, WebsocketLLMService):
|
class OpenAIResponsesLLMService(
|
||||||
|
_BaseOpenAIResponsesLLMService, WebsocketLLMService[OpenAIResponsesLLMAdapter]
|
||||||
|
):
|
||||||
"""OpenAI Responses API LLM service using WebSocket transport.
|
"""OpenAI Responses API LLM service using WebSocket transport.
|
||||||
|
|
||||||
Maintains a persistent WebSocket connection to ``wss://api.openai.com/v1/responses``
|
Maintains a persistent WebSocket connection to ``wss://api.openai.com/v1/responses``
|
||||||
@@ -747,7 +749,7 @@ class OpenAIResponsesLLMService(_BaseOpenAIResponsesLLMService, WebsocketLLMServ
|
|||||||
if self._needs_drain:
|
if self._needs_drain:
|
||||||
await self._drain_cancelled_response()
|
await self._drain_cancelled_response()
|
||||||
|
|
||||||
adapter: OpenAIResponsesLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: Generating response from universal context "
|
f"{self}: Generating response from universal context "
|
||||||
f"{adapter.get_messages_for_logging(context)}"
|
f"{adapter.get_messages_for_logging(context)}"
|
||||||
@@ -987,7 +989,7 @@ class OpenAIResponsesHttpLLMService(_BaseOpenAIResponsesLLMService):
|
|||||||
|
|
||||||
@traced_llm
|
@traced_llm
|
||||||
async def _process_context(self, context: LLMContext):
|
async def _process_context(self, context: LLMContext):
|
||||||
adapter: OpenAIResponsesLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"{self}: Generating response from universal context "
|
f"{self}: Generating response from universal context "
|
||||||
f"{adapter.get_messages_for_logging(context)}"
|
f"{adapter.get_messages_for_logging(context)}"
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import base64
|
|||||||
import json
|
import json
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal, cast
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -475,6 +475,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
|
|||||||
async def _connect_websocket(self):
|
async def _connect_websocket(self):
|
||||||
"""Establish the WebSocket connection to the transcription endpoint."""
|
"""Establish the WebSocket connection to the transcription endpoint."""
|
||||||
try:
|
try:
|
||||||
|
# `__init__` raises if websockets isn't installed, so these symbols
|
||||||
|
# are non-None by the time any method runs.
|
||||||
|
assert websocket_connect is not None and State is not None
|
||||||
if self._websocket and self._websocket.state is State.OPEN:
|
if self._websocket and self._websocket.state is State.OPEN:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -534,7 +537,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
|
|||||||
"""Send ``session.update`` to configure the transcription session."""
|
"""Send ``session.update`` to configure the transcription session."""
|
||||||
transcription: dict = {"model": self._settings.model}
|
transcription: dict = {"model": self._settings.model}
|
||||||
|
|
||||||
language = assert_given(self._settings.language)
|
# Technically `_settings.language` could be a raw string, but Language
|
||||||
|
# is a StrEnum so downstream handles either.
|
||||||
|
language = cast("Language | None", assert_given(self._settings.language))
|
||||||
language_code = self._language_to_code(language) if language else None
|
language_code = self._language_to_code(language) if language else None
|
||||||
if language_code:
|
if language_code:
|
||||||
transcription["language"] = language_code
|
transcription["language"] = language_code
|
||||||
@@ -611,6 +616,10 @@ class OpenAIRealtimeSTTService(WebsocketSTTService):
|
|||||||
Called by ``WebsocketService._receive_task_handler`` which wraps
|
Called by ``WebsocketService._receive_task_handler`` which wraps
|
||||||
this method with automatic reconnection on connection errors.
|
this method with automatic reconnection on connection errors.
|
||||||
"""
|
"""
|
||||||
|
# `_connect` only starts the receive task after `_websocket` is set,
|
||||||
|
# and reconnects re-establish it before the next iteration, so this
|
||||||
|
# invariant should always hold when this method runs.
|
||||||
|
assert self._websocket is not None
|
||||||
async for message in self._websocket:
|
async for message in self._websocket:
|
||||||
try:
|
try:
|
||||||
evt = json.loads(message)
|
evt = json.loads(message)
|
||||||
|
|||||||
@@ -235,12 +235,22 @@ class OpenAITTSService(TTSService):
|
|||||||
Frame: Audio frames containing the synthesized speech data.
|
Frame: Audio frames containing the synthesized speech data.
|
||||||
"""
|
"""
|
||||||
logger.debug(f"{self}: Generating TTS [{text}]")
|
logger.debug(f"{self}: Generating TTS [{text}]")
|
||||||
|
voice = assert_given(self._settings.voice)
|
||||||
|
if voice is None:
|
||||||
|
yield ErrorFrame(error="OpenAI TTS voice must be specified")
|
||||||
|
return
|
||||||
|
if voice not in VALID_VOICES:
|
||||||
|
yield ErrorFrame(
|
||||||
|
error=f"OpenAI TTS voice {voice!r} is not supported "
|
||||||
|
f"(must be one of: {', '.join(sorted(VALID_VOICES))})"
|
||||||
|
)
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
# Setup API parameters
|
# Setup API parameters
|
||||||
create_params = {
|
create_params = {
|
||||||
"input": text,
|
"input": text,
|
||||||
"model": self._settings.model,
|
"model": self._settings.model,
|
||||||
"voice": VALID_VOICES[assert_given(self._settings.voice)],
|
"voice": VALID_VOICES[voice],
|
||||||
"response_format": "pcm",
|
"response_format": "pcm",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from typing import Any
|
|||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
|
from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams
|
||||||
from pipecat.services.openai.base_llm import BaseOpenAILLMService
|
from pipecat.services.openai.base_llm import BaseOpenAILLMService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.services.settings import assert_given
|
from pipecat.services.settings import assert_given
|
||||||
@@ -96,7 +97,9 @@ class OpenRouterLLMService(OpenAILLMService):
|
|||||||
logger.debug(f"Creating OpenRouter client with api {base_url}")
|
logger.debug(f"Creating OpenRouter client with api {base_url}")
|
||||||
return super().create_client(api_key, base_url, **kwargs)
|
return super().create_client(api_key, base_url, **kwargs)
|
||||||
|
|
||||||
def build_chat_completion_params(self, params_from_context: dict[str, Any]) -> dict[str, Any]:
|
def build_chat_completion_params(
|
||||||
|
self, params_from_context: OpenAILLMInvocationParams
|
||||||
|
) -> dict[str, Any]:
|
||||||
"""Builds chat parameters, handling model-specific constraints.
|
"""Builds chat parameters, handling model-specific constraints.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
@@ -101,6 +101,8 @@ class PiperTTSService(TTSService):
|
|||||||
download_dir = download_dir or Path.cwd()
|
download_dir = download_dir or Path.cwd()
|
||||||
|
|
||||||
_voice = assert_given(self._settings.voice)
|
_voice = assert_given(self._settings.voice)
|
||||||
|
if _voice is None:
|
||||||
|
raise ValueError("Piper TTS voice must be specified")
|
||||||
model_file = f"{_voice}.onnx"
|
model_file = f"{_voice}.onnx"
|
||||||
model_path_resolved = Path(download_dir) / model_file
|
model_path_resolved = Path(download_dir) / model_file
|
||||||
|
|
||||||
|
|||||||
@@ -409,7 +409,9 @@ class ResembleAITTSService(WebsocketTTSService):
|
|||||||
|
|
||||||
await self.push_frame(TTSStoppedFrame(context_id=context_id))
|
await self.push_frame(TTSStoppedFrame(context_id=context_id))
|
||||||
await self.stop_all_metrics()
|
await self.stop_all_metrics()
|
||||||
await self.push_error(ErrorFrame(error=f"{self} error: {error_name} - {error_msg}"))
|
await self.push_error_frame(
|
||||||
|
ErrorFrame(error=f"{self} error: {error_name} - {error_msg}")
|
||||||
|
)
|
||||||
|
|
||||||
# Check if this is an unrecoverable error (connection-level failure)
|
# Check if this is an unrecoverable error (connection-level failure)
|
||||||
if status_code in [401, 403]:
|
if status_code in [401, 403]:
|
||||||
|
|||||||
@@ -216,14 +216,16 @@ def get_speakers_for_model(model: str) -> list[str]:
|
|||||||
return list(TTS_MODEL_CONFIGS["bulbul:v2"].speakers)
|
return list(TTS_MODEL_CONFIGS["bulbul:v2"].speakers)
|
||||||
|
|
||||||
|
|
||||||
def language_to_sarvam_language(language: Language) -> str | None:
|
def language_to_sarvam_language(language: Language) -> str:
|
||||||
"""Convert Pipecat Language enum to Sarvam AI language codes.
|
"""Convert Pipecat Language enum to Sarvam AI language codes.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding Sarvam AI language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the full language code string and
|
||||||
|
logs a warning (via ``resolve_language(..., use_base_code=False)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.BN: "bn-IN", # Bengali
|
Language.BN: "bn-IN", # Bengali
|
||||||
|
|||||||
@@ -51,14 +51,17 @@ class SmallestTTSModel(StrEnum):
|
|||||||
LIGHTNING_V3_1 = "lightning-v3.1"
|
LIGHTNING_V3_1 = "lightning-v3.1"
|
||||||
|
|
||||||
|
|
||||||
def language_to_smallest_tts_language(language: Language) -> str | None:
|
def language_to_smallest_tts_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to a Smallest TTS language string.
|
"""Convert a Language enum to a Smallest TTS language string.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The Smallest language code string, or None if unsupported.
|
The corresponding Smallest language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AR: "ar",
|
Language.AR: "ar",
|
||||||
|
|||||||
@@ -648,4 +648,7 @@ class SonioxSTTService(WebsocketSTTService):
|
|||||||
Args:
|
Args:
|
||||||
silence: Silent PCM audio bytes (unused, Soniox uses a protocol message).
|
silence: Silent PCM audio bytes (unused, Soniox uses a protocol message).
|
||||||
"""
|
"""
|
||||||
|
if self._websocket is None:
|
||||||
|
logger.warning(f"{self}: websocket unavailable, skipping keepalive")
|
||||||
|
return
|
||||||
await self._websocket.send(KEEPALIVE_MESSAGE)
|
await self._websocket.send(KEEPALIVE_MESSAGE)
|
||||||
|
|||||||
@@ -269,7 +269,10 @@ class STTService(AIService):
|
|||||||
language: The language to convert.
|
language: The language to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The service-specific language identifier, or None if not supported.
|
The service-specific language identifier. Return ``None`` to
|
||||||
|
indicate an unsupported language. This optional return is an
|
||||||
|
extension hook for future or third-party subclasses; as of
|
||||||
|
2026-04-28, first-party services return a string.
|
||||||
"""
|
"""
|
||||||
return Language(language)
|
return Language(language)
|
||||||
|
|
||||||
@@ -859,6 +862,10 @@ class WebsocketSTTService(STTService, WebsocketService):
|
|||||||
Args:
|
Args:
|
||||||
silence: Silent 16-bit mono PCM audio bytes.
|
silence: Silent 16-bit mono PCM audio bytes.
|
||||||
"""
|
"""
|
||||||
|
if (
|
||||||
|
self._websocket is None
|
||||||
|
): # should never happen — caller should gate on _is_keepalive_ready()
|
||||||
|
return
|
||||||
await self._websocket.send(silence)
|
await self._websocket.send(silence)
|
||||||
|
|
||||||
async def _report_error(self, error: ErrorFrame):
|
async def _report_error(self, error: ErrorFrame):
|
||||||
|
|||||||
@@ -467,7 +467,10 @@ class TTSService(AIService):
|
|||||||
language: The language to convert.
|
language: The language to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The service-specific language identifier, or None if not supported.
|
The service-specific language identifier. Return ``None`` to
|
||||||
|
indicate an unsupported language. This optional return is an
|
||||||
|
extension hook for future or third-party subclasses; as of
|
||||||
|
2026-04-28, first-party services return a string.
|
||||||
"""
|
"""
|
||||||
return Language(language)
|
return Language(language)
|
||||||
|
|
||||||
@@ -609,8 +612,10 @@ class TTSService(AIService):
|
|||||||
"""Handle the completion of a turn."""
|
"""Handle the completion of a turn."""
|
||||||
# For HTTP services they emit the frames synchronously, so close the audio context here
|
# For HTTP services they emit the frames synchronously, so close the audio context here
|
||||||
# once all frames (including TTSTextFrame above) have been enqueued.
|
# once all frames (including TTSTextFrame above) have been enqueued.
|
||||||
if self._is_yielding_frames_synchronously and self.audio_context_available(
|
if (
|
||||||
self._turn_context_id
|
self._is_yielding_frames_synchronously
|
||||||
|
and self._turn_context_id is not None
|
||||||
|
and self.audio_context_available(self._turn_context_id)
|
||||||
):
|
):
|
||||||
if self._push_stop_frames:
|
if self._push_stop_frames:
|
||||||
await self.append_to_audio_context(
|
await self.append_to_audio_context(
|
||||||
@@ -1171,16 +1176,18 @@ class TTSService(AIService):
|
|||||||
logger.trace(f"{self} created audio context {context_id}")
|
logger.trace(f"{self} created audio context {context_id}")
|
||||||
|
|
||||||
async def append_to_audio_context(
|
async def append_to_audio_context(
|
||||||
self, context_id: str, frame: Frame | _WordTimestampEntry | None
|
self, context_id: str | None, frame: Frame | _WordTimestampEntry | None
|
||||||
):
|
):
|
||||||
"""Append a frame or word-timestamp entry to an existing audio context queue.
|
"""Append a frame or word-timestamp entry to an existing audio context queue.
|
||||||
|
|
||||||
Passing ``None`` signals end-of-context (used by remove_audio_context to mark
|
Passing a ``frame`` of ``None`` signals end-of-context (used by remove_audio_context
|
||||||
the queue for deletion). If the context no longer exists but the context_id
|
to mark the queue for deletion). If the context no longer exists but the context_id
|
||||||
matches the active turn, the context is transparently recreated before appending.
|
matches the active turn, the context is transparently recreated before appending.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
context_id: The context to append to.
|
context_id: The context to append to. ``None`` is accepted as a no-op
|
||||||
|
(with a debug log) so callers can pass through values from
|
||||||
|
``get_active_audio_context_id()`` without an explicit guard.
|
||||||
frame: The frame, word-timestamp entry, or ``None`` (end-of-context sentinel)
|
frame: The frame, word-timestamp entry, or ``None`` (end-of-context sentinel)
|
||||||
to append.
|
to append.
|
||||||
"""
|
"""
|
||||||
@@ -1201,12 +1208,17 @@ class TTSService(AIService):
|
|||||||
else:
|
else:
|
||||||
logger.debug(f"{self} unable to append audio to context {context_id}")
|
logger.debug(f"{self} unable to append audio to context {context_id}")
|
||||||
|
|
||||||
async def remove_audio_context(self, context_id: str):
|
async def remove_audio_context(self, context_id: str | None):
|
||||||
"""Remove an existing audio context.
|
"""Remove an existing audio context.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
context_id: The context to remove.
|
context_id: The context to remove. ``None`` is accepted as a
|
||||||
|
no-op (logged) so callers can pass through values from
|
||||||
|
``get_active_audio_context_id()`` without an explicit guard.
|
||||||
"""
|
"""
|
||||||
|
if not context_id:
|
||||||
|
logger.debug(f"{self} unable to remove audio context: no context ID provided")
|
||||||
|
return
|
||||||
if self.audio_context_available(context_id):
|
if self.audio_context_available(context_id):
|
||||||
# We just mark the audio context for deletion by appending
|
# We just mark the audio context for deletion by appending
|
||||||
# None. Once we reach None while handling audio we know we can
|
# None. Once we reach None while handling audio we know we can
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ class WebsocketService(ABC):
|
|||||||
reconnect_on_error: Whether to automatically reconnect on connection errors.
|
reconnect_on_error: Whether to automatically reconnect on connection errors.
|
||||||
**kwargs: Additional arguments (unused, for compatibility).
|
**kwargs: Additional arguments (unused, for compatibility).
|
||||||
"""
|
"""
|
||||||
self._websocket: websockets.WebSocketClientProtocol | None = None
|
self._websocket: websockets.WebSocketClientProtocol | None = None # pyright: ignore[reportAttributeAccessIssue]
|
||||||
self._reconnect_on_error = reconnect_on_error
|
self._reconnect_on_error = reconnect_on_error
|
||||||
self._reconnect_in_progress: bool = False
|
self._reconnect_in_progress: bool = False
|
||||||
self._disconnecting: bool = False
|
self._disconnecting: bool = False
|
||||||
@@ -120,12 +120,17 @@ class WebsocketService(ABC):
|
|||||||
async def send_with_retry(self, message, report_error: Callable[[ErrorFrame], Awaitable[None]]):
|
async def send_with_retry(self, message, report_error: Callable[[ErrorFrame], Awaitable[None]]):
|
||||||
"""Attempt to send a message, retrying after reconnect if necessary."""
|
"""Attempt to send a message, retrying after reconnect if necessary."""
|
||||||
try:
|
try:
|
||||||
|
# If websocket isn't connected/present, treat as a send failure —
|
||||||
|
# the broad `except Exception` below will trigger a reconnect
|
||||||
|
# attempt.
|
||||||
|
if self._websocket is None:
|
||||||
|
raise ConnectionError(f"{self} no websocket connected")
|
||||||
await self._websocket.send(message)
|
await self._websocket.send(message)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"{self} send failed: {e}, will try to reconnect")
|
logger.error(f"{self} send failed: {e}, will try to reconnect")
|
||||||
# Try to reconnect before retrying
|
# Try to reconnect before retrying
|
||||||
success = await self._try_reconnect(report_error=report_error)
|
success = await self._try_reconnect(report_error=report_error)
|
||||||
if success:
|
if success and self._websocket is not None:
|
||||||
logger.info(f"{self} reconnected successfully, will retry send the message")
|
logger.info(f"{self} reconnected successfully, will retry send the message")
|
||||||
# trying to send the message one more time
|
# trying to send the message one more time
|
||||||
await self._websocket.send(message)
|
await self._websocket.send(message)
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ class BaseWhisperSTTSettings(STTSettings):
|
|||||||
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
|
temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
|
||||||
|
|
||||||
|
|
||||||
def language_to_whisper_language(language: Language) -> str | None:
|
def language_to_whisper_language(language: Language) -> str:
|
||||||
"""Maps pipecat Language enum to Whisper API language codes.
|
"""Maps pipecat Language enum to Whisper API language codes.
|
||||||
|
|
||||||
Language support for Whisper API.
|
Language support for Whisper API.
|
||||||
@@ -50,7 +50,10 @@ def language_to_whisper_language(language: Language) -> str | None:
|
|||||||
language: A Language enum value representing the input language.
|
language: A Language enum value representing the input language.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str or None: The corresponding Whisper language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AF: "af",
|
Language.AF: "af",
|
||||||
@@ -235,7 +238,7 @@ class BaseWhisperSTTService(SegmentedSTTService):
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str or None: The corresponding service language code, or None if not supported.
|
The corresponding service language code, or None if not supported.
|
||||||
"""
|
"""
|
||||||
return language_to_whisper_language(language)
|
return language_to_whisper_language(language)
|
||||||
|
|
||||||
|
|||||||
@@ -97,14 +97,17 @@ class MLXModel(Enum):
|
|||||||
LARGE_V3_TURBO_Q4 = "mlx-community/whisper-large-v3-turbo-q4"
|
LARGE_V3_TURBO_Q4 = "mlx-community/whisper-large-v3-turbo-q4"
|
||||||
|
|
||||||
|
|
||||||
def language_to_whisper_language(language: Language) -> str | None:
|
def language_to_whisper_language(language: Language) -> str:
|
||||||
"""Maps pipecat Language enum to Whisper language codes.
|
"""Maps pipecat Language enum to Whisper language codes.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: A Language enum value representing the input language.
|
language: A Language enum value representing the input language.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str or None: The corresponding Whisper language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
Only includes languages officially supported by Whisper.
|
Only includes languages officially supported by Whisper.
|
||||||
@@ -300,7 +303,7 @@ class WhisperSTTService(SegmentedSTTService):
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str or None: The corresponding Whisper language code, or None if not supported.
|
The corresponding Whisper language code, or None if not supported.
|
||||||
"""
|
"""
|
||||||
return language_to_whisper_language(language)
|
return language_to_whisper_language(language)
|
||||||
|
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ class GrokRealtimeLLMSettings(LLMSettings):
|
|||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
|
||||||
class GrokRealtimeLLMService(LLMService):
|
class GrokRealtimeLLMService(LLMService[GrokRealtimeLLMAdapter]):
|
||||||
"""Grok Realtime Voice Agent LLM service providing real-time audio and text communication.
|
"""Grok Realtime Voice Agent LLM service providing real-time audio and text communication.
|
||||||
|
|
||||||
Implements the Grok Voice Agent API with WebSocket communication for low-latency
|
Implements the Grok Voice Agent API with WebSocket communication for low-latency
|
||||||
@@ -596,7 +596,7 @@ class GrokRealtimeLLMService(LLMService):
|
|||||||
async def _send_session_update(self):
|
async def _send_session_update(self):
|
||||||
"""Update session settings on the server."""
|
"""Update session settings on the server."""
|
||||||
settings = assert_given(self._settings.session_properties)
|
settings = assert_given(self._settings.session_properties)
|
||||||
adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
|
|
||||||
if self._context:
|
if self._context:
|
||||||
llm_invocation_params = adapter.get_llm_invocation_params(
|
llm_invocation_params = adapter.get_llm_invocation_params(
|
||||||
@@ -871,7 +871,7 @@ class GrokRealtimeLLMService(LLMService):
|
|||||||
self._run_llm_when_api_session_ready = True
|
self._run_llm_when_api_session_ready = True
|
||||||
return
|
return
|
||||||
|
|
||||||
adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter()
|
adapter = self.get_llm_adapter()
|
||||||
|
|
||||||
if self._llm_needs_conversation_setup:
|
if self._llm_needs_conversation_setup:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_xai_stt_language(language: Language) -> str | None:
|
def language_to_xai_stt_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to the xAI STT language code.
|
"""Convert a Language enum to the xAI STT language code.
|
||||||
|
|
||||||
xAI STT accepts two-letter language codes (e.g. ``en``, ``fr``, ``de``,
|
xAI STT accepts two-letter language codes (e.g. ``en``, ``fr``, ``de``,
|
||||||
@@ -54,7 +54,10 @@ def language_to_xai_stt_language(language: Language) -> str | None:
|
|||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding xAI STT language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AR: "ar",
|
Language.AR: "ar",
|
||||||
|
|||||||
@@ -49,14 +49,17 @@ except ModuleNotFoundError as e:
|
|||||||
raise Exception(f"Missing module: {e}")
|
raise Exception(f"Missing module: {e}")
|
||||||
|
|
||||||
|
|
||||||
def language_to_xai_language(language: Language) -> str | None:
|
def language_to_xai_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to xAI language code.
|
"""Convert a Language enum to xAI language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding xAI language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.AR: "ar-EG",
|
Language.AR: "ar-EG",
|
||||||
|
|||||||
@@ -37,14 +37,17 @@ from pipecat.utils.tracing.service_decorators import traced_tts
|
|||||||
# https://github.com/coqui-ai/xtts-streaming-server
|
# https://github.com/coqui-ai/xtts-streaming-server
|
||||||
|
|
||||||
|
|
||||||
def language_to_xtts_language(language: Language) -> str | None:
|
def language_to_xtts_language(language: Language) -> str:
|
||||||
"""Convert a Language enum to XTTS language code.
|
"""Convert a Language enum to XTTS language code.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
language: The Language enum value to convert.
|
language: The Language enum value to convert.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
The corresponding XTTS language code, or None if not supported.
|
The corresponding service language code. If ``language`` is not in
|
||||||
|
the verified mapping, falls back to the base language code (e.g.,
|
||||||
|
``en`` from ``en-US``) and logs a warning (via
|
||||||
|
``resolve_language(..., use_base_code=True)``).
|
||||||
"""
|
"""
|
||||||
LANGUAGE_MAP = {
|
LANGUAGE_MAP = {
|
||||||
Language.CS: "cs",
|
Language.CS: "cs",
|
||||||
@@ -211,7 +214,11 @@ class XTTSService(TTSService):
|
|||||||
logger.error(f"{self} no studio speakers available")
|
logger.error(f"{self} no studio speakers available")
|
||||||
return
|
return
|
||||||
|
|
||||||
embeddings = self._studio_speakers[assert_given(self._settings.voice)]
|
voice = assert_given(self._settings.voice)
|
||||||
|
if voice is None:
|
||||||
|
yield ErrorFrame(error="XTTS voice must be specified")
|
||||||
|
return
|
||||||
|
embeddings = self._studio_speakers[voice]
|
||||||
|
|
||||||
url = self._base_url + "/tts_stream"
|
url = self._base_url + "/tts_stream"
|
||||||
|
|
||||||
|
|||||||
@@ -771,13 +771,16 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._bot_stopped_speaking()
|
await self._bot_stopped_speaking()
|
||||||
|
|
||||||
async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
|
async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
|
||||||
|
# Caller below only invokes this when `self._mixer` is set.
|
||||||
|
mixer = self._mixer
|
||||||
|
assert mixer is not None
|
||||||
last_frame_time = 0
|
last_frame_time = 0
|
||||||
silence = b"\x00" * self._audio_chunk_size
|
silence = b"\x00" * self._audio_chunk_size
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
frame = self._audio_queue.get_nowait()
|
frame = self._audio_queue.get_nowait()
|
||||||
if isinstance(frame, OutputAudioRawFrame):
|
if isinstance(frame, OutputAudioRawFrame):
|
||||||
frame.audio = await self._mixer.mix(frame.audio)
|
frame.audio = await mixer.mix(frame.audio)
|
||||||
last_frame_time = time.time()
|
last_frame_time = time.time()
|
||||||
yield frame
|
yield frame
|
||||||
self._audio_queue.task_done()
|
self._audio_queue.task_done()
|
||||||
@@ -788,7 +791,7 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._bot_stopped_speaking()
|
await self._bot_stopped_speaking()
|
||||||
# Generate an audio frame with only the mixer's part.
|
# Generate an audio frame with only the mixer's part.
|
||||||
frame = OutputAudioRawFrame(
|
frame = OutputAudioRawFrame(
|
||||||
audio=await self._mixer.mix(silence),
|
audio=await mixer.mix(silence),
|
||||||
sample_rate=self._sample_rate,
|
sample_rate=self._sample_rate,
|
||||||
num_channels=self._params.audio_out_channels,
|
num_channels=self._params.audio_out_channels,
|
||||||
)
|
)
|
||||||
@@ -927,6 +930,11 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def resize_frame(frame: OutputImageRawFrame) -> OutputImageRawFrame:
|
def resize_frame(frame: OutputImageRawFrame) -> OutputImageRawFrame:
|
||||||
|
# Without a format we can't decode the bytes, so leave the
|
||||||
|
# frame as-is and let the transport pass it through unchanged.
|
||||||
|
if frame.format is None:
|
||||||
|
return frame
|
||||||
|
|
||||||
desired_size = (self._params.video_out_width, self._params.video_out_height)
|
desired_size = (self._params.video_out_width, self._params.video_out_height)
|
||||||
|
|
||||||
# TODO: we should refactor in the future to support dynamic resolutions
|
# TODO: we should refactor in the future to support dynamic resolutions
|
||||||
|
|||||||
@@ -239,8 +239,9 @@ class HeyGenOutputTransport(BaseOutputTransport):
|
|||||||
logger.warning("self._event_id is already defined!")
|
logger.warning("self._event_id is already defined!")
|
||||||
self._event_id = str(frame.id)
|
self._event_id = str(frame.id)
|
||||||
elif isinstance(frame, BotStoppedSpeakingFrame):
|
elif isinstance(frame, BotStoppedSpeakingFrame):
|
||||||
await self._client.agent_speak_end(self._event_id)
|
if self._event_id is not None:
|
||||||
self._event_id = None
|
await self._client.agent_speak_end(self._event_id)
|
||||||
|
self._event_id = None
|
||||||
await super().push_frame(frame, direction)
|
await super().push_frame(frame, direction)
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
@@ -261,7 +262,8 @@ class HeyGenOutputTransport(BaseOutputTransport):
|
|||||||
"""
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
if isinstance(frame, InterruptionFrame):
|
if isinstance(frame, InterruptionFrame):
|
||||||
await self._client.interrupt(self._event_id)
|
if self._event_id is not None:
|
||||||
|
await self._client.interrupt(self._event_id)
|
||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
if isinstance(frame, UserStartedSpeakingFrame):
|
if isinstance(frame, UserStartedSpeakingFrame):
|
||||||
await self._client.start_agent_listening()
|
await self._client.start_agent_listening()
|
||||||
@@ -281,6 +283,11 @@ class HeyGenOutputTransport(BaseOutputTransport):
|
|||||||
audio = frame.audio
|
audio = frame.audio
|
||||||
if frame.sample_rate != HEY_GEN_SAMPLE_RATE:
|
if frame.sample_rate != HEY_GEN_SAMPLE_RATE:
|
||||||
audio = await self._resampler.resample(audio, frame.sample_rate, HEY_GEN_SAMPLE_RATE)
|
audio = await self._resampler.resample(audio, frame.sample_rate, HEY_GEN_SAMPLE_RATE)
|
||||||
|
if self._event_id is None:
|
||||||
|
# No active bot-speech event — drop the chunk rather than send a
|
||||||
|
# message the HeyGen API will reject.
|
||||||
|
logger.warning(f"{self}: dropping audio frame because no event_id is set")
|
||||||
|
return False
|
||||||
await self._client.agent_speak(bytes(audio), self._event_id)
|
await self._client.agent_speak(bytes(audio), self._event_id)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|||||||
@@ -312,6 +312,9 @@ class LemonSliceTransportClient:
|
|||||||
Args:
|
Args:
|
||||||
frame: The message frame to send.
|
frame: The message frame to send.
|
||||||
"""
|
"""
|
||||||
|
if self._daily_transport_client is None:
|
||||||
|
return
|
||||||
|
|
||||||
await self._daily_transport_client.send_message(frame)
|
await self._daily_transport_client.send_message(frame)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -224,6 +224,8 @@ class SmallWebRTCRequestHandler:
|
|||||||
)
|
)
|
||||||
|
|
||||||
answer = pipecat_connection.get_answer()
|
answer = pipecat_connection.get_answer()
|
||||||
|
if answer is None:
|
||||||
|
raise RuntimeError("SmallWebRTC connection produced no SDP answer")
|
||||||
|
|
||||||
if self._esp32_mode:
|
if self._esp32_mode:
|
||||||
from pipecat.runner.utils import smallwebrtc_sdp_munging
|
from pipecat.runner.utils import smallwebrtc_sdp_munging
|
||||||
|
|||||||
@@ -360,6 +360,9 @@ class TavusTransportClient:
|
|||||||
Args:
|
Args:
|
||||||
frame: The message frame to send.
|
frame: The message frame to send.
|
||||||
"""
|
"""
|
||||||
|
if self._client is None:
|
||||||
|
return
|
||||||
|
|
||||||
await self._client.send_message(frame)
|
await self._client.send_message(frame)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -416,6 +419,7 @@ class TavusTransportClient:
|
|||||||
"""
|
"""
|
||||||
if not self._client:
|
if not self._client:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
return await self._client.write_audio_frame(frame)
|
return await self._client.write_audio_frame(frame)
|
||||||
|
|
||||||
async def register_audio_destination(self, destination: str, auto_silence: bool | None = True):
|
async def register_audio_destination(self, destination: str, auto_silence: bool | None = True):
|
||||||
|
|||||||
@@ -64,9 +64,18 @@ class WebsocketClientCallbacks(BaseModel):
|
|||||||
on_message: Called when a message is received from the WebSocket.
|
on_message: Called when a message is received from the WebSocket.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
on_connected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
|
on_connected: Callable[
|
||||||
on_disconnected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
|
[websockets.WebSocketClientProtocol], # pyright: ignore[reportAttributeAccessIssue]
|
||||||
on_message: Callable[[websockets.WebSocketClientProtocol, websockets.Data], Awaitable[None]]
|
Awaitable[None],
|
||||||
|
]
|
||||||
|
on_disconnected: Callable[
|
||||||
|
[websockets.WebSocketClientProtocol], # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
Awaitable[None],
|
||||||
|
]
|
||||||
|
on_message: Callable[
|
||||||
|
[websockets.WebSocketClientProtocol, websockets.Data], # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
Awaitable[None],
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
class WebsocketClientSession:
|
class WebsocketClientSession:
|
||||||
@@ -98,7 +107,7 @@ class WebsocketClientSession:
|
|||||||
|
|
||||||
self._leave_counter = 0
|
self._leave_counter = 0
|
||||||
self._task_manager: BaseTaskManager | None = None
|
self._task_manager: BaseTaskManager | None = None
|
||||||
self._websocket: websockets.WebSocketClientProtocol | None = None
|
self._websocket: websockets.WebSocketClientProtocol | None = None # pyright: ignore[reportAttributeAccessIssue]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def task_manager(self) -> BaseTaskManager:
|
def task_manager(self) -> BaseTaskManager:
|
||||||
@@ -192,6 +201,10 @@ class WebsocketClientSession:
|
|||||||
|
|
||||||
async def _client_task_handler(self):
|
async def _client_task_handler(self):
|
||||||
"""Handle incoming messages from the WebSocket connection."""
|
"""Handle incoming messages from the WebSocket connection."""
|
||||||
|
# `connect()` only starts this task after `_websocket` is assigned, and
|
||||||
|
# `disconnect()` cancels the task before clearing `_websocket`, so this
|
||||||
|
# invariant should always hold when this method runs.
|
||||||
|
assert self._websocket is not None
|
||||||
try:
|
try:
|
||||||
# Handle incoming messages
|
# Handle incoming messages
|
||||||
async for message in self._websocket:
|
async for message in self._websocket:
|
||||||
|
|||||||
@@ -226,7 +226,7 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
# Notify disconnection
|
# Notify disconnection
|
||||||
await self._callbacks.on_client_disconnected(websocket)
|
await self._callbacks.on_client_disconnected(websocket)
|
||||||
|
|
||||||
await self._websocket.close()
|
await websocket.close()
|
||||||
self._websocket = None
|
self._websocket = None
|
||||||
|
|
||||||
logger.info(f"Client {websocket.remote_address} disconnected")
|
logger.info(f"Client {websocket.remote_address} disconnected")
|
||||||
|
|||||||
@@ -154,8 +154,17 @@ class WhatsAppClient:
|
|||||||
|
|
||||||
return int(challenge)
|
return int(challenge)
|
||||||
|
|
||||||
async def _validate_whatsapp_webhook_request(self, raw_body: bytes, sha256_signature: str):
|
async def _validate_whatsapp_webhook_request(
|
||||||
|
self, raw_body: bytes | None, sha256_signature: str | None
|
||||||
|
):
|
||||||
"""Common handler for both /start and /connect endpoints."""
|
"""Common handler for both /start and /connect endpoints."""
|
||||||
|
# Callers gate on `self._whatsapp_secret`, so the assert holds.
|
||||||
|
assert self._whatsapp_secret is not None
|
||||||
|
if raw_body is None:
|
||||||
|
raise Exception("Missing raw request body")
|
||||||
|
if not sha256_signature:
|
||||||
|
raise Exception("Missing X-Hub-Signature-256 header")
|
||||||
|
|
||||||
# Compute HMAC SHA256 using your App Secret
|
# Compute HMAC SHA256 using your App Secret
|
||||||
expected_signature = hmac.new(
|
expected_signature = hmac.new(
|
||||||
key=self._whatsapp_secret.encode("utf-8"),
|
key=self._whatsapp_secret.encode("utf-8"),
|
||||||
@@ -164,8 +173,6 @@ class WhatsAppClient:
|
|||||||
).hexdigest()
|
).hexdigest()
|
||||||
|
|
||||||
# Extract signature from header (strip 'sha256=' prefix)
|
# Extract signature from header (strip 'sha256=' prefix)
|
||||||
if not sha256_signature:
|
|
||||||
raise Exception("Missing X-Hub-Signature-256 header")
|
|
||||||
received_signature = sha256_signature.split("sha256=")[-1]
|
received_signature = sha256_signature.split("sha256=")[-1]
|
||||||
|
|
||||||
# Compare signatures securely
|
# Compare signatures securely
|
||||||
@@ -306,7 +313,12 @@ class WhatsAppClient:
|
|||||||
# Create and initialize WebRTC connection
|
# Create and initialize WebRTC connection
|
||||||
pipecat_connection = SmallWebRTCConnection(self._ice_servers)
|
pipecat_connection = SmallWebRTCConnection(self._ice_servers)
|
||||||
await pipecat_connection.initialize(sdp=call.session.sdp, type=call.session.sdp_type)
|
await pipecat_connection.initialize(sdp=call.session.sdp, type=call.session.sdp_type)
|
||||||
sdp_answer = pipecat_connection.get_answer().get("sdp")
|
answer = pipecat_connection.get_answer()
|
||||||
|
if answer is None:
|
||||||
|
raise RuntimeError("SmallWebRTC connection produced no SDP answer")
|
||||||
|
sdp_answer = answer.get("sdp")
|
||||||
|
if sdp_answer is None:
|
||||||
|
raise RuntimeError("SmallWebRTC SDP answer missing 'sdp' field")
|
||||||
sdp_answer = self._filter_sdp_for_whatsapp(sdp_answer)
|
sdp_answer = self._filter_sdp_for_whatsapp(sdp_answer)
|
||||||
|
|
||||||
logger.debug(f"SDP answer generated for call {call.id}")
|
logger.debug(f"SDP answer generated for call {call.id}")
|
||||||
|
|||||||
@@ -7,6 +7,8 @@
|
|||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import AsyncMock, patch
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
||||||
|
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
FunctionCallFromLLM,
|
FunctionCallFromLLM,
|
||||||
FunctionCallInProgressFrame,
|
FunctionCallInProgressFrame,
|
||||||
@@ -39,6 +41,25 @@ class MockLLMService(LLMService):
|
|||||||
super().__init__(settings=settings, **kwargs)
|
super().__init__(settings=settings, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class TestUnparameterizedSubclass(unittest.TestCase):
|
||||||
|
"""Backward-compat coverage: third-party providers subclass LLMService
|
||||||
|
without specifying a generic adapter parameter. That should keep working
|
||||||
|
after LLMService became `Generic[TAdapter]`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_unparameterized_subclass_instantiates(self):
|
||||||
|
# MockLLMService is declared as `class MockLLMService(LLMService):`
|
||||||
|
# — no generic bracket. The TypeVar's `bound=BaseLLMAdapter` should
|
||||||
|
# resolve TAdapter to BaseLLMAdapter for callers that don't opt in.
|
||||||
|
service = MockLLMService()
|
||||||
|
adapter = service.get_llm_adapter()
|
||||||
|
|
||||||
|
# Default adapter_class is OpenAILLMAdapter; the runtime instance
|
||||||
|
# should reflect that, regardless of how generics are erased.
|
||||||
|
self.assertIsInstance(adapter, OpenAILLMAdapter)
|
||||||
|
self.assertIsInstance(adapter, BaseLLMAdapter)
|
||||||
|
|
||||||
|
|
||||||
class TestLLMService(unittest.IsolatedAsyncioTestCase):
|
class TestLLMService(unittest.IsolatedAsyncioTestCase):
|
||||||
async def _run_function_calls_inline(self, service: MockLLMService):
|
async def _run_function_calls_inline(self, service: MockLLMService):
|
||||||
async def run_inline(runner_items):
|
async def run_inline(runner_items):
|
||||||
|
|||||||
Reference in New Issue
Block a user