ec574edd53
* Add Inworld Realtime LLM service Adds a WebSocket-based realtime service for Inworld's cascade STT/LLM/TTS API with semantic VAD, function calling, and streaming transcription support. New files: - src/pipecat/services/inworld/realtime/ (service, events) - src/pipecat/adapters/services/inworld_realtime_adapter.py - examples/foundational/19zb-inworld-realtime.py Also includes: - websockets dependency for inworld extra in pyproject.toml - Adapter and settings tests matching OpenAI/Grok realtime patterns - Fix for double-response when server-side VAD is enabled * Prefer init-provided system instruction in Inworld Realtime Adopt _resolve_system_instruction() from BaseLLMAdapter, matching the pattern applied to OpenAI Realtime, Grok Realtime, Gemini Live, and Nova Sonic in the pk/realtime-services-init-v-context-system-instructions-cleanup branch. * Update changelog entry with PR number * Fix changelog format to use bullet point * Polish PR: default model, example cleanup, changelog update - Change default model from gpt-4.1-nano to gpt-4.1-mini - Add function calling demo to example - Remove demo-testing artifact from system instruction - Mention Router support in changelog * Address PR review feedback for Inworld Realtime - Move example to examples/realtime/realtime-inworld.py - Change initial context role from "user" to "developer" - Remove explicit sample rates from example; sync them in _ensure_audio_config so Inworld gets the transport's actual rates - Add audio race condition guard in _handle_evt_audio_delta (matches OpenAI realtime pattern) - Convert remaining "system"/"developer" messages to "user" in adapter - Add clarifying comment for local-VAD vs server-VAD metrics paths * Simplify example, add provider tracking, remove local VAD path - Remove function calling from example, switch model to xai/grok-4-1-fast-non-reasoning - Add pipecat-realtime session key prefix and provider_data metadata for Inworld traffic attribution - Remove local VAD code path (Inworld only supports server-side VAD) - Use typed InputAudioBufferAppendEvent for audio sends * Default TTS model to inworld-tts-1.5-max * Remove dead shimmed tools code, set STT/VAD defaults - Remove non-functional AdapterType.SHIM custom tools code from adapter - Default STT model to assemblyai/u3-rt-pro - Default VAD eagerness to low
238 lines
9.4 KiB
Python
238 lines
9.4 KiB
Python
#
|
|
# Copyright (c) 2024-2026, Daily
|
|
#
|
|
# SPDX-License-Identifier: BSD 2-Clause License
|
|
#
|
|
|
|
import unittest
|
|
|
|
from openai.types.chat import ChatCompletionToolParam
|
|
|
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
|
from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
|
|
from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter
|
|
from pipecat.adapters.services.bedrock_adapter import AWSBedrockLLMAdapter
|
|
from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter
|
|
from pipecat.adapters.services.inworld_realtime_adapter import InworldRealtimeLLMAdapter
|
|
from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter
|
|
from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter
|
|
|
|
|
|
class TestFunctionAdapters(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
"""Sets up a common tools schema for all tests."""
|
|
function_def = FunctionSchema(
|
|
name="get_weather",
|
|
description="Get the weather in a given location",
|
|
properties={
|
|
"location": {"type": "string", "description": "The city, e.g. San Francisco"},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
required=["location", "format"],
|
|
)
|
|
self.tools_def = ToolsSchema(standard_tools=[function_def])
|
|
|
|
def test_openai_adapter(self):
|
|
"""Test OpenAI adapter format transformation."""
|
|
expected = [
|
|
ChatCompletionToolParam(
|
|
type="function",
|
|
function={
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
},
|
|
},
|
|
)
|
|
]
|
|
assert OpenAILLMAdapter().to_provider_tools_format(self.tools_def) == expected
|
|
|
|
def test_anthropic_adapter(self):
|
|
"""Test Anthropic adapter format transformation."""
|
|
expected = [
|
|
{
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
},
|
|
}
|
|
]
|
|
assert AnthropicLLMAdapter().to_provider_tools_format(self.tools_def) == expected
|
|
|
|
def test_gemini_adapter(self):
|
|
"""Test Gemini adapter format transformation."""
|
|
expected = [
|
|
{
|
|
"function_declarations": [
|
|
{
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
},
|
|
}
|
|
]
|
|
}
|
|
]
|
|
assert GeminiLLMAdapter().to_provider_tools_format(self.tools_def) == expected
|
|
|
|
def test_openai_realtime_adapter(self):
|
|
"""Test Anthropic adapter format transformation."""
|
|
expected = [
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
},
|
|
}
|
|
]
|
|
assert OpenAIRealtimeLLMAdapter().to_provider_tools_format(self.tools_def) == expected
|
|
|
|
def test_inworld_realtime_adapter(self):
|
|
"""Test Inworld Realtime adapter format transformation."""
|
|
expected = [
|
|
{
|
|
"type": "function",
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
},
|
|
}
|
|
]
|
|
assert InworldRealtimeLLMAdapter().to_provider_tools_format(self.tools_def) == expected
|
|
|
|
def test_gemini_adapter_with_custom_tools(self):
|
|
"""Test Gemini adapter format transformation."""
|
|
search_tool = {"google_search": {}}
|
|
expected = [
|
|
{
|
|
"function_declarations": [
|
|
{
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
},
|
|
}
|
|
]
|
|
},
|
|
search_tool,
|
|
]
|
|
tools_def = self.tools_def
|
|
tools_def.custom_tools = {AdapterType.GEMINI: [search_tool]}
|
|
assert GeminiLLMAdapter().to_provider_tools_format(tools_def) == expected
|
|
|
|
def test_bedrock_adapter(self):
|
|
"""Test AWS Bedrock adapter format transformation."""
|
|
expected = [
|
|
{
|
|
"toolSpec": {
|
|
"name": "get_weather",
|
|
"description": "Get the weather in a given location",
|
|
"inputSchema": {
|
|
"json": {
|
|
"type": "object",
|
|
"properties": {
|
|
"format": {
|
|
"type": "string",
|
|
"enum": ["celsius", "fahrenheit"],
|
|
"description": "The temperature unit to use.",
|
|
},
|
|
"location": {
|
|
"type": "string",
|
|
"description": "The city, e.g. San Francisco",
|
|
},
|
|
},
|
|
"required": ["location", "format"],
|
|
}
|
|
},
|
|
}
|
|
}
|
|
]
|
|
assert AWSBedrockLLMAdapter().to_provider_tools_format(self.tools_def) == expected
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|