From 70aeb5c7c2b05d6982ab2cff22b0744796085327 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 14:14:57 -0400 Subject: [PATCH 01/23] fix: resolve pyright errors in Anthropic get_messages_for_logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content items in MessageParam have a heterogeneous union type (Pydantic ContentBlock variants and TypedDict *BlockParam variants), neither of which supports the dict-style access and mutation this sanitizer does. Treat the deepcopied message as a plain dict and guard each content item with isinstance(item, dict) — matches the runtime shape produced by _from_standard_message and avoids crashing if a non-dict ever flows through the LLMSpecificMessage path. Drops anthropic_adapter.py from 115 to 23 pyright errors. --- .../adapters/services/anthropic_adapter.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index 0fc1636f7..6832192dc 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -121,16 +121,20 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): messages = self._from_universal_context_messages(self.get_messages(context)).messages # Sanitize messages for logging - messages_for_logging = [] + messages_for_logging: list[dict[str, Any]] = [] for message in messages: - msg = copy.deepcopy(message) - if "content" in msg: - if isinstance(msg["content"], list): - for item in msg["content"]: - if item["type"] == "image": - item["source"]["data"] = "..." - if item["type"] == "thinking" and item.get("signature"): - item["signature"] = "..." + msg: dict[str, Any] = copy.deepcopy(dict(message)) + content = msg.get("content") + if isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + if item.get("type") == "image": + source = item.get("source") + if isinstance(source, dict): + source["data"] = "..." + if item.get("type") == "thinking" and item.get("signature"): + item["signature"] = "..." messages_for_logging.append(msg) return messages_for_logging From c517b67baddf9fc41f3196289c6c6d6c6ada0e82 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 14:39:24 -0400 Subject: [PATCH 02/23] fix: resolve pyright error when merging consecutive Anthropic messages MessageParam types content as `str | Iterable[...]`, and Iterable has no `.extend()`. After the str-to-list conversions, pyright re-reads the TypedDict field as the original wide type rather than carrying the narrowing forward. Cast to `list[Any]` to express the codebase's existing str-or-list assumption. Drops anthropic_adapter.py from 23 to 21 pyright errors. --- src/pipecat/adapters/services/anthropic_adapter.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index 6832192dc..d0fdfca3b 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -9,7 +9,7 @@ import copy import json from dataclasses import dataclass -from typing import Any, TypedDict, TypeGuard, TypeVar +from typing import Any, TypedDict, TypeGuard, TypeVar, cast from anthropic import NOT_GIVEN, NotGiven from anthropic.types.message_param import MessageParam @@ -189,8 +189,13 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): ] if isinstance(next_message["content"], str): next_message["content"] = [{"type": "text", "text": next_message["content"]}] - # Concatenate the content - current_message["content"].extend(next_message["content"]) + # Concatenate the content. MessageParam types content as + # `str | Iterable[...]`, but this codebase assumes it's + # either a str or a list. The str case is handled above, so + # we assume that both are lists here. + cast(list[Any], current_message["content"]).extend( + cast(list[Any], next_message["content"]) + ) # Remove the next message from the list messages.pop(i + 1) else: From 252bb493af28eae44c9b2637fb51b418fce9da06 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 14:43:23 -0400 Subject: [PATCH 03/23] fix: cast Anthropic-format passthrough message to MessageParam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fallback path in `_from_universal_context_message` returns `message.message` from an `LLMSpecificMessage`, which is typed loosely (`Any | dict`). The surrounding comment already documents the assumption that the message is already in Anthropic format — make that assumption explicit to pyright with a cast. --- src/pipecat/adapters/services/anthropic_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index d0fdfca3b..cbe870d81 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -248,7 +248,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): } # Fall back to assuming that the message is already in Anthropic format - return copy.deepcopy(message.message) + return cast(MessageParam, copy.deepcopy(message.message)) def _from_standard_message(self, message: LLMStandardMessage) -> MessageParam: """Convert standard universal context message to Anthropic format. From 66f43baf8f938e439e9b9c6de694f6b27c574438 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 15:31:42 -0400 Subject: [PATCH 04/23] fix: resolve pyright errors in Anthropic _from_standard_message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function takes an OpenAI ChatCompletionMessageParam (a union of TypedDicts) and returns an Anthropic MessageParam (a different TypedDict). It does the conversion via dict-level mutations that don't type-check against either side's TypedDict schema. Work with the deepcopied message as a plain dict and cast to MessageParam at the return sites — matching the boundary-cast convention noted in llm_context.py. Drops anthropic_adapter.py from 20 to 2 pyright errors. --- .../adapters/services/anthropic_adapter.py | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index cbe870d81..1632662e5 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -289,20 +289,26 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): ] } """ - message = copy.deepcopy(message) - if message["role"] == "tool": - return { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": message["tool_call_id"], - "content": message["content"], - }, - ], - } - if message.get("tool_calls"): - tc = message["tool_calls"] + # ChatCompletionMessageParam (input) and MessageParam (output) are + # different TypedDicts — work with the message as a plain dict for the + # transformations below and cast back to MessageParam at return sites. + msg = cast(dict[str, Any], copy.deepcopy(message)) + if msg["role"] == "tool": + return cast( + MessageParam, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": msg["tool_call_id"], + "content": msg["content"], + }, + ], + }, + ) + if msg.get("tool_calls"): + tc = msg["tool_calls"] ret = {"role": "assistant", "content": []} for tool_call in tc: function = tool_call["function"] @@ -314,8 +320,8 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): "input": arguments, } ret["content"].append(new_tool_use) - return ret - content = message.get("content") + return cast(MessageParam, ret) + content = msg.get("content") if isinstance(content, str): # fix empty text if content == "": @@ -363,7 +369,7 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): image_item = content.pop(img_idx) content.insert(first_txt_idx, image_item) - return message + return cast(MessageParam, msg) def _with_cache_control_markers(self, messages: list[MessageParam]) -> list[MessageParam]: """Add cache control markers to messages for prompt caching. From 9ee123bf3364284b9e751feabedad74dd8a09c52 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 15:49:39 -0400 Subject: [PATCH 05/23] fix: resolve final pyright error in Anthropic cache control marker Same MessageParam content-typing issue as the consecutive-message merge fix: pyright doesn't carry the str-to-list narrowing forward, and Iterable has no `[-1]` access. Cast to `list[Any]` and document the chain of assumptions (list, non-empty, dict-typed last item) and where each is upheld upstream. This brings anthropic_adapter.py to 0 pyright errors (down from 115). --- src/pipecat/adapters/services/anthropic_adapter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py index 1632662e5..4446ed9ca 100644 --- a/src/pipecat/adapters/services/anthropic_adapter.py +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -384,7 +384,16 @@ class AnthropicLLMAdapter(BaseLLMAdapter[AnthropicLLMInvocationParams]): def add_cache_control_marker(message: MessageParam): if isinstance(message["content"], str): message["content"] = [{"type": "text", "text": message["content"]}] - message["content"][-1]["cache_control"] = {"type": "ephemeral"} + # Assumptions on the next line: + # - content is a list (str case handled above; this codebase only + # ever constructs content as a str or a list) + # - the list is non-empty (guaranteed by the empty-content + # replacement in `_from_universal_context_messages`) + # - the last item is a dict. The standard-message path enforces + # this via TypedDicts (which are dicts at runtime); the + # LLMSpecificMessage passthrough doesn't, but in practice + # callers use dicts. + cast(list[Any], message["content"])[-1]["cache_control"] = {"type": "ephemeral"} try: # Add cache control markers to the most recent two user messages. From 5e1bb4cbe59415f5397a89a6ff96448571dc4b42 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 15:51:04 -0400 Subject: [PATCH 06/23] chore: remove anthropic_adapter.py from pyright ignore list The file is now clean under pyright's basic type checking, so it can move out of the ignore list and be enforced on every run. --- pyrightconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 8f6b951c6..f77cf5e8c 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -6,7 +6,6 @@ "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ "tests", - "src/pipecat/adapters/services/anthropic_adapter.py", "src/pipecat/adapters/services/aws_nova_sonic_adapter.py", "src/pipecat/adapters/services/bedrock_adapter.py", "src/pipecat/adapters/services/gemini_adapter.py", From 191bdc733fa13bd22e13bb54559fe83978eaec60 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 16:05:25 -0400 Subject: [PATCH 07/23] fix: conform AWSNovaSonicLLMService.get_setup_params to its protocol The service implements the NovaSonicSessionSender protocol so the session-continuation helper can target either the current or next session. The protocol declares `get_setup_params(self) -> tuple[str | None, list]`, but the implementation was unannotated and could return NotGiven in the tools position when from_standard_tools fell through to its NotGiven sentinel. Add the matching return annotation and coerce the NotGiven case to an empty list. --- src/pipecat/services/aws/nova_sonic/llm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index e16f288e1..71a80e444 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -1106,7 +1106,7 @@ class AWSNovaSonicLLMService(LLMService): ''' await self.send_event(event_json, stream) - def get_setup_params(self): + def get_setup_params(self) -> tuple[str | None, list]: """Return ``(system_instruction, tools)`` for the next session setup.""" if not self._context: return None, [] @@ -1115,7 +1115,9 @@ class AWSNovaSonicLLMService(LLMService): self._context, system_instruction=self._settings.system_instruction ) tools = ( - llm_params["tools"] if llm_params["tools"] else adapter.from_standard_tools(self._tools) + llm_params["tools"] + if llm_params["tools"] + else (adapter.from_standard_tools(self._tools) or []) ) return llm_params["system_instruction"], tools From dabca707442fdb1a17d226490c2e10baed28b4af Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 16:16:44 -0400 Subject: [PATCH 08/23] fix: warn and bail in reset_conversation when no context exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reset_conversation is part of the public AWSNovaSonicLLMService API and is also called internally from the receive-task error handler. Previously it captured `self._context` (typed `LLMContext | None`) and unconditionally passed it to `_handle_context`, which expects a real context — silently doing the wrong thing if no initial context had been received yet. Treat that as developer error: log a warning and return early. Nothing to preserve means nothing to reset. --- src/pipecat/services/aws/nova_sonic/llm.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 71a80e444..d3396cef2 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -501,12 +501,18 @@ class AWSNovaSonicLLMService(LLMService): service, and reconnects with the preserved context. """ logger.debug("Resetting conversation") - if self._assistant_is_responding: - self._assistant_is_responding = False - await self._report_assistant_response_ended() # Grab context to carry through disconnect/reconnect context = self._context + if context is None: + logger.warning( + "reset_conversation called before an initial context was received; nothing to reset" + ) + return + + if self._assistant_is_responding: + self._assistant_is_responding = False + await self._report_assistant_response_ended() await self._disconnect() await self._start_connecting() From 53ce57b7fac4d0b3246a305d106199787f7f1bdf Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 16:41:38 -0400 Subject: [PATCH 09/23] fix: tighten _process_completed_function_calls in AWS Nova Sonic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three small changes that resolve pyright errors and sharpen the logic: - Guard `self._context` with the codebase's "should never happen" early-return pattern, so we don't blindly call `.get_messages()` on None. - Skip `LLMSpecificMessage` items in the iteration. They're opaque provider-specific payloads with no `.get()`, and the surrounding logic only applies to standard tool-result messages. - Match `role == "tool"` explicitly. The previous truthy-only check was working by accident — the `tool_call_id` filter further down was effectively narrowing to tool messages, but the intent is clearer when stated upfront. --- src/pipecat/services/aws/nova_sonic/llm.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index d3396cef2..ddf430f00 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -49,7 +49,7 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage from pipecat.processors.frame_processor import FrameDirection from pipecat.services.aws.nova_sonic.session_continuation import ( SessionContinuationHelper, @@ -612,9 +612,18 @@ class AWSNovaSonicLLMService(LLMService): await self._disconnect() async def _process_completed_function_calls(self, send_new_results: bool): + if not self._context: # should never happen + return # Check for set of completed function calls in the context for message in self._context.get_messages(): - if message.get("role") and message.get("content") not in ["IN_PROGRESS", "CANCELLED"]: + # LLMSpecificMessages are opaque provider-specific payloads, not + # standard tool-result messages — skip them. + if isinstance(message, LLMSpecificMessage): + continue + if message.get("role") == "tool" and message.get("content") not in [ + "IN_PROGRESS", + "CANCELLED", + ]: tool_call_id = message.get("tool_call_id") if tool_call_id and tool_call_id not in self._completed_tool_calls: # Found a newly-completed function call - send the result to the service From d23bdaaacd94ea274becfbf6df0cf4003dfc962b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 27 Apr 2026 16:53:21 -0400 Subject: [PATCH 10/23] fix: handle NotGiven from from_standard_tools in Nova Sonic connect Same pattern as the earlier get_setup_params fix: when context tools are absent, the fallback `adapter.from_standard_tools(self._tools)` can return the NotGiven sentinel, and `_send_prompt_start_event` expects a list. Coerce via `or []` so the NotGiven case becomes an empty list. --- src/pipecat/services/aws/nova_sonic/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index ddf430f00..d3d927f57 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -654,7 +654,7 @@ class AWSNovaSonicLLMService(LLMService): tools = ( llm_connection_params["tools"] if llm_connection_params["tools"] - else adapter.from_standard_tools(self._tools) + else (adapter.from_standard_tools(self._tools) or []) ) logger.debug(f"Using tools: {tools}") await self._send_prompt_start_event(tools) From 49068ff55778f3927fc15a783f9e40ed3c1905fa Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 09:38:01 -0400 Subject: [PATCH 11/23] refactor: make LLMService generic over its adapter type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, `LLMService.get_llm_adapter()` returned `BaseLLMAdapter`, which forced every caller that wanted the precise adapter type to write `adapter: SomeAdapter = self.get_llm_adapter()` and accept pyright's complaint that the assignment doesn't match the declared type. That pattern existed in 17 places across the LLM services. Make `LLMService` generic over its adapter type — `LLMService(..., Generic[TAdapter])` with `TAdapter = TypeVar("TAdapter", bound=BaseLLMAdapter)` — so subclasses opt in via `LLMService[XAdapter]` and callers get the precise type back from `get_llm_adapter()` automatically. Backward-compatible for third-party providers: code that says `class MyService(LLMService):` (no bracket) still type-checks, with TAdapter resolving to BaseLLMAdapter from the bound — identical to the pre-refactor behavior. The `adapter_class` attribute keeps its loose `type[BaseLLMAdapter] = OpenAILLMAdapter` typing so the default remains usable; one localized cast in `__init__` bridges the loose class attr to the precise instance attr. In-tree subclasses opted in: - AnthropicLLMService -> LLMService[AnthropicLLMAdapter] - AWSBedrockLLMService -> LLMService[AWSBedrockLLMAdapter] - AWSNovaSonicLLMService -> LLMService[AWSNovaSonicLLMAdapter] - BaseOpenAILLMService -> LLMService[OpenAILLMAdapter] (propagates to ~15 OpenAI-compatible providers like Cerebras, Groq, Together) - GeminiLiveLLMService -> LLMService[GeminiLLMAdapter] - GoogleLLMService -> LLMService[GeminiLLMAdapter] - GrokRealtimeLLMService -> LLMService[GrokRealtimeLLMAdapter] - InworldRealtimeLLMService -> LLMService[InworldRealtimeLLMAdapter] - OpenAIRealtimeLLMService -> LLMService[OpenAIRealtimeLLMAdapter] - _BaseOpenAIResponsesLLMService -> LLMService[OpenAIResponsesLLMAdapter] - WebsocketLLMService is also generic so the multi-inheritance case (OpenAIResponsesLLMService) can keep both bases agreeing on TAdapter. All 17 redundant `adapter: SomeAdapter = self.get_llm_adapter()` annotations are now plain `adapter = self.get_llm_adapter()`. --- src/pipecat/services/anthropic/llm.py | 6 +++--- src/pipecat/services/aws/llm.py | 6 +++--- src/pipecat/services/aws/nova_sonic/llm.py | 6 +++--- .../services/google/gemini_live/llm.py | 12 +++++------ src/pipecat/services/google/llm.py | 2 +- src/pipecat/services/inworld/realtime/llm.py | 6 +++--- src/pipecat/services/llm_service.py | 20 +++++++++++++++---- src/pipecat/services/openai/base_llm.py | 4 ++-- src/pipecat/services/openai/realtime/llm.py | 6 +++--- src/pipecat/services/openai/responses/llm.py | 12 ++++++----- src/pipecat/services/xai/realtime/llm.py | 6 +++--- 11 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 138aa6981..c028ec7aa 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -105,7 +105,7 @@ class AnthropicLLMSettings(LLMSettings): return instance -class AnthropicLLMService(LLMService): +class AnthropicLLMService(LLMService[AnthropicLLMAdapter]): """LLM service for Anthropic's Claude models. Provides inference capabilities with Claude models including support for @@ -293,7 +293,7 @@ class AnthropicLLMService(LLMService): effective_instruction = system_instruction or assert_given( self._settings.system_instruction ) - adapter: AnthropicLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() invocation_params = adapter.get_llm_invocation_params( context, enable_prompt_caching=assert_given(self._settings.enable_prompt_caching), @@ -328,7 +328,7 @@ class AnthropicLLMService(LLMService): return next((block.text for block in response.content if hasattr(block, "text")), None) def _get_llm_invocation_params(self, context: LLMContext) -> AnthropicLLMInvocationParams: - adapter: AnthropicLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() params: AnthropicLLMInvocationParams = adapter.get_llm_invocation_params( context, enable_prompt_caching=assert_given(self._settings.enable_prompt_caching), diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 5420addf0..7266821a1 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -74,7 +74,7 @@ class AWSBedrockLLMSettings(LLMSettings): ) -class AWSBedrockLLMService(LLMService): +class AWSBedrockLLMService(LLMService[AWSBedrockLLMAdapter]): """AWS Bedrock Large Language Model service implementation. Provides inference capabilities for AWS Bedrock models including Amazon Nova @@ -282,7 +282,7 @@ class AWSBedrockLLMService(LLMService): effective_instruction = system_instruction or assert_given( self._settings.system_instruction ) - adapter: AWSBedrockLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params( context, system_instruction=effective_instruction ) @@ -371,7 +371,7 @@ class AWSBedrockLLMService(LLMService): } def _get_llm_invocation_params(self, context: LLMContext) -> AWSBedrockLLMInvocationParams: - adapter: AWSBedrockLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params( context, system_instruction=assert_given(self._settings.system_instruction) ) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index d3d927f57..55a4dd813 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -235,7 +235,7 @@ class AWSNovaSonicLLMSettings(LLMSettings): endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) -class AWSNovaSonicLLMService(LLMService): +class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): """AWS Nova Sonic speech-to-speech LLM service. Provides bidirectional audio streaming, real-time transcription, text generation, @@ -644,7 +644,7 @@ class AWSNovaSonicLLMService(LLMService): await self._process_completed_function_calls(send_new_results=False) # Read context - adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() llm_connection_params = adapter.get_llm_invocation_params( self._context, system_instruction=assert_given(self._settings.system_instruction) ) @@ -1125,7 +1125,7 @@ class AWSNovaSonicLLMService(LLMService): """Return ``(system_instruction, tools)`` for the next session setup.""" if not self._context: return None, [] - adapter: AWSNovaSonicLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() llm_params = adapter.get_llm_invocation_params( self._context, system_instruction=self._settings.system_instruction ) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 7c0c22933..d6966cbc1 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -351,7 +351,7 @@ class GeminiLiveLLMSettings(LLMSettings): proactivity: ProactivityConfig | dict | _NotGiven = field(default_factory=lambda: NOT_GIVEN) -class GeminiLiveLLMService(LLMService): +class GeminiLiveLLMService(LLMService[GeminiLLMAdapter]): """Provides access to Google's Gemini Live API. This service enables real-time conversations with Gemini, supporting both @@ -778,7 +778,7 @@ class GeminiLiveLLMService(LLMService): # init-provided values). Note that the determination of "effective" # system instruction is delegated to the adapter, which still # chooses the init-provided value if there is one. - adapter: GeminiLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() params = adapter.get_llm_invocation_params( self._context, system_instruction=assert_given(self._system_instruction_from_init) ) @@ -840,7 +840,7 @@ class GeminiLiveLLMService(LLMService): async def _process_completed_function_calls(self, send_new_results: bool): # Check for set of completed function calls in the context - adapter: GeminiLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() messages = adapter.get_llm_invocation_params(self._context).get("messages", []) for message in messages: if message.parts: @@ -1027,7 +1027,7 @@ class GeminiLiveLLMService(LLMService): # Add system instruction and tools to configuration, if provided. # These settings from the context take precedence over the ones # provided at initialization time. - adapter: GeminiLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() system_instruction = None tools = None if self._context: @@ -1333,7 +1333,7 @@ class GeminiLiveLLMService(LLMService): self._run_llm_when_session_ready = True return - adapter: GeminiLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() messages = adapter.get_llm_invocation_params(self._context).get("messages", []) if not messages: # No messages to seed convo with, so we're ready for realtime input right away @@ -1392,7 +1392,7 @@ class GeminiLiveLLMService(LLMService): # Create a throwaway context just for the purpose of getting messages # in the right format context = LLMContext(messages=messages_list) - adapter: GeminiLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() messages = adapter.get_llm_invocation_params(context).get("messages", []) if not messages: diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index faba2868b..355291e1b 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -124,7 +124,7 @@ class GoogleLLMSettings(LLMSettings): return instance -class GoogleLLMService(LLMService): +class GoogleLLMService(LLMService[GeminiLLMAdapter]): """Google AI (Gemini) LLM service implementation. This class implements inference with Google's AI models, translating internally diff --git a/src/pipecat/services/inworld/realtime/llm.py b/src/pipecat/services/inworld/realtime/llm.py index 46b025006..c859c42be 100644 --- a/src/pipecat/services/inworld/realtime/llm.py +++ b/src/pipecat/services/inworld/realtime/llm.py @@ -189,7 +189,7 @@ _NON_FATAL_ERROR_CODES = { } -class InworldRealtimeLLMService(LLMService): +class InworldRealtimeLLMService(LLMService[InworldRealtimeLLMAdapter]): """Inworld Realtime LLM service for real-time audio and text communication. Implements the Inworld Realtime API with WebSocket communication for @@ -664,7 +664,7 @@ class InworldRealtimeLLMService(LLMService): async def _send_session_update(self): """Update session settings on the server.""" settings = assert_given(self._settings.session_properties) - adapter: InworldRealtimeLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() if self._context: llm_invocation_params = adapter.get_llm_invocation_params( @@ -963,7 +963,7 @@ class InworldRealtimeLLMService(LLMService): self._run_llm_when_api_session_ready = True return - adapter: InworldRealtimeLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() if self._llm_needs_conversation_setup: logger.debug( diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 6d8caacab..9310ec78b 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -16,7 +16,10 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from typing import ( Any, + Generic, Protocol, + TypeVar, + cast, ) from loguru import logger @@ -190,7 +193,10 @@ class FunctionCallRunnerItem: group_id: str | None = None -class LLMService(UserTurnCompletionLLMServiceMixin, AIService): +TAdapter = TypeVar("TAdapter", bound=BaseLLMAdapter) + + +class LLMService(UserTurnCompletionLLMServiceMixin, AIService, Generic[TAdapter]): """Base class for all LLM services. Handles function calling registration and execution with support for both @@ -222,6 +228,7 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): """ _settings: LLMSettings + _adapter: TAdapter # OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations. # However, subclasses should override this with a more specific adapter when necessary. @@ -269,7 +276,12 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): self._filter_incomplete_user_turns: bool = False self._async_tool_cancellation_enabled: bool = False self._base_system_instruction: str | None = None - self._adapter = self.adapter_class() + # `adapter_class` is typed as `type[BaseLLMAdapter]` so subclasses + # don't need to spell out the generic parameter just to subclass + # (backward compatibility for 3rd-party providers outside this repo). + # Cast to TAdapter to keep `_adapter` and `get_llm_adapter()` precisely + # typed for callers that opt into `LLMService[XAdapter]`. + self._adapter = cast(TAdapter, self.adapter_class()) self._functions: dict[str | None, FunctionCallRegistryItem] = {} self._function_call_tasks: dict[asyncio.Task | None, FunctionCallRunnerItem] = {} self._sequential_runner_task: asyncio.Task | None = None @@ -280,7 +292,7 @@ class LLMService(UserTurnCompletionLLMServiceMixin, AIService): self._register_event_handler("on_function_calls_cancelled") self._register_event_handler("on_completion_timeout") - def get_llm_adapter(self) -> BaseLLMAdapter: + def get_llm_adapter(self) -> TAdapter: """Get the LLM adapter instance. Returns: @@ -1112,7 +1124,7 @@ class WebsocketReconnectedError(Exception): pass -class WebsocketLLMService(LLMService, WebsocketService): +class WebsocketLLMService(LLMService[TAdapter], WebsocketService, Generic[TAdapter]): """Base class for websocket-based LLM services. Each LLM inference is a discrete request/response exchange: send one diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 8f494b193..144927a8c 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -26,7 +26,7 @@ from openai._types import NotGiven as OpenAINotGiven from openai.types.chat import ChatCompletionChunk from pydantic import BaseModel, Field -from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams +from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter, OpenAILLMInvocationParams from pipecat.frames.frames import ( Frame, LLMContextFrame, @@ -71,7 +71,7 @@ class OpenAILLMSettings(LLMSettings): ) -class BaseOpenAILLMService(LLMService): +class BaseOpenAILLMService(LLMService[OpenAILLMAdapter]): """Base class for all services that use the AsyncOpenAI client. This service consumes LLMContextFrame frames, which contain a reference to diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index cf4f8943e..8519fddd8 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -194,7 +194,7 @@ class OpenAIRealtimeLLMSettings(LLMSettings): return instance -class OpenAIRealtimeLLMService(LLMService): +class OpenAIRealtimeLLMService(LLMService[OpenAIRealtimeLLMAdapter]): """OpenAI Realtime LLM service providing real-time audio and text communication. Implements the OpenAI Realtime API with WebSocket communication for low-latency @@ -657,7 +657,7 @@ class OpenAIRealtimeLLMService(LLMService): async def _send_session_update(self): settings = assert_given(self._settings.session_properties) - adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() if self._context: llm_invocation_params = adapter.get_llm_invocation_params( @@ -1002,7 +1002,7 @@ class OpenAIRealtimeLLMService(LLMService): self._run_llm_when_api_session_ready = True return - adapter: OpenAIRealtimeLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() # Configure the LLM for this session if needed if self._llm_needs_conversation_setup: diff --git a/src/pipecat/services/openai/responses/llm.py b/src/pipecat/services/openai/responses/llm.py index 6528303f9..b15e36f71 100644 --- a/src/pipecat/services/openai/responses/llm.py +++ b/src/pipecat/services/openai/responses/llm.py @@ -115,7 +115,7 @@ class OpenAIResponsesLLMSettings(LLMSettings): # --------------------------------------------------------------------------- -class _BaseOpenAIResponsesLLMService(LLMService): +class _BaseOpenAIResponsesLLMService(LLMService[OpenAIResponsesLLMAdapter]): """Shared base for HTTP and WebSocket OpenAI Responses API services. Contains settings, adapter reference, HTTP client creation, parameter @@ -294,7 +294,7 @@ class _BaseOpenAIResponsesLLMService(LLMService): Returns: The LLM's response as a string, or None if no response is generated. """ - adapter: OpenAIResponsesLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() effective_instruction = system_instruction or assert_given( self._settings.system_instruction ) @@ -353,7 +353,9 @@ class _BaseOpenAIResponsesLLMService(LLMService): # --------------------------------------------------------------------------- -class OpenAIResponsesLLMService(_BaseOpenAIResponsesLLMService, WebsocketLLMService): +class OpenAIResponsesLLMService( + _BaseOpenAIResponsesLLMService, WebsocketLLMService[OpenAIResponsesLLMAdapter] +): """OpenAI Responses API LLM service using WebSocket transport. Maintains a persistent WebSocket connection to ``wss://api.openai.com/v1/responses`` @@ -747,7 +749,7 @@ class OpenAIResponsesLLMService(_BaseOpenAIResponsesLLMService, WebsocketLLMServ if self._needs_drain: await self._drain_cancelled_response() - adapter: OpenAIResponsesLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() logger.debug( f"{self}: Generating response from universal context " f"{adapter.get_messages_for_logging(context)}" @@ -987,7 +989,7 @@ class OpenAIResponsesHttpLLMService(_BaseOpenAIResponsesLLMService): @traced_llm async def _process_context(self, context: LLMContext): - adapter: OpenAIResponsesLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() logger.debug( f"{self}: Generating response from universal context " f"{adapter.get_messages_for_logging(context)}" diff --git a/src/pipecat/services/xai/realtime/llm.py b/src/pipecat/services/xai/realtime/llm.py index 448b5e339..55a7be4cd 100644 --- a/src/pipecat/services/xai/realtime/llm.py +++ b/src/pipecat/services/xai/realtime/llm.py @@ -179,7 +179,7 @@ class GrokRealtimeLLMSettings(LLMSettings): return instance -class GrokRealtimeLLMService(LLMService): +class GrokRealtimeLLMService(LLMService[GrokRealtimeLLMAdapter]): """Grok Realtime Voice Agent LLM service providing real-time audio and text communication. Implements the Grok Voice Agent API with WebSocket communication for low-latency @@ -596,7 +596,7 @@ class GrokRealtimeLLMService(LLMService): async def _send_session_update(self): """Update session settings on the server.""" settings = assert_given(self._settings.session_properties) - adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() if self._context: llm_invocation_params = adapter.get_llm_invocation_params( @@ -871,7 +871,7 @@ class GrokRealtimeLLMService(LLMService): self._run_llm_when_api_session_ready = True return - adapter: GrokRealtimeLLMAdapter = self.get_llm_adapter() + adapter = self.get_llm_adapter() if self._llm_needs_conversation_setup: logger.debug( From c4f5f1ebbbcb8b822b981deefe14461b4036332e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 09:43:07 -0400 Subject: [PATCH 12/23] test, refactor: follow-ups to LLMService generic refactor Two follow-ups now that LLMService is generic over its adapter: - Add an explicit backward-compat test verifying that an LLMService subclass with no generic parameter (the third-party-provider pattern) instantiates and returns a usable adapter. The existing MockLLMService (declared without brackets) already exercised this implicitly, but it's worth a named assertion. - Drop the now-redundant `params: SomeLLMInvocationParams = ...` variable annotations on `adapter.get_llm_invocation_params()` results. Since `get_llm_adapter()` now returns the precise adapter type, and `BaseLLMAdapter` is generic in its invocation-params type, the call already infers the right TypedDict. --- src/pipecat/services/anthropic/llm.py | 2 +- src/pipecat/services/aws/llm.py | 4 ++-- src/pipecat/services/google/llm.py | 6 +++--- src/pipecat/services/openai/base_llm.py | 4 ++-- tests/test_llm_service.py | 21 +++++++++++++++++++++ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index c028ec7aa..cd00f15b8 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -329,7 +329,7 @@ class AnthropicLLMService(LLMService[AnthropicLLMAdapter]): def _get_llm_invocation_params(self, context: LLMContext) -> AnthropicLLMInvocationParams: adapter = self.get_llm_adapter() - params: AnthropicLLMInvocationParams = adapter.get_llm_invocation_params( + params = adapter.get_llm_invocation_params( context, enable_prompt_caching=assert_given(self._settings.enable_prompt_caching), system_instruction=assert_given(self._settings.system_instruction), diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 7266821a1..6e68c5c6d 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -283,7 +283,7 @@ class AWSBedrockLLMService(LLMService[AWSBedrockLLMAdapter]): self._settings.system_instruction ) adapter = self.get_llm_adapter() - params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params( + params = adapter.get_llm_invocation_params( context, system_instruction=effective_instruction ) messages = params["messages"] @@ -372,7 +372,7 @@ class AWSBedrockLLMService(LLMService[AWSBedrockLLMAdapter]): def _get_llm_invocation_params(self, context: LLMContext) -> AWSBedrockLLMInvocationParams: adapter = self.get_llm_adapter() - params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params( + params = adapter.get_llm_invocation_params( context, system_instruction=assert_given(self._settings.system_instruction) ) return params diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 355291e1b..da9659872 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -21,7 +21,7 @@ from loguru import logger from PIL import Image from pydantic import BaseModel, Field -from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter, GeminiLLMInvocationParams +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter from pipecat.frames.frames import ( AssistantImageRawFrame, Frame, @@ -292,7 +292,7 @@ class GoogleLLMService(LLMService[GeminiLLMAdapter]): tools = [] effective_instruction = system_instruction or self._settings.system_instruction adapter = self.get_llm_adapter() - params: GeminiLLMInvocationParams = adapter.get_llm_invocation_params( + params = adapter.get_llm_invocation_params( context, system_instruction=effective_instruction ) messages = params["messages"] @@ -387,7 +387,7 @@ class GoogleLLMService(LLMService[GeminiLLMAdapter]): async def _stream_content(self, context: LLMContext) -> AsyncIterator[GenerateContentResponse]: adapter = self.get_llm_adapter() - params: GeminiLLMInvocationParams = adapter.get_llm_invocation_params( + params = adapter.get_llm_invocation_params( context, system_instruction=assert_given(self._settings.system_instruction) ) diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 144927a8c..72d4360a9 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -297,7 +297,7 @@ class BaseOpenAILLMService(LLMService[OpenAILLMAdapter]): f"{self}: Generating chat from context {adapter.get_messages_for_logging(context)}" ) - params_from_context: OpenAILLMInvocationParams = adapter.get_llm_invocation_params( + params_from_context = adapter.get_llm_invocation_params( context, system_instruction=self._settings.system_instruction, convert_developer_to_user=not self.supports_developer_role, @@ -374,7 +374,7 @@ class BaseOpenAILLMService(LLMService[OpenAILLMAdapter]): """ effective_instruction = system_instruction or self._settings.system_instruction adapter = self.get_llm_adapter() - invocation_params: OpenAILLMInvocationParams = adapter.get_llm_invocation_params( + invocation_params = adapter.get_llm_invocation_params( context, system_instruction=effective_instruction, convert_developer_to_user=not self.supports_developer_role, diff --git a/tests/test_llm_service.py b/tests/test_llm_service.py index c7018d9f0..707c255f3 100644 --- a/tests/test_llm_service.py +++ b/tests/test_llm_service.py @@ -7,6 +7,8 @@ import unittest from unittest.mock import AsyncMock, patch +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter from pipecat.frames.frames import ( FunctionCallFromLLM, FunctionCallInProgressFrame, @@ -39,6 +41,25 @@ class MockLLMService(LLMService): super().__init__(settings=settings, **kwargs) +class TestUnparameterizedSubclass(unittest.TestCase): + """Backward-compat coverage: third-party providers subclass LLMService + without specifying a generic adapter parameter. That should keep working + after LLMService became `Generic[TAdapter]`. + """ + + def test_unparameterized_subclass_instantiates(self): + # MockLLMService is declared as `class MockLLMService(LLMService):` + # — no generic bracket. The TypeVar's `bound=BaseLLMAdapter` should + # resolve TAdapter to BaseLLMAdapter for callers that don't opt in. + service = MockLLMService() + adapter = service.get_llm_adapter() + + # Default adapter_class is OpenAILLMAdapter; the runtime instance + # should reflect that, regardless of how generics are erased. + self.assertIsInstance(adapter, OpenAILLMAdapter) + self.assertIsInstance(adapter, BaseLLMAdapter) + + class TestLLMService(unittest.IsolatedAsyncioTestCase): async def _run_function_calls_inline(self, service: MockLLMService): async def run_inline(runner_items): From 1cd73b1ef82d68bc23fdce98e5e24459e8d8b49c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 10:21:25 -0400 Subject: [PATCH 13/23] refactor: give TAdapter a default to restore precise typing for unparameterized LLMService subclasses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After making LLMService generic, an unparameterized subclass (`class MyService(LLMService):` with no bracket — the third-party provider pattern) saw `get_llm_adapter()` return `Unknown` rather than `BaseLLMAdapter` as it did before the refactor. Add `default=BaseLLMAdapter` (PEP 696) on the TypeVar — via `typing_extensions.TypeVar` so older Python targets keep working — so unparameterized callers get `LLMService[BaseLLMAdapter]` and `get_llm_adapter()` returns `BaseLLMAdapter`, matching the pre-refactor type precision. Two internal fallouts of having a default (where the default makes unannotated `LLMService` resolve invariantly to `LLMService[BaseLLMAdapter]`): - `FunctionCallParams.llm` is now `LLMService[Any]` so concrete parameterizations like `LLMService[OpenAILLMAdapter]` can be passed where the field is set. - The explicit `LLMService.__init__(self, **kwargs)` in `WebsocketLLMService.__init__` gets a `pyright: ignore[reportArgumentType]` comment — pyright's invariance handling can't see through the multi-inheritance + generic + default combination, but the runtime call is correct (generics are erased). --- src/pipecat/services/llm_service.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 9310ec78b..77c1412a5 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -18,11 +18,11 @@ from typing import ( Any, Generic, Protocol, - TypeVar, cast, ) from loguru import logger +from typing_extensions import TypeVar from websockets.exceptions import ConnectionClosed from websockets.protocol import State @@ -119,7 +119,12 @@ class FunctionCallParams: function_name: str tool_call_id: str arguments: Mapping[str, Any] - llm: LLMService + # `LLMService[Any]` so any concrete subclass (regardless of how — or + # whether — it parameterizes the adapter type) can be assigned here. + # Plain `LLMService` would invoke the TypeVar default and pyright would + # treat it invariantly, rejecting `LLMService[XAdapter]` at the call + # sites that build FunctionCallParams. + llm: LLMService[Any] context: LLMContext result_callback: FunctionCallResultCallback app_resources: Any = None @@ -193,7 +198,11 @@ class FunctionCallRunnerItem: group_id: str | None = None -TAdapter = TypeVar("TAdapter", bound=BaseLLMAdapter) +# `default=BaseLLMAdapter` (PEP 696) so that unparameterized subclasses +# (e.g. third-party `class MyService(LLMService):` with no bracket) get +# `TAdapter = BaseLLMAdapter` instead of `Unknown` at type-check time — +# matching the pre-generic behavior of `get_llm_adapter()`. +TAdapter = TypeVar("TAdapter", bound=BaseLLMAdapter, default=BaseLLMAdapter) class LLMService(UserTurnCompletionLLMServiceMixin, AIService, Generic[TAdapter]): @@ -1172,7 +1181,11 @@ class WebsocketLLMService(LLMService[TAdapter], WebsocketService, Generic[TAdapt reconnect_on_error: Whether to automatically reconnect on websocket errors. **kwargs: Additional arguments passed to parent classes. """ - LLMService.__init__(self, **kwargs) + # pyright stumbles here because the TypeVar default makes + # `LLMService` resolve to `LLMService[BaseLLMAdapter]` invariantly, + # while `self` is `WebsocketLLMService[TAdapter]` for an arbitrary + # TAdapter. The runtime call is fine — generics are erased. + LLMService.__init__(self, **kwargs) # pyright: ignore[reportArgumentType] WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) self._register_event_handler("on_connection_error") From bec407ce3a3011827b3939248419c76b3ece0fe9 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 11:57:54 -0400 Subject: [PATCH 14/23] fix: handle Optional websocket/client receivers across services MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pyright flagged 19 sites where `await self._.send/recv/...` was called on a receiver typed `X | None`. Each kind of call site needed a slightly different fix to be both type-safe and behaviour- preserving: Streaming/user-facing paths (early return + warn — drop and warn is the right runtime fail-safe when reconnect didn't succeed): - cartesia/stt.py (run_stt) - soniox/stt.py (_send_keepalive) - elevenlabs/tts.py (run_tts — yields ErrorFrame and returns) - deepgram/sagemaker/tts.py (run_tts) - transports/lemonslice/transport.py (send_message) - transports/tavus/transport.py (send_message) "Should never happen" cases (early return with comment, no warn — caller already gated on a separate `_is_*` check, so a warn would be noise): - deepgram/flux/stt.py (transport methods, gated by _transport_is_active) - deepgram/flux/sagemaker/stt.py (same) - stt_service.py (_send_keepalive, gated by _is_keepalive_ready) - elevenlabs/stt.py (_send_keepalive, same) - llm_service.py (_ws_recv — raises ConnectionError to match _ensure_connected's contract) - heygen/client.py (receive loop, gated by self._connected) Just-assigned-above (use a local variable so pyright keeps the narrowing across statements): - lmnt/tts.py - gradium/stt.py - fish/tts.py Other: - transports/websocket/server.py — used the existing local `websocket` parameter in scope instead of `self._websocket` for the close call. - websocket_service.py — `send_with_retry` raises ConnectionError when `self._websocket` is None inside the existing try-block, so the broad `except Exception` triggers reconnect just as it would on a real send failure (preserving the prior behaviour where None silently fell through to the AttributeError-driven reconnect path). Drops three now-clean files from the pyright ignore list: cartesia/stt.py, elevenlabs/stt.py, and soniox/stt.py. --- pyrightconfig.json | 3 --- src/pipecat/services/cartesia/stt.py | 5 +++++ src/pipecat/services/deepgram/flux/sagemaker/stt.py | 8 ++++++++ src/pipecat/services/deepgram/flux/stt.py | 8 ++++++++ src/pipecat/services/deepgram/sagemaker/tts.py | 4 ++++ src/pipecat/services/elevenlabs/stt.py | 4 ++++ src/pipecat/services/elevenlabs/tts.py | 5 +++++ src/pipecat/services/fish/tts.py | 5 +++-- src/pipecat/services/gradium/stt.py | 7 ++++--- src/pipecat/services/heygen/client.py | 2 ++ src/pipecat/services/llm_service.py | 5 +++++ src/pipecat/services/lmnt/tts.py | 5 +++-- src/pipecat/services/soniox/stt.py | 3 +++ src/pipecat/services/stt_service.py | 4 ++++ src/pipecat/services/websocket_service.py | 7 ++++++- src/pipecat/transports/lemonslice/transport.py | 3 +++ src/pipecat/transports/tavus/transport.py | 4 ++++ src/pipecat/transports/websocket/server.py | 2 +- 18 files changed, 72 insertions(+), 12 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index f77cf5e8c..9370d4387 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -40,14 +40,12 @@ "src/pipecat/services/aws/utils.py", "src/pipecat/services/azure/stt.py", "src/pipecat/services/azure/tts.py", - "src/pipecat/services/cartesia/stt.py", "src/pipecat/services/deepgram/flux/base.py", "src/pipecat/services/deepgram/flux/sagemaker/stt.py", "src/pipecat/services/deepgram/flux/stt.py", "src/pipecat/services/deepgram/sagemaker/stt.py", "src/pipecat/services/deepgram/sagemaker/tts.py", "src/pipecat/services/deepgram/tts.py", - "src/pipecat/services/elevenlabs/stt.py", "src/pipecat/services/elevenlabs/tts.py", "src/pipecat/services/fish/tts.py", "src/pipecat/services/gladia/stt.py", @@ -92,7 +90,6 @@ "src/pipecat/services/sarvam/tts.py", "src/pipecat/services/simli/video.py", "src/pipecat/services/smallest/tts.py", - "src/pipecat/services/soniox/stt.py", "src/pipecat/services/speechmatics/stt.py", "src/pipecat/services/stt_service.py", "src/pipecat/services/tavus/video.py", diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 018d95e6a..af3418edd 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -290,6 +290,11 @@ class CartesiaSTTService(WebsocketSTTService): if not self._websocket or self._websocket.state is not State.OPEN: await self._connect() + if self._websocket is None: + logger.warning(f"{self}: websocket unavailable after reconnect, dropping audio") + yield None + return + try: await self._websocket.send(audio) except Exception as e: diff --git a/src/pipecat/services/deepgram/flux/sagemaker/stt.py b/src/pipecat/services/deepgram/flux/sagemaker/stt.py index 1b110cf1e..0f7a6fded 100644 --- a/src/pipecat/services/deepgram/flux/sagemaker/stt.py +++ b/src/pipecat/services/deepgram/flux/sagemaker/stt.py @@ -145,9 +145,17 @@ class DeepgramFluxSageMakerSTTService(DeepgramFluxSTTBase): # ------------------------------------------------------------------ async def _transport_send_audio(self, audio: bytes): + if ( + self._client is None + ): # should never happen — caller should gate on _transport_is_active() + return await self._client.send_audio_chunk(audio) async def _transport_send_json(self, message: dict): + if ( + self._client is None + ): # should never happen — caller should gate on _transport_is_active() + return await self._client.send_json(message) def _transport_is_active(self) -> bool: diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index df14442eb..98cf17ec2 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -240,9 +240,17 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): # ------------------------------------------------------------------ async def _transport_send_audio(self, audio: bytes): + if ( + self._websocket is None + ): # should never happen — caller should gate on _transport_is_active() + return await self._websocket.send(audio) async def _transport_send_json(self, message: dict): + if ( + self._websocket is None + ): # should never happen — caller should gate on _transport_is_active() + return await self._websocket.send(json.dumps(message)) def _transport_is_active(self) -> bool: diff --git a/src/pipecat/services/deepgram/sagemaker/tts.py b/src/pipecat/services/deepgram/sagemaker/tts.py index 171da06cf..b0c24e117 100644 --- a/src/pipecat/services/deepgram/sagemaker/tts.py +++ b/src/pipecat/services/deepgram/sagemaker/tts.py @@ -337,6 +337,10 @@ class DeepgramSageMakerTTSService(TTSService): the response processor). """ logger.debug(f"{self}: Generating TTS [{text}]") + if self._client is None: + logger.warning(f"{self}: client unavailable, skipping TTS") + yield ErrorFrame(error="client unavailable") + return try: await self._client.send_json({"type": "Speak", "text": text}) yield None diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 7f65fd90b..5571f8700 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -739,6 +739,10 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService): Args: silence: Silent 16-bit mono PCM audio bytes. """ + if ( + self._websocket is None + ): # should never happen — caller should gate on _is_keepalive_ready() + return audio_base64 = base64.b64encode(silence).decode("utf-8") message = { "message_type": "input_audio_chunk", diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 88254c0c3..4da94643c 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -905,6 +905,11 @@ class ElevenLabsTTSService(WebsocketTTSService): if not self._websocket or self._websocket.state is State.CLOSED: await self._connect() + if self._websocket is None: + logger.warning(f"{self}: websocket unavailable after reconnect, skipping TTS") + yield ErrorFrame(error="websocket unavailable") + return + try: if not self.audio_context_available(context_id): await self.create_audio_context(context_id) diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 83512da45..327e650ec 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -281,7 +281,8 @@ class FishAudioTTSService(InterruptibleTTSService): model = assert_given(self._settings.model) if model is not None: headers["model"] = model - self._websocket = await websocket_connect(self._base_url, additional_headers=headers) + websocket = await websocket_connect(self._base_url, additional_headers=headers) + self._websocket = websocket # Send initial start message with ormsgpack request_settings = { @@ -300,7 +301,7 @@ class FishAudioTTSService(InterruptibleTTSService): if self._settings.top_p is not None: request_settings["top_p"] = self._settings.top_p start_message = {"event": "start", "request": {"text": "", **request_settings}} - await self._websocket.send(ormsgpack.packb(start_message)) + await websocket.send(ormsgpack.packb(start_message)) logger.debug("Sent start message to Fish Audio") await self._call_event_handler("on_connected") diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py index 0b6a81cdd..6f1a48f60 100644 --- a/src/pipecat/services/gradium/stt.py +++ b/src/pipecat/services/gradium/stt.py @@ -383,10 +383,11 @@ class GradiumSTTService(WebsocketSTTService): "x-api-key": self._api_key, "x-api-source": "pipecat", } - self._websocket = await websocket_connect( + websocket = await websocket_connect( ws_url, additional_headers=headers, ) + self._websocket = websocket await self._call_event_handler("on_connected") setup_msg = { "type": "setup", @@ -406,8 +407,8 @@ class GradiumSTTService(WebsocketSTTService): json_config["delay_in_frames"] = self._settings.delay_in_frames if json_config: setup_msg["json_config"] = json_config - await self._websocket.send(json.dumps(setup_msg)) - ready_msg = await self._websocket.recv() + await websocket.send(json.dumps(setup_msg)) + ready_msg = await websocket.recv() ready_msg = json.loads(ready_msg) if ready_msg["type"] == "error": raise Exception(f"received error {ready_msg['message']}") diff --git a/src/pipecat/services/heygen/client.py b/src/pipecat/services/heygen/client.py index a02d515a7..80e5adb3f 100644 --- a/src/pipecat/services/heygen/client.py +++ b/src/pipecat/services/heygen/client.py @@ -291,6 +291,8 @@ class HeyGenClient: """Handle incoming WebSocket messages.""" try: while self._connected: + if self._websocket is None: # should never happen while _connected is True + break try: message = await self._websocket.recv() parsed_message = json.loads(message) diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 77c1412a5..95bf2c762 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -1265,6 +1265,11 @@ class WebsocketLLMService(LLMService[TAdapter], WebsocketService, Generic[TAdapt Returns: The parsed JSON message as a dict. """ + # Should never happen — `_ensure_connected` (which callers must invoke + # first) raises ConnectionError if it can't establish a websocket. + # Match that contract here. + if self._websocket is None: + raise ConnectionError(f"{self} _ws_recv called without a websocket") try: raw = await self._websocket.recv() return json.loads(raw) diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index b098b43c2..81dffac7e 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -267,10 +267,11 @@ class LmntTTSService(InterruptibleTTSService): } # Connect to LMNT's websocket directly - self._websocket = await websocket_connect("wss://api.lmnt.com/v1/ai/speech/stream") + websocket = await websocket_connect("wss://api.lmnt.com/v1/ai/speech/stream") + self._websocket = websocket # Send initialization message - await self._websocket.send(json.dumps(init_msg)) + await websocket.send(json.dumps(init_msg)) await self._call_event_handler("on_connected") except Exception as e: diff --git a/src/pipecat/services/soniox/stt.py b/src/pipecat/services/soniox/stt.py index 1732b3f91..ca99e69c2 100644 --- a/src/pipecat/services/soniox/stt.py +++ b/src/pipecat/services/soniox/stt.py @@ -648,4 +648,7 @@ class SonioxSTTService(WebsocketSTTService): Args: silence: Silent PCM audio bytes (unused, Soniox uses a protocol message). """ + if self._websocket is None: + logger.warning(f"{self}: websocket unavailable, skipping keepalive") + return await self._websocket.send(KEEPALIVE_MESSAGE) diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index b72547e1a..7c6028ea1 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -859,6 +859,10 @@ class WebsocketSTTService(STTService, WebsocketService): Args: silence: Silent 16-bit mono PCM audio bytes. """ + if ( + self._websocket is None + ): # should never happen — caller should gate on _is_keepalive_ready() + return await self._websocket.send(silence) async def _report_error(self, error: ErrorFrame): diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 83ddb3746..2e6040ead 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -120,12 +120,17 @@ class WebsocketService(ABC): async def send_with_retry(self, message, report_error: Callable[[ErrorFrame], Awaitable[None]]): """Attempt to send a message, retrying after reconnect if necessary.""" try: + # If websocket isn't connected/present, treat as a send failure — + # the broad `except Exception` below will trigger a reconnect + # attempt. + if self._websocket is None: + raise ConnectionError(f"{self} no websocket connected") await self._websocket.send(message) except Exception as e: logger.error(f"{self} send failed: {e}, will try to reconnect") # Try to reconnect before retrying success = await self._try_reconnect(report_error=report_error) - if success: + if success and self._websocket is not None: logger.info(f"{self} reconnected successfully, will retry send the message") # trying to send the message one more time await self._websocket.send(message) diff --git a/src/pipecat/transports/lemonslice/transport.py b/src/pipecat/transports/lemonslice/transport.py index 90966ade2..83d8c0e9e 100644 --- a/src/pipecat/transports/lemonslice/transport.py +++ b/src/pipecat/transports/lemonslice/transport.py @@ -312,6 +312,9 @@ class LemonSliceTransportClient: Args: frame: The message frame to send. """ + if self._daily_transport_client is None: + return + await self._daily_transport_client.send_message(frame) @property diff --git a/src/pipecat/transports/tavus/transport.py b/src/pipecat/transports/tavus/transport.py index 9ad28979d..698d0dc95 100644 --- a/src/pipecat/transports/tavus/transport.py +++ b/src/pipecat/transports/tavus/transport.py @@ -360,6 +360,9 @@ class TavusTransportClient: Args: frame: The message frame to send. """ + if self._client is None: + return + await self._client.send_message(frame) @property @@ -416,6 +419,7 @@ class TavusTransportClient: """ if not self._client: return False + return await self._client.write_audio_frame(frame) async def register_audio_destination(self, destination: str, auto_silence: bool | None = True): diff --git a/src/pipecat/transports/websocket/server.py b/src/pipecat/transports/websocket/server.py index 82b899f71..0a78f7504 100644 --- a/src/pipecat/transports/websocket/server.py +++ b/src/pipecat/transports/websocket/server.py @@ -226,7 +226,7 @@ class WebsocketServerInputTransport(BaseInputTransport): # Notify disconnection await self._callbacks.on_client_disconnected(websocket) - await self._websocket.close() + await websocket.close() self._websocket = None logger.info(f"Client {websocket.remote_address} disconnected") From 2a731336beda6c7d87af2fe6254c41a790aa0d9e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 16:02:47 -0400 Subject: [PATCH 15/23] fix: tighten language_to__language return types to plain str MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These provider-specific helpers are all thin wrappers around `resolve_language(...)`, which itself returns `str` — never `None`. The `str | None` annotations were misleading and were producing spurious pyright errors at the call sites that assigned the result into a `str` field. Update each helper's signature to `str` and rewrite the `Returns:` docstring to describe the actual fallback behaviour (resolve to base or full code, with a warning). Importantly, the per-class `language_to_service_language(...)` methods on `STTService`/`TTSService` subclasses keep `str | None` as their return type. That signature is an extension hook for future and/or third-party subclasses that may genuinely not be able to produce a code for some languages, even though all in-tree first- party services currently return a string. Also includes one small unrelated tightening in azure/stt.py: wrap `self._settings.language` with `assert_given(...)` so the truthy fallback to `language_to_azure_language(Language.EN_US)` doesn't silently swallow a NotGiven sentinel. Net: -3 pyright errors (full-config run: 785 -> 782). --- src/pipecat/services/asyncai/tts.py | 7 +++++-- src/pipecat/services/aws/tts.py | 6 ++++-- src/pipecat/services/azure/common.py | 6 ++++-- src/pipecat/services/azure/stt.py | 6 +++--- src/pipecat/services/camb/tts.py | 7 +++++-- src/pipecat/services/cartesia/tts.py | 7 +++++-- src/pipecat/services/deepgram/flux/base.py | 2 +- src/pipecat/services/elevenlabs/stt.py | 6 ++++-- src/pipecat/services/elevenlabs/tts.py | 7 +++++-- src/pipecat/services/fal/stt.py | 7 +++++-- src/pipecat/services/gladia/stt.py | 9 ++++++--- src/pipecat/services/google/gemini_live/llm.py | 6 ++++-- src/pipecat/services/google/stt.py | 7 +++++-- src/pipecat/services/google/tts.py | 12 ++++++++---- src/pipecat/services/gradium/stt.py | 7 +++++-- src/pipecat/services/lmnt/tts.py | 7 +++++-- src/pipecat/services/minimax/tts.py | 6 ++++-- src/pipecat/services/neuphonic/tts.py | 7 +++++-- src/pipecat/services/nvidia/stt.py | 6 ++++-- src/pipecat/services/sarvam/tts.py | 6 ++++-- src/pipecat/services/smallest/tts.py | 7 +++++-- src/pipecat/services/stt_service.py | 5 ++++- src/pipecat/services/tts_service.py | 5 ++++- src/pipecat/services/whisper/base_stt.py | 9 ++++++--- src/pipecat/services/whisper/stt.py | 9 ++++++--- src/pipecat/services/xai/stt.py | 7 +++++-- src/pipecat/services/xai/tts.py | 7 +++++-- src/pipecat/services/xtts/tts.py | 7 +++++-- 28 files changed, 131 insertions(+), 59 deletions(-) diff --git a/src/pipecat/services/asyncai/tts.py b/src/pipecat/services/asyncai/tts.py index 0f72b3b9c..1923bb035 100644 --- a/src/pipecat/services/asyncai/tts.py +++ b/src/pipecat/services/asyncai/tts.py @@ -42,14 +42,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_async_language(language: Language) -> str | None: +def language_to_async_language(language: Language) -> str: """Convert a Language enum to Async language code. Args: language: The Language enum value to convert. Returns: - The corresponding Async language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.EN: "en", diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index d49aa796a..a683a94ea 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -37,14 +37,16 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_aws_language(language: Language) -> str | None: +def language_to_aws_language(language: Language) -> str: """Convert a Language enum to AWS Polly language code. Args: language: The Language enum value to convert. Returns: - The corresponding AWS Polly language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Arabic diff --git a/src/pipecat/services/azure/common.py b/src/pipecat/services/azure/common.py index 8bb48cd04..c2f21f1b1 100644 --- a/src/pipecat/services/azure/common.py +++ b/src/pipecat/services/azure/common.py @@ -9,14 +9,16 @@ from pipecat.transcriptions.language import Language, resolve_language -def language_to_azure_language(language: Language) -> str | None: +def language_to_azure_language(language: Language) -> str: """Convert a Language enum to Azure language code. Args: language: The Language enum value to convert. Returns: - The corresponding Azure language code, or None if not supported. + The corresponding Azure language code. If ``language`` is not in the + verified mapping, falls back to the full language code string and logs + a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Afrikaans diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index 72a8f76b0..e130fe980 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -182,9 +182,9 @@ class AzureSTTService(STTService): changed = await super()._update_settings(delta) if "language" in changed: - self._speech_config.speech_recognition_language = ( - self._settings.language or language_to_azure_language(Language.EN_US) - ) + self._speech_config.speech_recognition_language = assert_given( + self._settings.language + ) or language_to_azure_language(Language.EN_US) if self._audio_stream: await self._disconnect() await self._connect() diff --git a/src/pipecat/services/camb/tts.py b/src/pipecat/services/camb/tts.py index 7b3aab768..2449a13dd 100644 --- a/src/pipecat/services/camb/tts.py +++ b/src/pipecat/services/camb/tts.py @@ -44,14 +44,17 @@ MODEL_SAMPLE_RATES: dict[str, int] = { } -def language_to_camb_language(language: Language) -> str | None: +def language_to_camb_language(language: Language) -> str: """Convert a Pipecat Language enum to Camb.ai language code. Args: language: The Language enum value to convert. Returns: - The corresponding Camb.ai language code (BCP-47 format), or None if not supported. + The corresponding Camb.ai language code (BCP-47 format). If + ``language`` is not in the verified mapping, falls back to the base + language code (e.g., ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.EN: "en-us", diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 6b3fc4b83..d88f462c5 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -62,14 +62,17 @@ class GenerationConfig(BaseModel): emotion: str | None = None -def language_to_cartesia_language(language: Language) -> str | None: +def language_to_cartesia_language(language: Language) -> str: """Convert a Language enum to Cartesia language code. Args: language: The Language enum value to convert. Returns: - The corresponding Cartesia language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AR: "ar", diff --git a/src/pipecat/services/deepgram/flux/base.py b/src/pipecat/services/deepgram/flux/base.py index 1d8ad4794..f2042d799 100644 --- a/src/pipecat/services/deepgram/flux/base.py +++ b/src/pipecat/services/deepgram/flux/base.py @@ -32,7 +32,7 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt -def language_to_deepgram_flux_language(language: Language) -> str | None: +def language_to_deepgram_flux_language(language: Language) -> str: """Convert a Pipecat Language to a Deepgram Flux language code. Only honored by the ``flux-general-multi`` model. Locale variants diff --git a/src/pipecat/services/elevenlabs/stt.py b/src/pipecat/services/elevenlabs/stt.py index 5571f8700..6fed6c5e5 100644 --- a/src/pipecat/services/elevenlabs/stt.py +++ b/src/pipecat/services/elevenlabs/stt.py @@ -54,7 +54,7 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_elevenlabs_language(language: Language) -> str | None: +def language_to_elevenlabs_language(language: Language) -> str: """Convert a Language enum to ElevenLabs language code. Source: @@ -64,7 +64,9 @@ def language_to_elevenlabs_language(language: Language) -> str | None: language: The Language enum value to convert. Returns: - The corresponding ElevenLabs language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { Language.AF: "afr", # Afrikaans diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 4da94643c..239e6a64e 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -69,14 +69,17 @@ ELEVENLABS_MULTILINGUAL_MODELS = { } -def language_to_elevenlabs_language(language: Language) -> str | None: +def language_to_elevenlabs_language(language: Language) -> str: """Convert a Language enum to ElevenLabs language code. Args: language: The Language enum value to convert. Returns: - The corresponding ElevenLabs language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AR: "ar", diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 1e18c7f84..5aa16c09c 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -28,14 +28,17 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt -def language_to_fal_language(language: Language) -> str | None: +def language_to_fal_language(language: Language) -> str: """Convert a Language enum to Fal's Wizper language code. Args: language: The Language enum value to convert. Returns: - The corresponding Fal Wizper language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AF: "af", diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index ec3277943..cc8e5279b 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -56,14 +56,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_gladia_language(language: Language) -> str | None: +def language_to_gladia_language(language: Language) -> str: """Convert a Language enum to Gladia's language code format. Args: language: The Language enum value to convert. Returns: - The Gladia language code string or None if not supported. + The corresponding Gladia language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AF: "af", @@ -361,7 +364,7 @@ class GladiaSTTService(WebsocketSTTService): language: The Language enum value to convert. Returns: - The Gladia language code string or None if not supported. + The Gladia language code string, or None if not supported. """ return language_to_gladia_language(language) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index d6966cbc1..adcbe28b8 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -111,7 +111,7 @@ MAX_CONSECUTIVE_FAILURES = 3 CONNECTION_ESTABLISHED_THRESHOLD = 10.0 # seconds -def language_to_gemini_language(language: Language) -> str | None: +def language_to_gemini_language(language: Language) -> str: """Maps a Language enum value to a Gemini Live supported language code. Source: @@ -121,7 +121,9 @@ def language_to_gemini_language(language: Language) -> str | None: language: The language enum value to convert. Returns: - The Gemini language code string, or None if the language is not supported. + The Gemini language code string. If ``language`` is not in the + verified mapping, falls back to the full language code string and logs + a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Arabic diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index d9fb0a160..a386566c5 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -60,14 +60,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_google_stt_language(language: Language) -> str | None: +def language_to_google_stt_language(language: Language) -> str: """Maps Language enum to Google Speech-to-Text V2 language codes. Args: language: Language enum value. Returns: - Optional[str]: Google STT language code or None if not supported. + The corresponding Google STT language code. If ``language`` is not + in the verified mapping, falls back to the full language code string + and logs a warning (via + ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Afrikaans diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index b507fc26f..58bc31691 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -60,7 +60,7 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_google_tts_language(language: Language) -> str | None: +def language_to_google_tts_language(language: Language) -> str: """Convert a Language enum to Google TTS language code. Source: @@ -70,7 +70,9 @@ def language_to_google_tts_language(language: Language) -> str | None: language: The Language enum value to convert. Returns: - The corresponding Google TTS language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Arabic @@ -219,7 +221,7 @@ def language_to_google_tts_language(language: Language) -> str | None: return resolve_language(language, LANGUAGE_MAP, use_base_code=False) -def language_to_gemini_tts_language(language: Language) -> str | None: +def language_to_gemini_tts_language(language: Language) -> str: """Convert a Language enum to Gemini TTS language code. Source: @@ -229,7 +231,9 @@ def language_to_gemini_tts_language(language: Language) -> str | None: language: The Language enum value to convert. Returns: - The corresponding Gemini TTS language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Afrikaans (Preview) diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py index 6f1a48f60..e58383b45 100644 --- a/src/pipecat/services/gradium/stt.py +++ b/src/pipecat/services/gradium/stt.py @@ -79,14 +79,17 @@ def _input_format_from_encoding(encoding: str, sample_rate: int) -> str: return encoding -def language_to_gradium_language(language: Language) -> str | None: +def language_to_gradium_language(language: Language) -> str: """Convert a Language enum to Gradium's language code format. Args: language: The Language enum value to convert. Returns: - The Gradium language code string or None if not supported. + The corresponding Gradium language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.DE: "de", diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index 81dffac7e..7972780fc 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -37,14 +37,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_lmnt_language(language: Language) -> str | None: +def language_to_lmnt_language(language: Language) -> str: """Convert a Language enum to LMNT language code. Args: language: The Language enum value to convert. Returns: - The corresponding LMNT language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AR: "ar", diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 25b7feade..3ef1b49c3 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -31,14 +31,16 @@ from pipecat.transcriptions.language import Language, resolve_language from pipecat.utils.tracing.service_decorators import traced_tts -def language_to_minimax_language(language: Language) -> str | None: +def language_to_minimax_language(language: Language) -> str: """Convert a Language enum to MiniMax language format. Args: language: The Language enum value to convert. Returns: - The corresponding MiniMax language name, or None if not supported. + The corresponding MiniMax language name. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { Language.AF: "Afrikaans", diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index a7303da6b..6a405a1d2 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -44,14 +44,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_neuphonic_lang_code(language: Language) -> str | None: +def language_to_neuphonic_lang_code(language: Language) -> str: """Convert a Language enum to Neuphonic language code. Args: language: The Language enum value to convert. Returns: - The corresponding Neuphonic language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.DE: "de", diff --git a/src/pipecat/services/nvidia/stt.py b/src/pipecat/services/nvidia/stt.py index 49fb4573e..d7064c04c 100644 --- a/src/pipecat/services/nvidia/stt.py +++ b/src/pipecat/services/nvidia/stt.py @@ -49,7 +49,7 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_nvidia_nemotron_speech_language(language: Language) -> str | None: +def language_to_nvidia_nemotron_speech_language(language: Language) -> str: """Maps Language enum to NVIDIA Nemotron Speech ASR language codes. Source: @@ -59,7 +59,9 @@ def language_to_nvidia_nemotron_speech_language(language: Language) -> str | Non language: Language enum value. Returns: - str | None: NVIDIA Nemotron Speech language code or None if not supported. + The NVIDIA Nemotron Speech language code. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { # Arabic diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index 1ab69ef15..100df80af 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -216,14 +216,16 @@ def get_speakers_for_model(model: str) -> list[str]: return list(TTS_MODEL_CONFIGS["bulbul:v2"].speakers) -def language_to_sarvam_language(language: Language) -> str | None: +def language_to_sarvam_language(language: Language) -> str: """Convert Pipecat Language enum to Sarvam AI language codes. Args: language: The Language enum value to convert. Returns: - The corresponding Sarvam AI language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the full language code string and + logs a warning (via ``resolve_language(..., use_base_code=False)``). """ LANGUAGE_MAP = { Language.BN: "bn-IN", # Bengali diff --git a/src/pipecat/services/smallest/tts.py b/src/pipecat/services/smallest/tts.py index 6a2071f44..80f98cb54 100644 --- a/src/pipecat/services/smallest/tts.py +++ b/src/pipecat/services/smallest/tts.py @@ -51,14 +51,17 @@ class SmallestTTSModel(StrEnum): LIGHTNING_V3_1 = "lightning-v3.1" -def language_to_smallest_tts_language(language: Language) -> str | None: +def language_to_smallest_tts_language(language: Language) -> str: """Convert a Language enum to a Smallest TTS language string. Args: language: The Language enum value to convert. Returns: - The Smallest language code string, or None if unsupported. + The corresponding Smallest language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AR: "ar", diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index 7c6028ea1..e54581919 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -269,7 +269,10 @@ class STTService(AIService): language: The language to convert. Returns: - The service-specific language identifier, or None if not supported. + The service-specific language identifier. Return ``None`` to + indicate an unsupported language. This optional return is an + extension hook for future or third-party subclasses; as of + 2026-04-28, first-party services return a string. """ return Language(language) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 46c5c7528..f386b5bcb 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -467,7 +467,10 @@ class TTSService(AIService): language: The language to convert. Returns: - The service-specific language identifier, or None if not supported. + The service-specific language identifier. Return ``None`` to + indicate an unsupported language. This optional return is an + extension hook for future or third-party subclasses; as of + 2026-04-28, first-party services return a string. """ return Language(language) diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index 9ac84c41c..6a6bd6b12 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -40,7 +40,7 @@ class BaseWhisperSTTSettings(STTSettings): temperature: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN) -def language_to_whisper_language(language: Language) -> str | None: +def language_to_whisper_language(language: Language) -> str: """Maps pipecat Language enum to Whisper API language codes. Language support for Whisper API. @@ -50,7 +50,10 @@ def language_to_whisper_language(language: Language) -> str | None: language: A Language enum value representing the input language. Returns: - str or None: The corresponding Whisper language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AF: "af", @@ -235,7 +238,7 @@ class BaseWhisperSTTService(SegmentedSTTService): language: The Language enum value to convert. Returns: - str or None: The corresponding service language code, or None if not supported. + The corresponding service language code, or None if not supported. """ return language_to_whisper_language(language) diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index 47f896056..443664348 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -97,14 +97,17 @@ class MLXModel(Enum): LARGE_V3_TURBO_Q4 = "mlx-community/whisper-large-v3-turbo-q4" -def language_to_whisper_language(language: Language) -> str | None: +def language_to_whisper_language(language: Language) -> str: """Maps pipecat Language enum to Whisper language codes. Args: language: A Language enum value representing the input language. Returns: - str or None: The corresponding Whisper language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). Note: Only includes languages officially supported by Whisper. @@ -300,7 +303,7 @@ class WhisperSTTService(SegmentedSTTService): language: The Language enum value to convert. Returns: - str or None: The corresponding Whisper language code, or None if not supported. + The corresponding Whisper language code, or None if not supported. """ return language_to_whisper_language(language) diff --git a/src/pipecat/services/xai/stt.py b/src/pipecat/services/xai/stt.py index 3cc3a23b2..e219cfb6e 100644 --- a/src/pipecat/services/xai/stt.py +++ b/src/pipecat/services/xai/stt.py @@ -44,7 +44,7 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_xai_stt_language(language: Language) -> str | None: +def language_to_xai_stt_language(language: Language) -> str: """Convert a Language enum to the xAI STT language code. xAI STT accepts two-letter language codes (e.g. ``en``, ``fr``, ``de``, @@ -54,7 +54,10 @@ def language_to_xai_stt_language(language: Language) -> str | None: language: The Language enum value to convert. Returns: - The corresponding xAI STT language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AR: "ar", diff --git a/src/pipecat/services/xai/tts.py b/src/pipecat/services/xai/tts.py index 83e7781f2..ad7f05bbd 100644 --- a/src/pipecat/services/xai/tts.py +++ b/src/pipecat/services/xai/tts.py @@ -49,14 +49,17 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -def language_to_xai_language(language: Language) -> str | None: +def language_to_xai_language(language: Language) -> str: """Convert a Language enum to xAI language code. Args: language: The Language enum value to convert. Returns: - The corresponding xAI language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.AR: "ar-EG", diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index f9caa3819..d323e9b67 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -37,14 +37,17 @@ from pipecat.utils.tracing.service_decorators import traced_tts # https://github.com/coqui-ai/xtts-streaming-server -def language_to_xtts_language(language: Language) -> str | None: +def language_to_xtts_language(language: Language) -> str: """Convert a Language enum to XTTS language code. Args: language: The Language enum value to convert. Returns: - The corresponding XTTS language code, or None if not supported. + The corresponding service language code. If ``language`` is not in + the verified mapping, falls back to the base language code (e.g., + ``en`` from ``en-US``) and logs a warning (via + ``resolve_language(..., use_base_code=True)``). """ LANGUAGE_MAP = { Language.CS: "cs", From ef226c8a8e61d293faeeb2c4f2bf4597e8e08c57 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 16:19:14 -0400 Subject: [PATCH 16/23] fix: silence _settings NotGiven leaks and tighten Google STT language method Six pyright errors followed the same pattern: a value flowed out of `self._settings.X` (typed `T | _NotGiven`) into a context that wanted the plain `T`. Wrap each with `assert_given(...)` so the sentinel gets stripped at the boundary: - aws/nova_sonic/llm.py: `_settings.model` (in InvokeModel...Input) and `_settings.system_instruction` (passed to the adapter). - deepgram/flux/base.py: iterating `_settings.keyterm`. - google/stt.py: iterating `_settings.languages`. - google/tts.py: iterating `_settings.speaker_configs`. - openai/base_llm.py: `_settings.system_instruction` passed to the adapter. Also takes a deeper pass at the related Google STT issue: the override of `language_to_service_language` had been broadened to take `Language | list[Language]` and return `str | list[str]`, a Liskov violation against the base's `Language -> str | None` contract. External callers always pass a single Language, and the only consumer of the list path was Google STT's own `_get_language_codes`. Restore the override to a single-Language signature and let `_get_language_codes` iterate. The override is also tightened to return `str` (narrower than the base's `str | None`, which is LSP-compatible) since it always falls back to `"en-US"` rather than returning None. Net: -7 pyright errors (full-config run: 782 -> 775). --- src/pipecat/services/aws/nova_sonic/llm.py | 6 ++++-- src/pipecat/services/deepgram/flux/base.py | 2 +- src/pipecat/services/google/stt.py | 20 ++++++++++++-------- src/pipecat/services/google/tts.py | 2 +- src/pipecat/services/openai/base_llm.py | 4 ++-- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/pipecat/services/aws/nova_sonic/llm.py b/src/pipecat/services/aws/nova_sonic/llm.py index 55a4dd813..7c09ad6d0 100644 --- a/src/pipecat/services/aws/nova_sonic/llm.py +++ b/src/pipecat/services/aws/nova_sonic/llm.py @@ -974,7 +974,9 @@ class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): async def open_stream(self, client): """Open a bidirectional stream on the given client.""" return await client.invoke_model_with_bidirectional_stream( - InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model) + InvokeModelWithBidirectionalStreamOperationInput( + model_id=assert_given(self._settings.model) + ) ) async def send_event(self, event_json: str, stream): @@ -1127,7 +1129,7 @@ class AWSNovaSonicLLMService(LLMService[AWSNovaSonicLLMAdapter]): return None, [] adapter = self.get_llm_adapter() llm_params = adapter.get_llm_invocation_params( - self._context, system_instruction=self._settings.system_instruction + self._context, system_instruction=assert_given(self._settings.system_instruction) ) tools = ( llm_params["tools"] diff --git a/src/pipecat/services/deepgram/flux/base.py b/src/pipecat/services/deepgram/flux/base.py index f2042d799..173da5dc7 100644 --- a/src/pipecat/services/deepgram/flux/base.py +++ b/src/pipecat/services/deepgram/flux/base.py @@ -253,7 +253,7 @@ class DeepgramFluxSTTBase(STTService): params.append(f"mip_opt_out={str(self._mip_opt_out).lower()}") # Add keyterm parameters (can have multiple) - for keyterm in self._settings.keyterm: + for keyterm in assert_given(self._settings.keyterm): params.append(urlencode({"keyterm": keyterm})) # Add tag parameters (can have multiple) diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index a386566c5..5b64869e9 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -620,17 +620,20 @@ class GoogleSTTService(STTService): """ return True - def language_to_service_language(self, language: Language | list[Language]) -> str | list[str]: - """Convert Language enum(s) to Google STT language code(s). + def language_to_service_language(self, language: Language) -> str: + """Convert a Language enum to a Google STT language code. + + Narrower return type than the base class's ``str | None``: this + override always returns a string, falling back to ``"en-US"`` for + languages not in the verified mapping (see + :func:`language_to_google_stt_language`). Args: - language: Single Language enum or list of Language enums. + language: The Language enum value to convert. Returns: - str | List[str]: Google STT language code(s). + The Google STT language code. """ - if isinstance(language, list): - return [language_to_google_stt_language(lang) or "en-US" for lang in language] return language_to_google_stt_language(language) or "en-US" def _get_language_codes(self) -> list[str]: @@ -642,8 +645,9 @@ class GoogleSTTService(STTService): Returns: List[str]: Google STT language code strings. """ - if self._settings.languages: - return [self.language_to_service_language(lang) for lang in self._settings.languages] + languages = assert_given(self._settings.languages) + if languages: + return [self.language_to_service_language(lang) for lang in languages] language_codes = assert_given(self._settings.language_codes) if language_codes: return list(language_codes) diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 58bc31691..94f689d44 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -1417,7 +1417,7 @@ class GeminiTTSService(GoogleBaseTTSService): if self._settings.multi_speaker and self._settings.speaker_configs: # Multi-speaker mode speaker_voice_configs = [] - for speaker_config in self._settings.speaker_configs: + for speaker_config in assert_given(self._settings.speaker_configs): speaker_voice_configs.append( texttospeech_v1.MultispeakerPrebuiltVoice( speaker_alias=speaker_config["speaker_alias"], diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 72d4360a9..b5287067f 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -39,7 +39,7 @@ from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN -from pipecat.services.settings import LLMSettings, _NotGiven +from pipecat.services.settings import LLMSettings, _NotGiven, assert_given from pipecat.utils.tracing.service_decorators import traced_llm @@ -299,7 +299,7 @@ class BaseOpenAILLMService(LLMService[OpenAILLMAdapter]): params_from_context = adapter.get_llm_invocation_params( context, - system_instruction=self._settings.system_instruction, + system_instruction=assert_given(self._settings.system_instruction), convert_developer_to_user=not self.supports_developer_role, ) From 5e24027fd532860b14f71d24335c54f77abac91c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 17:15:35 -0400 Subject: [PATCH 17/23] fix: type fixes (and a few latent bug fixes) in 4 LLM adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same shape of fix we applied to anthropic_adapter.py earlier — these adapters do dict-style mutation on values typed as ChatCompletionMessageParam (a union of TypedDicts) or against Optional fields. Apply boundary casts (`cast(dict[str, Any], ...)` for the mutation block, cast back to the TypedDict at return sites). Most changes are pure typing (rename + cast); a handful in gemini and openai_realtime are small defensive bug fixes for code paths that were latently broken by Optional fields slipping through: perplexity_adapter.py — pure typing. Cast the deepcopied messages to `list[dict[str, Any]]` for the role-merging / system-conversion / trailing-assistant-removal transformations and cast back to ChatCompletionMessageParam at the return. bedrock_adapter.py — pure typing. Cast the message to `dict[str, Any]` at the top of `_from_standard_message` for the tool-result / tool-use / image-content transformations. Cast the constructed dict at the return site of `get_llm_invocation_params`. gemini_adapter.py — typing + several None guards on Content.parts and related Optional fields. Each guard turns a latent `TypeError`/`AttributeError` (when the type-system-allowed None showed up at runtime) into a defensive skip — the type annotations say these can be None and we now handle that. open_ai_realtime_adapter.py: - Typing: cast the deepcopied messages, cast back where needed. - LLMSpecificMessage handling: previously the function would crash on the first `.get()` call if any LLMSpecificMessage was in the list. Filter them out and document the limitation — this adapter's pack-into-single-text-message strategy doesn't compose with opaque per-provider payloads. - Real bug fix: `events.ConversationItem` is a Pydantic BaseModel, not a TypedDict. The bulk-packing path was constructing a raw dict where a ConversationItem was expected. Replaced with proper constructor calls (matches what the single-user-message path already does). - Real bug fix: `_from_universal_context_message` was declared `-> events.ConversationItem` but on the unhandled-message fallthrough it logged and returned None implicitly. Raise ValueError so the violation is loud, not silent. Removes 4 newly-clean files from the pyright ignore list: adapters/services/{perplexity,bedrock,gemini,open_ai_realtime}_adapter.py. Net: -95 pyright errors (full-config: 775 -> 680). --- pyrightconfig.json | 4 -- .../adapters/services/bedrock_adapter.py | 60 +++++++++--------- .../adapters/services/gemini_adapter.py | 37 +++++++---- .../services/open_ai_realtime_adapter.py | 61 ++++++++++++------- .../adapters/services/perplexity_adapter.py | 27 ++++---- 5 files changed, 112 insertions(+), 77 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 9370d4387..7cbaf2ce3 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -7,14 +7,10 @@ "ignore": [ "tests", "src/pipecat/adapters/services/aws_nova_sonic_adapter.py", - "src/pipecat/adapters/services/bedrock_adapter.py", - "src/pipecat/adapters/services/gemini_adapter.py", "src/pipecat/adapters/services/grok_realtime_adapter.py", "src/pipecat/adapters/services/inworld_realtime_adapter.py", "src/pipecat/adapters/services/open_ai_adapter.py", - "src/pipecat/adapters/services/open_ai_realtime_adapter.py", "src/pipecat/adapters/services/open_ai_responses_adapter.py", - "src/pipecat/adapters/services/perplexity_adapter.py", "src/pipecat/audio/dtmf/utils.py", "src/pipecat/audio/filters/aic_filter.py", "src/pipecat/audio/filters/krisp_viva_filter.py", diff --git a/src/pipecat/adapters/services/bedrock_adapter.py b/src/pipecat/adapters/services/bedrock_adapter.py index bb1223880..9ff8082ed 100644 --- a/src/pipecat/adapters/services/bedrock_adapter.py +++ b/src/pipecat/adapters/services/bedrock_adapter.py @@ -10,7 +10,7 @@ import base64 import copy import json from dataclasses import dataclass -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from loguru import logger @@ -68,16 +68,19 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): system_instruction, discard_context_system=True, ) - return { - "system": [{"text": effective_system}] if effective_system else None, - "messages": converted.messages, - # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) - "tools": self.from_standard_tools(context.tools) or [], - # To avoid refactoring in AWSBedrockLLMService, we just pass through tool_choice. - # Eventually (when we don't have to maintain the non-LLMContext code path) we should do - # the conversion to Bedrock's expected format here rather than in AWSBedrockLLMService. - "tool_choice": context.tool_choice, - } + return cast( + AWSBedrockLLMInvocationParams, + { + "system": [{"text": effective_system}] if effective_system else None, + "messages": converted.messages, + # NOTE: LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) + "tools": self.from_standard_tools(context.tools) or [], + # To avoid refactoring in AWSBedrockLLMService, we just pass through tool_choice. + # Eventually (when we don't have to maintain the non-LLMContext code path) we should do + # the conversion to Bedrock's expected format here rather than in AWSBedrockLLMService. + "tool_choice": context.tool_choice, + }, + ) def get_messages_for_logging(self, context) -> list[dict[str, Any]]: """Get messages from a universal LLM context in a format ready for logging about AWS Bedrock. @@ -213,35 +216,36 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): ] } """ - message = copy.deepcopy(message) - if message["role"] == "tool": + # ChatCompletionMessageParam (input) and the dict shape Bedrock expects + # are different — work with the deepcopied message as a plain dict for + # the transformations below. + msg = cast(dict[str, Any], copy.deepcopy(message)) + if msg["role"] == "tool": # Try to parse the content as JSON if it looks like JSON try: - if message["content"].strip().startswith("{") and message[ - "content" - ].strip().endswith("}"): - content_json = json.loads(message["content"]) + if msg["content"].strip().startswith("{") and msg["content"].strip().endswith("}"): + content_json = json.loads(msg["content"]) tool_result_content = [{"json": content_json}] else: - tool_result_content = [{"text": message["content"]}] + tool_result_content = [{"text": msg["content"]}] except (json.JSONDecodeError, ValueError, AttributeError): - tool_result_content = [{"text": message["content"]}] + tool_result_content = [{"text": msg["content"]}] return { "role": "user", "content": [ { "toolResult": { - "toolUseId": message["tool_call_id"], + "toolUseId": msg["tool_call_id"], "content": tool_result_content, }, }, ], } - if message.get("tool_calls"): - tc = message["tool_calls"] - ret = {"role": "assistant", "content": []} + if msg.get("tool_calls"): + tc = msg["tool_calls"] + ret: dict[str, Any] = {"role": "assistant", "content": []} for tool_call in tc: function = tool_call["function"] arguments = json.loads(function["arguments"]) @@ -256,12 +260,12 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): return ret # Handle text content - content = message.get("content") + content = msg.get("content") if isinstance(content, str): if content == "": - return {"role": message["role"], "content": [{"text": "(empty)"}]} + return {"role": msg["role"], "content": [{"text": "(empty)"}]} else: - return {"role": message["role"], "content": [{"text": content}]} + return {"role": msg["role"], "content": [{"text": content}]} elif isinstance(content, list): new_content = [] for item in content: @@ -300,9 +304,9 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]): # Move image before the first text image_item = new_content.pop(img_idx) new_content.insert(first_txt_idx, image_item) - return {"role": message["role"], "content": new_content} + return {"role": msg["role"], "content": new_content} - return message + return msg @staticmethod def _to_bedrock_function_format(function: FunctionSchema) -> dict[str, Any]: diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py index aede18e7c..4e9e20e14 100644 --- a/src/pipecat/adapters/services/gemini_adapter.py +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -9,7 +9,7 @@ import base64 import json from dataclasses import dataclass, field -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from loguru import logger from openai import NotGiven @@ -154,9 +154,12 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): messages = self._from_universal_context_messages(self.get_messages(context)).messages # Sanitize messages for logging - messages_for_logging = [] + messages_for_logging: list[dict[str, Any]] = [] for message in messages: - obj = message.to_json_dict() + # `to_json_dict()` returns `dict[str, object]`; treat as a plain + # dict for the value indexing/mutation below. The broad `except` + # below is the safety net if any item isn't shaped as expected. + obj: dict[str, Any] = cast(dict[str, Any], message.to_json_dict()) try: if "parts" in obj: for part in obj["parts"]: @@ -274,7 +277,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): # Check if we only have function-related messages (no regular text) effective_system = extracted_system or system_instruction has_regular_messages = any( - len(msg.parts) == 1 + msg.parts is not None + and len(msg.parts) == 1 and getattr(msg.parts[0], "text", None) and not getattr(msg.parts[0], "function_call", None) and not getattr(msg.parts[0], "function_response", None) @@ -346,8 +350,11 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))] ) """ - role = message["role"] - content = message.get("content", []) + # ChatCompletionMessageParam (a union of TypedDicts) doesn't allow + # the dict-style key access used below; treat it as a plain dict. + msg = cast(dict[str, Any], message) + role = msg["role"] + content = msg.get("content", []) # Convert non-initial system/developer messages to user role, # as Gemini doesn't support these as input messages. @@ -359,8 +366,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): parts = [] tool_call_id_to_name_mapping = {} - if message.get("tool_calls"): - for tc in message["tool_calls"]: + if msg.get("tool_calls"): + for tc in msg["tool_calls"]: id = tc["id"] name = tc["function"]["name"] tool_call_id_to_name_mapping[id] = name @@ -376,7 +383,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): elif role == "tool": role = "user" try: - response = json.loads(message["content"]) + response = json.loads(msg["content"]) if isinstance(response, dict): response_dict = response else: @@ -384,10 +391,10 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): except Exception as e: # Response might not be JSON-deserializable. # This occurs with a UserImageFrame, for example, where we get a plain "COMPLETED" string. - response_dict = {"value": message["content"]} + response_dict = {"value": msg["content"]} # Get function name from mapping using tool_call_id, or fallback - tool_call_id = message.get("tool_call_id") + tool_call_id = msg.get("tool_call_id") function_name = "tool_call_result" # Default fallback if tool_call_id and tool_call_id in params.tool_call_id_to_name_mapping: function_name = params.tool_call_id_to_name_mapping[tool_call_id] @@ -491,7 +498,7 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): def is_tool_call_message(msg: Content) -> bool: """Check if message contains only function_call parts.""" - return ( + return bool( msg.role == "model" and msg.parts and all(getattr(part, "function_call", None) for part in msg.parts) @@ -499,6 +506,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): def message_has_thought_signature(msg: Content) -> bool: """Check if any part in the message has a thought_signature.""" + if msg.parts is None: + return False return any(getattr(part, "thought_signature", None) for part in msg.parts) merged_messages = [] @@ -564,6 +573,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): logger.debug(f"Thought signatures to apply: {len(thought_signature_dicts)}") for ts in thought_signature_dicts: bookmark = ts.get("bookmark") + if bookmark is None: + continue if bookmark.get("function_call"): logger.trace(f" - To function call: {bookmark['function_call']}") elif bookmark.get("text"): @@ -665,6 +676,8 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]): if ( hasattr(part, "inline_data") and part.inline_data + and part.inline_data.data is not None + and bookmark_inline_data.data is not None # Comparing length should be good enough for matching inline data, # especially since we're already matching thought signatures in # strict message order. Comparing actual data is expensive. diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py index 7df7e45c5..fb555b039 100644 --- a/src/pipecat/adapters/services/open_ai_realtime_adapter.py +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -9,7 +9,7 @@ import copy import json from dataclasses import dataclass -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from loguru import logger @@ -81,7 +81,7 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): Returns: List of messages in a format ready for logging about OpenAI Realtime. """ - return self.get_messages(context, truncate_large_values=True) + return cast(list[dict[str, Any]], self.get_messages(context, truncate_large_values=True)) @dataclass class ConvertedMessages: @@ -101,12 +101,24 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): if not universal_context_messages: return self.ConvertedMessages(messages=[]) - messages = copy.deepcopy(universal_context_messages) + # NOTE: This adapter does not yet handle ``LLMSpecificMessage`` — those + # are filtered out below. Other adapters (e.g. Anthropic) dispatch + # LLMSpecific items through a per-provider passthrough. For OpenAI + # Realtime, the strategy here packs a multi-message history into a + # single text message (see comment further down), which doesn't + # compose with opaque per-provider payloads. If/when this adapter + # adopts the per-message strategy, LLMSpecific items can flow + # through `_from_universal_context_message` like in other adapters. + messages: list[dict[str, Any]] = [ + cast(dict[str, Any], m) + for m in copy.deepcopy(universal_context_messages) + if isinstance(m, dict) + ] system_instruction = None # If we have a "system" message as our first message, # pull that out into session "instructions" - if messages[0].get("role") == "system": + if messages and messages[0].get("role") == "system": system = messages.pop(0) content = system.get("content") if isinstance(content, str): @@ -124,7 +136,9 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): # If we have just a single "user" item, we can just send it normally if len(messages) == 1 and messages[0].get("role") == "user": return self.ConvertedMessages( - messages=[self._from_universal_context_message(messages[0])], + messages=[ + self._from_universal_context_message(cast(LLMContextMessage, messages[0])) + ], system_instruction=system_instruction, ) @@ -142,18 +156,18 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): return self.ConvertedMessages( messages=[ - { - "role": "user", - "type": "message", - "content": [ - { - "type": "input_text", - "text": "\n\n".join( + events.ConversationItem( + role="user", + type="message", + content=[ + events.ItemContent( + type="input_text", + text="\n\n".join( [intro_text, json.dumps(messages, indent=2), trailing_text] ), - } + ) ], - } + ) ], system_instruction=system_instruction, ) @@ -161,31 +175,34 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): def _from_universal_context_message( self, message: LLMContextMessage ) -> events.ConversationItem: - if message.get("role") == "user": - content = message.get("content") - if isinstance(message.get("content"), list): + # NOTE: ``LLMSpecificMessage`` is not yet handled here — see the + # corresponding note in `_from_universal_context_messages`. + msg = cast(dict[str, Any], message) + if msg.get("role") == "user": + content = msg.get("content") + if isinstance(content, list): content = "" - for c in message.get("content"): + for c in msg.get("content", []): if c.get("type") == "text": content += " " + c.get("text") else: logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" + f"Unhandled content type in context message: {c.get('type')} - {msg}" ) return events.ConversationItem( role="user", type="message", content=[events.ItemContent(type="input_text", text=content)], ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] + if msg.get("role") == "assistant" and msg.get("tool_calls"): + tc = msg["tool_calls"][0] return events.ConversationItem( type="function_call", call_id=tc["id"], name=tc["function"]["name"], arguments=tc["function"]["arguments"], ) - logger.error(f"Unhandled message type in _from_universal_context_message: {message}") + raise ValueError(f"Unhandled message type in _from_universal_context_message: {msg}") @staticmethod def _to_openai_realtime_function_format(function: FunctionSchema) -> dict[str, Any]: diff --git a/src/pipecat/adapters/services/perplexity_adapter.py b/src/pipecat/adapters/services/perplexity_adapter.py index 188092b78..7e984d83e 100644 --- a/src/pipecat/adapters/services/perplexity_adapter.py +++ b/src/pipecat/adapters/services/perplexity_adapter.py @@ -28,6 +28,7 @@ the messages are sent to Perplexity's API. """ import copy +from typing import Any, cast from openai.types.chat import ChatCompletionMessageParam @@ -116,7 +117,11 @@ class PerplexityLLMAdapter(OpenAILLMAdapter): if not messages: return messages - messages = copy.deepcopy(messages) + # ChatCompletionMessageParam is a union of TypedDicts; the + # transformations below mutate by key/index in ways those TypedDicts + # don't permit. Work against a plain-dict view for the duration of + # the transformation and cast back at the return site. + msgs: list[dict[str, Any]] = cast(list[dict[str, Any]], copy.deepcopy(messages)) # Note: "developer" → "user" conversion is handled by the parent adapter # via the convert_developer_to_user parameter. @@ -125,10 +130,10 @@ class PerplexityLLMAdapter(OpenAILLMAdapter): # Perplexity allows system messages at the start, but rejects them # after any non-system message. in_initial_system_block = True - for i in range(len(messages)): - if messages[i].get("role") == "system": + for i in range(len(msgs)): + if msgs[i].get("role") == "system": if not in_initial_system_block: - messages[i]["role"] = "user" + msgs[i]["role"] = "user" else: in_initial_system_block = False @@ -137,9 +142,9 @@ class PerplexityLLMAdapter(OpenAILLMAdapter): # messages that violate Perplexity's strict alternation requirement. # Skip consecutive system messages at the start — Perplexity allows those. i = 0 - while i < len(messages) - 1: - current = messages[i] - next_msg = messages[i + 1] + while i < len(msgs) - 1: + current = msgs[i] + next_msg = msgs[i + 1] if current["role"] == next_msg["role"] == "system": # Perplexity allows multiple initial system messages, don't merge i += 1 @@ -154,7 +159,7 @@ class PerplexityLLMAdapter(OpenAILLMAdapter): next_msg.get("content"), list ): current["content"].extend(next_msg["content"]) - messages.pop(i + 1) + msgs.pop(i + 1) else: i += 1 @@ -162,7 +167,7 @@ class PerplexityLLMAdapter(OpenAILLMAdapter): # Perplexity requires the last message to be "user" or "tool". # OpenAI appears to silently ignore trailing assistant messages # server-side, so removing them preserves equivalent behavior. - while messages and messages[-1].get("role") == "assistant": - messages.pop() + while msgs and msgs[-1].get("role") == "assistant": + msgs.pop() - return messages + return cast(list[ChatCompletionMessageParam], msgs) From 96756bc1f6354a12e5b8fc827518a5885f617dd8 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 28 Apr 2026 17:35:22 -0400 Subject: [PATCH 18/23] fix: clean up TypedDict / Optional patterns in 6 more LLM adapters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same approach as the previous round — apply boundary casts where the code does dict-style mutation on TypedDict-typed values, narrow at return sites, and document the LLMSpecificMessage limitation in realtime adapters that pack history into a single text message. aws_nova_sonic_adapter.py — pure typing + small narrowing fixes: - Filter LLMSpecific items in `_from_universal_context_messages` (documented). - `_from_universal_context_message` now declared `-> AWSNovaSonicConversationHistoryMessage | None` (it already had paths returning None implicitly). - `get_messages_for_logging` returns `dict[str, Any]` per element via `dataclasses.asdict`, matching the declared return type. - Use a local `role` variable so pyright keeps the narrowing across the truthy-content guard. grok_realtime_adapter.py / inworld_realtime_adapter.py — same shape of fix as `open_ai_realtime_adapter.py` from the previous batch. The two files are essentially copies of the OpenAI Realtime adapter, so the same template applies: cast at the boundary, filter LLMSpecificMessage with a documented note, replace the implicit-None fallthrough with `raise ValueError`, and switch the `text_content +=` pattern (which fails when one of the parts is None) to a `text_parts.append(...)` + `" ".join(...)` pattern. open_ai_adapter.py — pure typing. Cast at the `OpenAILLMInvocationParams` return, narrow the system-instruction warning's `initial_content` to `str | None`, and cast the custom-tools list to `list[ChatCompletionToolParam]`. open_ai_responses_adapter.py — pure typing. Same shape: narrow `first_content` to `str | None` for the warning resolver, cast the constructed dict literals at append sites where the target is `ResponseInputItemParam`, and cast `get_messages_for_logging`'s return to the declared `list[dict[str, Any]]`. processors/aggregators/llm_context.py — pure typing. Cast the deepcopied message in the redaction loop in `get_messages` to `dict[str, Any]` and the create_image/audio_message return-dict literals to `LLMContextMessage`. Removes 6 newly-clean files from the pyright ignore list. Net: -77 pyright errors (full-config: 680 -> 603). --- pyrightconfig.json | 6 --- .../services/aws_nova_sonic_adapter.py | 47 ++++++++++++------ .../services/grok_realtime_adapter.py | 45 +++++++++++------ .../services/inworld_realtime_adapter.py | 45 +++++++++++------ .../adapters/services/open_ai_adapter.py | 49 ++++++++++++------- .../services/open_ai_responses_adapter.py | 22 ++++++--- .../processors/aggregators/llm_context.py | 15 +++--- 7 files changed, 150 insertions(+), 79 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 7cbaf2ce3..d882b2106 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -6,11 +6,6 @@ "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ "tests", - "src/pipecat/adapters/services/aws_nova_sonic_adapter.py", - "src/pipecat/adapters/services/grok_realtime_adapter.py", - "src/pipecat/adapters/services/inworld_realtime_adapter.py", - "src/pipecat/adapters/services/open_ai_adapter.py", - "src/pipecat/adapters/services/open_ai_responses_adapter.py", "src/pipecat/audio/dtmf/utils.py", "src/pipecat/audio/filters/aic_filter.py", "src/pipecat/audio/filters/krisp_viva_filter.py", @@ -18,7 +13,6 @@ "src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py", "src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py", "src/pipecat/audio/vad/silero.py", - "src/pipecat/processors/aggregators/llm_context.py", "src/pipecat/processors/aggregators/llm_response_universal.py", "src/pipecat/processors/frame_processor.py", "src/pipecat/processors/frameworks/langchain.py", diff --git a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py index e38fe901b..6d3d90bff 100644 --- a/src/pipecat/adapters/services/aws_nova_sonic_adapter.py +++ b/src/pipecat/adapters/services/aws_nova_sonic_adapter.py @@ -8,9 +8,9 @@ import copy import json -from dataclasses import dataclass +from dataclasses import asdict, dataclass from enum import Enum -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from loguru import logger @@ -110,7 +110,10 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): Returns: List of messages in a format ready for logging about AWS Nova Sonic. """ - return self._from_universal_context_messages(self.get_messages(context)).messages + return [ + asdict(m) + for m in self._from_universal_context_messages(self.get_messages(context)).messages + ] @dataclass class ConvertedMessages: @@ -123,18 +126,27 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): self, universal_context_messages: list[LLMContextMessage] ) -> ConvertedMessages: system_instruction = None - messages = [] + messages: list[AWSNovaSonicConversationHistoryMessage] = [] # Bail if there are no messages if not universal_context_messages: - return self.ConvertedMessages() + return self.ConvertedMessages(messages=[]) - universal_context_messages = copy.deepcopy(universal_context_messages) + # NOTE: This adapter does not yet handle ``LLMSpecificMessage`` — + # those are filtered out below (the role-extraction and conversion + # logic only applies to standard message dicts). If/when this + # adapter grows a per-provider passthrough like the Anthropic + # adapter has, LLMSpecific items can flow through. + ucm: list[dict[str, Any]] = [ + cast(dict[str, Any], m) + for m in copy.deepcopy(universal_context_messages) + if isinstance(m, dict) + ] # If we have a "system" message as our first message, # pull that out into "instruction" - if universal_context_messages[0].get("role") == "system": - system = universal_context_messages.pop(0) + if ucm and ucm[0].get("role") == "system": + system = ucm.pop(0) content = system.get("content") if isinstance(content, str): system_instruction = content @@ -145,19 +157,21 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): # Convert any remaining "system"/"developer" messages to "user", # as Nova Sonic only supports "user" and "assistant" in history. - for msg in universal_context_messages: + for msg in ucm: if msg.get("role") in ("system", "developer"): msg["role"] = "user" # Process remaining messages to fill out conversation history. - for universal_context_message in universal_context_messages: + for universal_context_message in ucm: message = self._from_universal_context_message(universal_context_message) if message: messages.append(message) return self.ConvertedMessages(messages=messages, system_instruction=system_instruction) - def _from_universal_context_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + def _from_universal_context_message( + self, message: dict[str, Any] + ) -> AWSNovaSonicConversationHistoryMessage | None: """Convert standard message format to Nova Sonic format. Args: @@ -167,17 +181,18 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter[AWSNovaSonicLLMInvocationParams]): Nova Sonic conversation history message, or None if not convertible. """ role = message.get("role") - if message.get("role") == "user" or message.get("role") == "assistant": + if role == "user" or role == "assistant": content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): + if isinstance(content, list): + text_parts = [] + for c in content: if c.get("type") == "text": - content += " " + c.get("text") + text_parts.append(c.get("text")) else: logger.error( f"Unhandled content type in context message: {c.get('type')} - {message}" ) + content = " ".join(t for t in text_parts if t) # There won't be content if this is an assistant tool call entry. # We're ignoring those since they can't be loaded into AWS Nova Sonic conversation # history diff --git a/src/pipecat/adapters/services/grok_realtime_adapter.py b/src/pipecat/adapters/services/grok_realtime_adapter.py index 75ca61030..87bde52a7 100644 --- a/src/pipecat/adapters/services/grok_realtime_adapter.py +++ b/src/pipecat/adapters/services/grok_realtime_adapter.py @@ -13,7 +13,7 @@ Grok's Voice Agent API. import copy import json from dataclasses import dataclass -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from loguru import logger @@ -85,7 +85,10 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter): Returns: List of messages with sensitive data redacted. """ - return self.get_messages(context, truncate_large_values=True) + return cast( + list[dict[str, Any]], + self.get_messages(context, truncate_large_values=True), + ) @dataclass class ConvertedMessages: @@ -111,11 +114,20 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter): if not universal_context_messages: return self.ConvertedMessages(messages=[]) - messages = copy.deepcopy(universal_context_messages) + # NOTE: This adapter does not yet handle ``LLMSpecificMessage`` — + # those are filtered out below. Other adapters (e.g. Anthropic) + # dispatch LLMSpecific items through a per-provider passthrough. + # The pack-into-single-text-message strategy here doesn't compose + # with opaque per-provider payloads. + messages: list[dict[str, Any]] = [ + cast(dict[str, Any], m) + for m in copy.deepcopy(universal_context_messages) + if isinstance(m, dict) + ] system_instruction = None # Extract system message as session instructions - if messages[0].get("role") == "system": + if messages and messages[0].get("role") == "system": system = messages.pop(0) content = system.get("content") if isinstance(content, str): @@ -133,7 +145,9 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter): # Single user message can be sent normally if len(messages) == 1 and messages[0].get("role") == "user": return self.ConvertedMessages( - messages=[self._from_universal_context_message(messages[0])], + messages=[ + self._from_universal_context_message(cast(LLMContextMessage, messages[0])) + ], system_instruction=system_instruction, ) @@ -181,26 +195,29 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter): Returns: ConversationItem formatted for Grok Realtime API. """ - if message.get("role") == "user": - content = message.get("content") + # NOTE: ``LLMSpecificMessage`` is not yet handled here — see the + # corresponding note in `_from_universal_context_messages`. + msg = cast(dict[str, Any], message) + if msg.get("role") == "user": + content = msg.get("content") if isinstance(content, list): - text_content = "" + text_parts = [] for c in content: if c.get("type") == "text": - text_content += " " + c.get("text") + text_parts.append(c.get("text")) else: logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" + f"Unhandled content type in context message: {c.get('type')} - {msg}" ) - content = text_content.strip() + content = " ".join(t for t in text_parts if t).strip() return events.ConversationItem( role="user", type="message", content=[events.ItemContent(type="input_text", text=content)], ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] + if msg.get("role") == "assistant" and msg.get("tool_calls"): + tc = msg["tool_calls"][0] return events.ConversationItem( type="function_call", call_id=tc["id"], @@ -208,7 +225,7 @@ class GrokRealtimeLLMAdapter(BaseLLMAdapter): arguments=tc["function"]["arguments"], ) - logger.error(f"Unhandled message type in _from_universal_context_message: {message}") + raise ValueError(f"Unhandled message type in _from_universal_context_message: {msg}") @staticmethod def _to_grok_function_format(function: FunctionSchema) -> dict[str, Any]: diff --git a/src/pipecat/adapters/services/inworld_realtime_adapter.py b/src/pipecat/adapters/services/inworld_realtime_adapter.py index db07256f5..444c5d8bc 100644 --- a/src/pipecat/adapters/services/inworld_realtime_adapter.py +++ b/src/pipecat/adapters/services/inworld_realtime_adapter.py @@ -13,7 +13,7 @@ Inworld's Realtime API. import copy import json from dataclasses import dataclass -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from loguru import logger @@ -85,7 +85,10 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter): Returns: List of messages with sensitive data redacted. """ - return self.get_messages(context, truncate_large_values=True) + return cast( + list[dict[str, Any]], + self.get_messages(context, truncate_large_values=True), + ) @dataclass class ConvertedMessages: @@ -111,11 +114,20 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter): if not universal_context_messages: return self.ConvertedMessages(messages=[]) - messages = copy.deepcopy(universal_context_messages) + # NOTE: This adapter does not yet handle ``LLMSpecificMessage`` — + # those are filtered out below. Other adapters (e.g. Anthropic) + # dispatch LLMSpecific items through a per-provider passthrough. + # The pack-into-single-text-message strategy here doesn't compose + # with opaque per-provider payloads. + messages: list[dict[str, Any]] = [ + cast(dict[str, Any], m) + for m in copy.deepcopy(universal_context_messages) + if isinstance(m, dict) + ] system_instruction = None # Extract system message as session instructions - if messages[0].get("role") == "system": + if messages and messages[0].get("role") == "system": system = messages.pop(0) content = system.get("content") if isinstance(content, str): @@ -133,7 +145,9 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter): # Single user message can be sent normally if len(messages) == 1 and messages[0].get("role") == "user": return self.ConvertedMessages( - messages=[self._from_universal_context_message(messages[0])], + messages=[ + self._from_universal_context_message(cast(LLMContextMessage, messages[0])) + ], system_instruction=system_instruction, ) @@ -181,26 +195,29 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter): Returns: ConversationItem formatted for Inworld Realtime API. """ - if message.get("role") == "user": - content = message.get("content") + # NOTE: ``LLMSpecificMessage`` is not yet handled here — see the + # corresponding note in `_from_universal_context_messages`. + msg = cast(dict[str, Any], message) + if msg.get("role") == "user": + content = msg.get("content") if isinstance(content, list): - text_content = "" + text_parts = [] for c in content: if c.get("type") == "text": - text_content += " " + c.get("text") + text_parts.append(c.get("text")) else: logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" + f"Unhandled content type in context message: {c.get('type')} - {msg}" ) - content = text_content.strip() + content = " ".join(t for t in text_parts if t).strip() return events.ConversationItem( role="user", type="message", content=[events.ItemContent(type="input_text", text=content)], ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] + if msg.get("role") == "assistant" and msg.get("tool_calls"): + tc = msg["tool_calls"][0] return events.ConversationItem( type="function_call", call_id=tc["id"], @@ -208,7 +225,7 @@ class InworldRealtimeLLMAdapter(BaseLLMAdapter): arguments=tc["function"]["arguments"], ) - logger.error(f"Unhandled message type in _from_universal_context_message: {message}") + raise ValueError(f"Unhandled message type in _from_universal_context_message: {msg}") @staticmethod def _to_inworld_function_format(function: FunctionSchema) -> dict[str, Any]: diff --git a/src/pipecat/adapters/services/open_ai_adapter.py b/src/pipecat/adapters/services/open_ai_adapter.py index b5ba63c75..fe2670f76 100644 --- a/src/pipecat/adapters/services/open_ai_adapter.py +++ b/src/pipecat/adapters/services/open_ai_adapter.py @@ -127,12 +127,15 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]): ) if system_instruction: - # Detect initial system message for warning purposes (don't extract) - initial_content = ( - messages[0].get("content", "") - if messages and messages[0].get("role") == "system" - else None - ) + # Detect initial system message for warning purposes (don't extract). + # ChatCompletionMessageParam.content is `str | Iterable[...]`; we + # only forward it for warning purposes, so coerce non-strings to + # None — the resolver handles None. + initial_content: str | None = None + if messages and messages[0].get("role") == "system": + raw_content = messages[0].get("content", "") + if isinstance(raw_content, str): + initial_content = raw_content self._resolve_system_instruction( initial_content, system_instruction, @@ -140,12 +143,15 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]): ) messages = [{"role": "system", "content": system_instruction}] + messages - return { - "messages": messages, - # NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) - "tools": self.from_standard_tools(context.tools), - "tool_choice": _openai_from_llm_context_tool_choice(context.tool_choice), - } + return cast( + OpenAILLMInvocationParams, + { + "messages": messages, + # NOTE; LLMContext's tools are guaranteed to be a ToolsSchema (or NOT_GIVEN) + "tools": self.from_standard_tools(context.tools), + "tool_choice": _openai_from_llm_context_tool_choice(context.tool_choice), + }, + ) def to_provider_tools_format(self, tools_schema: ToolsSchema) -> list[ChatCompletionToolParam]: """Convert function schemas to OpenAI's function-calling format. @@ -158,13 +164,19 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]): with ChatCompletion API. """ functions_schema = tools_schema.standard_tools - formatted_standard_tools = [ - ChatCompletionToolParam(type="function", function=func.to_default_dict()) + # `function=...` expects a `FunctionDefinition` TypedDict; the dict + # produced by `to_default_dict()` is structurally compatible. Cast at + # the boundary. + formatted_standard_tools: list[ChatCompletionToolParam] = [ + ChatCompletionToolParam(type="function", function=cast(Any, func.to_default_dict())) for func in functions_schema ] - custom_openai_tools = [] + custom_openai_tools: list[ChatCompletionToolParam] = [] if tools_schema.custom_tools: - custom_openai_tools = tools_schema.custom_tools.get(AdapterType.OPENAI, []) + custom_openai_tools = cast( + list[ChatCompletionToolParam], + tools_schema.custom_tools.get(AdapterType.OPENAI, []), + ) return formatted_standard_tools + custom_openai_tools def get_messages_for_logging(self, context: LLMContext) -> list[dict[str, Any]]: @@ -178,7 +190,10 @@ class OpenAILLMAdapter(BaseLLMAdapter[OpenAILLMInvocationParams]): Returns: List of messages in a format ready for logging about OpenAI. """ - return self.get_messages(context, truncate_large_values=True) + return cast( + list[dict[str, Any]], + self.get_messages(context, truncate_large_values=True), + ) def _from_universal_context_messages( self, diff --git a/src/pipecat/adapters/services/open_ai_responses_adapter.py b/src/pipecat/adapters/services/open_ai_responses_adapter.py index c5c6cbc7a..d19e7d117 100644 --- a/src/pipecat/adapters/services/open_ai_responses_adapter.py +++ b/src/pipecat/adapters/services/open_ai_responses_adapter.py @@ -6,7 +6,7 @@ """OpenAI Responses API adapter for Pipecat.""" -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from openai._types import NotGiven as OpenAINotGiven from openai.types.responses import FunctionToolParam, ResponseInputItemParam, ToolParam @@ -64,8 +64,11 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam if system_instruction and messages: first_msg = messages[0] if not isinstance(messages[0], LLMSpecificMessage) else None if first_msg and first_msg.get("role") == "system": + # `content` is `str | Iterable[...]`; we only forward it for + # warning purposes. Coerce non-strings to None. + first_content = first_msg.get("content", "") self._resolve_system_instruction( - first_msg.get("content", ""), + first_content if isinstance(first_content, str) else None, system_instruction, discard_context_system=False, ) @@ -143,7 +146,10 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam Returns: List of messages in a format ready for logging. """ - return self.get_messages(context, truncate_large_values=True) + return cast( + list[dict[str, Any]], + self.get_messages(context, truncate_large_values=True), + ) def _convert_messages_to_input( self, messages: list[LLMContextMessage] @@ -169,13 +175,15 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam content = message.get("content", "") if isinstance(content, list): content = self._convert_multimodal_content(content) - result.append({"role": "developer", "content": content}) + result.append( + cast(ResponseInputItemParam, {"role": "developer", "content": content}) + ) elif role == "user": content = message.get("content", "") if isinstance(content, list): content = self._convert_multimodal_content(content) - result.append({"role": "user", "content": content}) + result.append(cast(ResponseInputItemParam, {"role": "user", "content": content})) elif role == "assistant": tool_calls = message.get("tool_calls") @@ -194,7 +202,9 @@ class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParam content = message.get("content", "") if isinstance(content, list): content = self._convert_multimodal_content(content) - result.append({"role": "assistant", "content": content}) + result.append( + cast(ResponseInputItemParam, {"role": "assistant", "content": content}) + ) elif role == "tool": content = message.get("content", "") diff --git a/src/pipecat/processors/aggregators/llm_context.py b/src/pipecat/processors/aggregators/llm_context.py index ecf7e9239..2145c7d83 100644 --- a/src/pipecat/processors/aggregators/llm_context.py +++ b/src/pipecat/processors/aggregators/llm_context.py @@ -21,7 +21,7 @@ import io import wave from collections.abc import Callable from dataclasses import dataclass -from typing import Any, TypeAlias, TypeGuard, TypeVar +from typing import Any, TypeAlias, TypeGuard, TypeVar, cast from loguru import logger from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN @@ -129,13 +129,13 @@ class LLMContext: url: The URL of the image. text: Optional text to include with the image. """ - content = [] + content: list[dict[str, Any]] = [] if text: content.append({"type": "text", "text": text}) content.append({"type": "image_url", "image_url": {"url": url}}) - return {"role": role, "content": content} + return cast(LLMContextMessage, {"role": role, "content": content}) @staticmethod async def create_image_message( @@ -187,7 +187,7 @@ class LLMContext: audio_frames: List of audio frame objects to include. text: Optional text to include with the audio. """ - content = [{"type": "text", "text": text}] + content: list[dict[str, Any]] = [{"type": "text", "text": text}] def encode_audio(): sample_rate = audio_frames[0].sample_rate @@ -214,7 +214,7 @@ class LLMContext: } ) - return {"role": role, "content": content} + return cast(LLMContextMessage, {"role": role, "content": content}) @property def messages(self) -> list[LLMContextMessage]: @@ -295,7 +295,10 @@ class LLMContext: result.append(msg_copy) continue - msg = copy.deepcopy(message) + # The standard message variant is a union of TypedDicts; the + # mutations below operate on plain dicts at runtime. Treat as + # such for the duration of the redaction loop. + msg: dict[str, Any] = cast(dict[str, Any], copy.deepcopy(message)) content = msg.get("content") if isinstance(content, list): for item in content: From 814f00ce414c6076093c9136f9eda867e897e620 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Apr 2026 10:56:04 -0400 Subject: [PATCH 19/23] fix: clear 19 TTS/STT/etc. services from pyright ignore list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several adjacent fix shapes that together drop 19 files from the pyrightconfig.json ignore list (96 → 77) and full-pyright errors from 605 → 580. Default pyright stays clean. TTS voice/context_id None handling — most files in this batch had a single error of the shape "value typed `T | None` passed where `T` is required" coming out of `assert_given(self._settings.voice)` (which strips `_NotGiven` but not `None`) or `get_active_audio_context_id()`. Two patterns: - For services where a missing voice means the request can't proceed (hume, openai, xtts, groq, kokoro, piper), added an explicit None check. Inside `run_tts` we yield an `ErrorFrame` and return — matching each service's existing error-emission style (a few wrap `Exception` broadly and were fine; openai/hume/xtts had narrower or no try blocks so a bare `raise ValueError` would have escaped uncaught). Piper validates in `__init__`, where failing fast at construction is the right shape. OpenAI also gained a `voice not in VALID_VOICES` guard with a clear message listing supported voices. - For services where a missing audio context just means "skip this message" (fish, lmnt, smallest, sarvam, neuphonic), widened `TTSService.append_to_audio_context`'s `context_id` signature to `str | None`. The function body already explicitly handled the None case with a debug log + early return, so the prior `str` annotation was a lie; making it honest cleared call sites without local guards. inworld's `_close_context` got the same treatment. google.genai imports — switched `from google import genai` to `import google.genai as genai` in google/image.py and google/llm.py. The dotted form sidesteps a PEP 420 namespace-package stub gap (the `google` namespace stubs come from a different distribution and don't declare `genai`), which means pyright now resolves `genai` to the real module rather than `Unknown`. IDE autocomplete on `genai.` works for the first time. In image.py this surfaced three latent bugs that the `Unknown` resolution had been hiding (model was `str | _NotGiven | None` not narrowed before passing to the SDK; two spots accessed `.image_bytes` on an `Image | None` without a guard) — all fixed. llm.py's dotted import surfaced 8 errors (Content-list typing nuances, internal `_api_client` access, a few small Optionals); deferred to a future pass since they're outside this commit's scope, so the file stays in the ignore list with the dotted import. Latent bug fixes spotted along the way: - resembleai/tts.py was calling `push_error(ErrorFrame(...))`, but `push_error` takes a string — there's a separate `push_error_frame` for the frame case. Switched to the right method. - openai/base_llm.py: `max_completion_tokens` was the only sibling field on `OpenAILLMSettings` missing `| None` in its type, which caused the assignment in openai/llm.py from `params.max_completion_tokens` (`int | None`) to fail. Added `| None` for consistency with `max_tokens` etc. - heygen/base_api.py: `livekit_url: str = None` and `ws_url: str = None` declared `str` while defaulting to `None`. Removed the bogus defaults — both fields are required at construction in every in-tree call site, and the previous `str = None` was a Pydantic footgun. Other small ones: gladia/stt.py needed a None guard on `_session_url` before `websocket_connect`; openrouter/llm.py's `build_chat_completion_params` override widened to `dict[str, Any]` diverging from the parent's `OpenAILLMInvocationParams` — restored the parent's type; neuphonic/tts.py guarded the receive loop's `async for message in self._websocket` with a local-variable narrowing matching the pattern from 9e9b1f39e. groq/tts.py: tightened `output_format`'s typing to `Literal["flac","mp3","mulaw","ogg","wav"] | str = "wav"`. The literal side gives IDE autocomplete hints for the currently-supported set; the `| str` side keeps callers unblocked if groq adds a new format before this list is updated. A `cast` at the API boundary satisfies groq's stricter `Literal` parameter type. The literal alias mirrors the inlined Literal on `groq.resources.audio.speech.AsyncSpeech.create`'s `response_format` (the SDK doesn't export it as a named symbol). websocket_service.py: scoped `# pyright: ignore[reportAttributeAccessIssue]` on `websockets.WebSocketClientProtocol`. That alias is now a deprecated re-export from the legacy submodule and pyright doesn't surface it on the top-level `websockets` namespace; runtime is fine. Migrating to `websockets.ClientConnection` is a separate piece of work (transports/websocket/client.py uses the same alias four times) and left for a future commit. Files dropped from the ignore list: fish/tts.py, gladia/stt.py, google/image.py, groq/tts.py, heygen/base_api.py, hume/tts.py, inworld/tts.py, kokoro/tts.py, lmnt/tts.py, neuphonic/tts.py, openai/llm.py, openai/tts.py, openrouter/llm.py, piper/tts.py, resembleai/tts.py, sarvam/tts.py, smallest/tts.py, websocket_service.py, xtts/tts.py. --- pyrightconfig.json | 19 ----------------- src/pipecat/services/gladia/stt.py | 2 ++ src/pipecat/services/google/image.py | 13 ++++++++---- src/pipecat/services/google/llm.py | 2 +- src/pipecat/services/groq/tts.py | 26 ++++++++++++++++++++--- src/pipecat/services/heygen/base_api.py | 4 ++-- src/pipecat/services/hume/tts.py | 7 +++++- src/pipecat/services/inworld/tts.py | 6 ++++-- src/pipecat/services/kokoro/tts.py | 2 ++ src/pipecat/services/neuphonic/tts.py | 5 ++++- src/pipecat/services/openai/base_llm.py | 2 +- src/pipecat/services/openai/tts.py | 12 ++++++++++- src/pipecat/services/openrouter/llm.py | 5 ++++- src/pipecat/services/piper/tts.py | 2 ++ src/pipecat/services/resembleai/tts.py | 4 +++- src/pipecat/services/tts_service.py | 10 +++++---- src/pipecat/services/websocket_service.py | 2 +- src/pipecat/services/xtts/tts.py | 6 +++++- 18 files changed, 86 insertions(+), 43 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index d882b2106..0e54af4c7 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -37,58 +37,39 @@ "src/pipecat/services/deepgram/sagemaker/tts.py", "src/pipecat/services/deepgram/tts.py", "src/pipecat/services/elevenlabs/tts.py", - "src/pipecat/services/fish/tts.py", - "src/pipecat/services/gladia/stt.py", "src/pipecat/services/google/gemini_live/llm.py", "src/pipecat/services/google/gemini_live/vertex/llm.py", - "src/pipecat/services/google/image.py", "src/pipecat/services/google/llm.py", "src/pipecat/services/google/stt.py", "src/pipecat/services/google/tts.py", "src/pipecat/services/gradium/stt.py", - "src/pipecat/services/groq/tts.py", "src/pipecat/services/heygen/api_interactive_avatar.py", - "src/pipecat/services/heygen/base_api.py", "src/pipecat/services/heygen/client.py", "src/pipecat/services/heygen/video.py", - "src/pipecat/services/hume/tts.py", "src/pipecat/services/inworld/realtime/llm.py", - "src/pipecat/services/inworld/tts.py", - "src/pipecat/services/kokoro/tts.py", "src/pipecat/services/llm_service.py", - "src/pipecat/services/lmnt/tts.py", "src/pipecat/services/mem0/memory.py", "src/pipecat/services/mistral/stt.py", "src/pipecat/services/mistral/tts.py", "src/pipecat/services/moondream/vision.py", - "src/pipecat/services/neuphonic/tts.py", "src/pipecat/services/nvidia/stt.py", "src/pipecat/services/nvidia/tts.py", "src/pipecat/services/openai/base_llm.py", "src/pipecat/services/openai/image.py", - "src/pipecat/services/openai/llm.py", "src/pipecat/services/openai/realtime/llm.py", "src/pipecat/services/openai/responses/llm.py", "src/pipecat/services/openai/stt.py", - "src/pipecat/services/openai/tts.py", - "src/pipecat/services/openrouter/llm.py", - "src/pipecat/services/piper/tts.py", - "src/pipecat/services/resembleai/tts.py", "src/pipecat/services/rime/tts.py", "src/pipecat/services/sambanova/llm.py", "src/pipecat/services/sarvam/stt.py", - "src/pipecat/services/sarvam/tts.py", "src/pipecat/services/simli/video.py", - "src/pipecat/services/smallest/tts.py", "src/pipecat/services/speechmatics/stt.py", "src/pipecat/services/stt_service.py", "src/pipecat/services/tavus/video.py", "src/pipecat/services/tts_service.py", "src/pipecat/services/ultravox/llm.py", - "src/pipecat/services/websocket_service.py", "src/pipecat/services/whisper/stt.py", "src/pipecat/services/xai/realtime/llm.py", - "src/pipecat/services/xtts/tts.py", "src/pipecat/transports/base_output.py", "src/pipecat/transports/daily/transport.py", "src/pipecat/transports/heygen/transport.py", diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index cc8e5279b..26fea653c 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -542,6 +542,8 @@ class GladiaSTTService(WebsocketSTTService): logger.debug(f"{self}Connecting to Gladia WebSocket") + if self._session_url is None: + raise RuntimeError(f"{self} session URL is not initialized") self._websocket = await websocket_connect(self._session_url) self._connection_active = True diff --git a/src/pipecat/services/google/image.py b/src/pipecat/services/google/image.py index 1ed8adc2c..bada7d0d6 100644 --- a/src/pipecat/services/google/image.py +++ b/src/pipecat/services/google/image.py @@ -30,7 +30,7 @@ from pipecat.services.image_service import ImageGenService from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, assert_given try: - from google import genai + import google.genai as genai from google.genai import types except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -153,8 +153,12 @@ class GoogleImageGenService(ImageGenService): await self.start_ttfb_metrics() try: + model = assert_given(self._settings.model) + if model is None: + yield ErrorFrame("Google image generation model must be specified") + return response = await self._client.aio.models.generate_images( - model=self._settings.model, + model=model, prompt=prompt, config=types.GenerateImagesConfig( number_of_images=assert_given(self._settings.number_of_images), @@ -169,8 +173,9 @@ class GoogleImageGenService(ImageGenService): for img_response in response.generated_images: # Google returns the image data directly - image_bytes = img_response.image.image_bytes - image = Image.open(io.BytesIO(image_bytes)) + if img_response.image is None or img_response.image.image_bytes is None: + continue + image = Image.open(io.BytesIO(img_response.image.image_bytes)) frame = URLImageRawFrame( url=None, # Google doesn't provide URLs, only image data diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index da9659872..599c203ab 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -52,7 +52,7 @@ from pipecat.utils.tracing.service_decorators import traced_llm os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" try: - from google import genai + import google.genai as genai from google.api_core.exceptions import DeadlineExceeded from google.genai.types import ( GenerateContentConfig, diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 36e01ff8b..00038f2f0 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -10,6 +10,7 @@ import io import wave from collections.abc import AsyncGenerator from dataclasses import dataclass, field +from typing import Literal, cast from loguru import logger from pydantic import BaseModel @@ -31,6 +32,19 @@ except ModuleNotFoundError as e: logger.error("In order to use Groq, you need to `pip install pipecat-ai[groq]`.") raise Exception(f"Missing module: {e}") +# Hint set for `output_format`. The values mirror the Literal that +# `groq.resources.audio.speech.AsyncSpeech.create` accepts on its +# `response_format` parameter (also visible as the `response_format` field of +# `groq.types.audio.SpeechCreateParams`). The groq SDK does not export this as +# a named alias, so we redeclare it here. +# +# This alias is used in unions like `GroqAudioFormat | str`, so pyright shows +# these values as completion hints without rejecting other strings. If groq +# adds a new format before this list is updated, callers can still pass it and +# we forward it through (with a cast at the API boundary). Keep in sync on a +# best-effort basis when bumping the groq dep. +GroqAudioFormat = Literal["flac", "mp3", "mulaw", "ogg", "wav"] + @dataclass class GroqTTSSettings(TTSSettings): @@ -74,7 +88,7 @@ class GroqTTSService(TTSService): self, *, api_key: str, - output_format: str = "wav", + output_format: GroqAudioFormat | str = "wav", params: InputParams | None = None, model_name: str | None = None, voice_id: str | None = None, @@ -147,7 +161,7 @@ class GroqTTSService(TTSService): ) self._api_key = api_key - self._output_format = output_format + self._output_format: str = output_format self._client = AsyncGroq(api_key=self._api_key) @@ -178,12 +192,18 @@ class GroqTTSService(TTSService): speed = assert_given(self._settings.speed) if model is None: raise ValueError("Groq TTS model must be specified") + if voice is None: + raise ValueError("Groq TTS voice must be specified") if speed is None: raise ValueError("Groq TTS speed must be specified") response = await self._client.audio.speech.create( model=model, voice=voice, - response_format=self._output_format, + # Cast satisfies groq's stricter Literal typing while letting + # callers pass any string (e.g. a newer groq format we haven't + # yet added to GroqAudioFormat). If the value is unsupported, + # groq's API will surface a runtime error with a clear message. + response_format=cast(GroqAudioFormat, self._output_format), # Note: as of 2026-02-25, only a speed of 1.0 is supported, but # here we pass it for completeness and future-proofing speed=speed, diff --git a/src/pipecat/services/heygen/base_api.py b/src/pipecat/services/heygen/base_api.py index c866cfb11..57ca2e19e 100644 --- a/src/pipecat/services/heygen/base_api.py +++ b/src/pipecat/services/heygen/base_api.py @@ -33,8 +33,8 @@ class StandardSessionResponse(BaseModel): access_token: str livekit_agent_token: str - livekit_url: str = None - ws_url: str = None + livekit_url: str + ws_url: str raw_response: Any diff --git a/src/pipecat/services/hume/tts.py b/src/pipecat/services/hume/tts.py index ac17efd48..75a6d5b72 100644 --- a/src/pipecat/services/hume/tts.py +++ b/src/pipecat/services/hume/tts.py @@ -19,6 +19,7 @@ from pipecat import version as pipecat_version from pipecat.frames.frames import ( CancelFrame, EndFrame, + ErrorFrame, Frame, InterruptionFrame, StartFrame, @@ -289,10 +290,14 @@ class HumeTTSService(TTSService): """ logger.debug(f"{self}: Generating Hume TTS: [{text}]") + voice_id = assert_given(self._settings.voice) + if voice_id is None: + yield ErrorFrame(error="Hume TTS voice must be specified") + return # Build the request payload utterance_kwargs: dict[str, Any] = { "text": text, - "voice": PostedUtteranceVoiceWithId(id=assert_given(self._settings.voice)), + "voice": PostedUtteranceVoiceWithId(id=voice_id), } if self._settings.description is not None: utterance_kwargs["description"] = self._settings.description diff --git a/src/pipecat/services/inworld/tts.py b/src/pipecat/services/inworld/tts.py index ef3f0f956..0e087082c 100644 --- a/src/pipecat/services/inworld/tts.py +++ b/src/pipecat/services/inworld/tts.py @@ -794,8 +794,10 @@ class InworldTTSService(WebsocketTTSService): return word_times - async def _close_context(self, context_id: str): - if context_id and self._websocket: + async def _close_context(self, context_id: str | None): + if not context_id: + return + if self._websocket: logger.info(f"{self}: Closing context {context_id} due to interruption or completion") try: await self._send_close_context(context_id) diff --git a/src/pipecat/services/kokoro/tts.py b/src/pipecat/services/kokoro/tts.py index 81c90487f..dd96ee90b 100644 --- a/src/pipecat/services/kokoro/tts.py +++ b/src/pipecat/services/kokoro/tts.py @@ -216,6 +216,8 @@ class KokoroTTSService(TTSService): await self.start_tts_usage_metrics(text) voice = assert_given(self._settings.voice) + if voice is None: + raise ValueError("Kokoro TTS voice must be specified") lang = assert_given(self._settings.language) if lang is None: raise ValueError("Kokoro TTS language must be specified") diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index 6a405a1d2..5f27efaf1 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -331,7 +331,10 @@ class NeuphonicTTSService(InterruptibleTTSService): async def _receive_messages(self): """Receive and process messages from Neuphonic WebSocket.""" - async for message in self._websocket: + websocket = self._websocket + if websocket is None: + return + async for message in websocket: if isinstance(message, str): msg = json.loads(message) if msg.get("data") and msg["data"].get("audio"): diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index b5287067f..ae4b03560 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -66,7 +66,7 @@ class OpenAILLMSettings(LLMSettings): ) top_p: float | None | _NotGiven | OpenAINotGiven = field(default_factory=lambda: _NOT_GIVEN) max_tokens: int | None | _NotGiven | OpenAINotGiven = field(default_factory=lambda: _NOT_GIVEN) - max_completion_tokens: int | _NotGiven | OpenAINotGiven = field( + max_completion_tokens: int | None | _NotGiven | OpenAINotGiven = field( default_factory=lambda: _NOT_GIVEN ) diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index a0e6a93dd..a6528f59e 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -235,12 +235,22 @@ class OpenAITTSService(TTSService): Frame: Audio frames containing the synthesized speech data. """ logger.debug(f"{self}: Generating TTS [{text}]") + voice = assert_given(self._settings.voice) + if voice is None: + yield ErrorFrame(error="OpenAI TTS voice must be specified") + return + if voice not in VALID_VOICES: + yield ErrorFrame( + error=f"OpenAI TTS voice {voice!r} is not supported " + f"(must be one of: {', '.join(sorted(VALID_VOICES))})" + ) + return try: # Setup API parameters create_params = { "input": text, "model": self._settings.model, - "voice": VALID_VOICES[assert_given(self._settings.voice)], + "voice": VALID_VOICES[voice], "response_format": "pcm", } diff --git a/src/pipecat/services/openrouter/llm.py b/src/pipecat/services/openrouter/llm.py index 624dd3141..4de5190dd 100644 --- a/src/pipecat/services/openrouter/llm.py +++ b/src/pipecat/services/openrouter/llm.py @@ -15,6 +15,7 @@ from typing import Any from loguru import logger +from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams from pipecat.services.openai.base_llm import BaseOpenAILLMService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.settings import assert_given @@ -96,7 +97,9 @@ class OpenRouterLLMService(OpenAILLMService): logger.debug(f"Creating OpenRouter client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) - def build_chat_completion_params(self, params_from_context: dict[str, Any]) -> dict[str, Any]: + def build_chat_completion_params( + self, params_from_context: OpenAILLMInvocationParams + ) -> dict[str, Any]: """Builds chat parameters, handling model-specific constraints. Args: diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 5cc725c2c..1bb9eaac8 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -101,6 +101,8 @@ class PiperTTSService(TTSService): download_dir = download_dir or Path.cwd() _voice = assert_given(self._settings.voice) + if _voice is None: + raise ValueError("Piper TTS voice must be specified") model_file = f"{_voice}.onnx" model_path_resolved = Path(download_dir) / model_file diff --git a/src/pipecat/services/resembleai/tts.py b/src/pipecat/services/resembleai/tts.py index 3053e0b70..c1dfc8a92 100644 --- a/src/pipecat/services/resembleai/tts.py +++ b/src/pipecat/services/resembleai/tts.py @@ -409,7 +409,9 @@ class ResembleAITTSService(WebsocketTTSService): await self.push_frame(TTSStoppedFrame(context_id=context_id)) await self.stop_all_metrics() - await self.push_error(ErrorFrame(error=f"{self} error: {error_name} - {error_msg}")) + await self.push_error_frame( + ErrorFrame(error=f"{self} error: {error_name} - {error_msg}") + ) # Check if this is an unrecoverable error (connection-level failure) if status_code in [401, 403]: diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index f386b5bcb..5cc4a4b45 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -1174,16 +1174,18 @@ class TTSService(AIService): logger.trace(f"{self} created audio context {context_id}") async def append_to_audio_context( - self, context_id: str, frame: Frame | _WordTimestampEntry | None + self, context_id: str | None, frame: Frame | _WordTimestampEntry | None ): """Append a frame or word-timestamp entry to an existing audio context queue. - Passing ``None`` signals end-of-context (used by remove_audio_context to mark - the queue for deletion). If the context no longer exists but the context_id + Passing a ``frame`` of ``None`` signals end-of-context (used by remove_audio_context + to mark the queue for deletion). If the context no longer exists but the context_id matches the active turn, the context is transparently recreated before appending. Args: - context_id: The context to append to. + context_id: The context to append to. ``None`` is accepted as a no-op + (with a debug log) so callers can pass through values from + ``get_active_audio_context_id()`` without an explicit guard. frame: The frame, word-timestamp entry, or ``None`` (end-of-context sentinel) to append. """ diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 2e6040ead..d7d3cd505 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -42,7 +42,7 @@ class WebsocketService(ABC): reconnect_on_error: Whether to automatically reconnect on connection errors. **kwargs: Additional arguments (unused, for compatibility). """ - self._websocket: websockets.WebSocketClientProtocol | None = None + self._websocket: websockets.WebSocketClientProtocol | None = None # pyright: ignore[reportAttributeAccessIssue] self._reconnect_on_error = reconnect_on_error self._reconnect_in_progress: bool = False self._disconnecting: bool = False diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index d323e9b67..0d1a068f7 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -214,7 +214,11 @@ class XTTSService(TTSService): logger.error(f"{self} no studio speakers available") return - embeddings = self._studio_speakers[assert_given(self._settings.voice)] + voice = assert_given(self._settings.voice) + if voice is None: + yield ErrorFrame(error="XTTS voice must be specified") + return + embeddings = self._studio_speakers[voice] url = self._base_url + "/tts_stream" From 31ff07916f7223f2dfb69b664d1224e6981b3708 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Apr 2026 12:48:16 -0400 Subject: [PATCH 20/23] fix: clear 10 more services from pyright ignore list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second pass over the low-error-count files in the ignore list. Drops 10 files (77 → 67) and full-pyright errors from 580 → 555. Default pyright stays clean. Three coherent shapes plus a handful of one-offs: `Language | str | None` → `Language | None` at STT frame boundaries. `assert_given(self._settings.language)` returns `Language | str | None` (strips `_NotGiven`, keeps the rest), but `TranscriptionFrame.language` expects `Language | None`. In practice both `_settings.language` and SDK-supplied codes resolve to a `Language` enum value, but technically they could be raw strings — and `Language` is a StrEnum, so downstream consumers (which mostly compare/serialize as strings) handle either. Used `cast("Language | None", ...)` at each call site rather than a runtime-validating helper, so an unrecognised code (e.g. one we haven't added to the enum yet) still flows through unchanged. Cleared azure/stt.py, aws/stt.py, gradium/stt.py; mistral/stt.py keeps the cast at the SDK boundary (storing under `_detected_language: Language | None`) but stays in the ignore list because of two unrelated Optional-access errors. aiobotocore `async with` stub gap. `aioboto3.Session().client(...)` is an async context manager at runtime but its stubs don't advertise `__aenter__`/`__aexit__` to pyright. Scoped `# pyright: ignore[reportGeneralTypeIssues]` on the two affected sites: aws/agent_core.py and aws/tts.py. aws/tts.py also had a latent bug on the no-`AudioStream` path: the original code set `audio_data = None` and then crashed in `resample(...)` and `len(audio_data)` below; replaced with an early `return` after logging — matches the convention elsewhere (OpenAI TTS, etc.) of not recording usage metrics on the error path. heygen `event_id: str | None` → `str` at transport→client boundary. Three call sites in transports/heygen/transport.py passed `self._event_id` (`str | None`) into client methods that take `str`. Added a guard at each: `agent_speak_end` and `interrupt` only fire when `_event_id` is set; `write_audio_frame` warn-and-drops when there's no active bot event rather than sending a malformed message. `OpenAIResponsesLLMInvocationParams` TypedDict. `get_llm_invocation_params` always sets both `input` and `tools` in the same dict literal, but the TypedDict was `total=False` so direct subscript access (`invocation_params["input"]`) tripped `reportTypedDictNotRequiredAccess` in services/openai/responses/llm.py. Marked both keys `Required[...]`; `instructions` stays non-required since it's only added when a system instruction is present. Latent bug in heygen/api_interactive_avatar.py: the code accessed `request_data.voice.voiceId` and `request_data.voice.elevenlabsSettings`, but those names are Pydantic *aliases*; the actual attribute names (used for attribute access) are `voice_id` and `elevenlabs_settings`. Switched to the field names — those camelCase accesses would have raised AttributeError at runtime if `voice` was set. Other small fixes: - assemblyai/stt.py: the deprecated `connection_params=` init path was reading `formatted_finals` and `word_finalization_max_wait_time` off `AssemblyAIConnectionParams`, but those fields were never on the deprecated input model — they were added to Settings later. Removed the reads (with a comment noting they're only available via the canonical `settings=...` API); the deprecated input model is unchanged. - rtvi/processor.py: two `about: Mapping[str, Any] = None` parameter signatures — declared `Mapping`, defaulted to `None`, and both function bodies already handled the None case. Widened to `Mapping[str, Any] | None = None`. - aws/stt.py: `subprotocols=["mqtt"]` failed against websockets' `Sequence[Subprotocol] | None` (Subprotocol is a NewType wrapper). Wrapped: `subprotocols=[Subprotocol("mqtt")]`. Files dropped from the ignore list (77 → 67): processors/frameworks/rtvi/processor.py, services/assemblyai/stt.py, services/aws/agent_core.py, services/aws/stt.py, services/aws/tts.py, services/azure/stt.py, services/gradium/stt.py, services/heygen/api_interactive_avatar.py, services/openai/responses/llm.py, transports/heygen/transport.py. --- pyrightconfig.json | 10 ---------- .../services/open_ai_responses_adapter.py | 8 +++++--- .../processors/frameworks/rtvi/processor.py | 4 ++-- src/pipecat/services/assemblyai/stt.py | 9 +++++---- src/pipecat/services/aws/agent_core.py | 6 +++++- src/pipecat/services/aws/stt.py | 11 ++++++++--- src/pipecat/services/aws/tts.py | 8 ++++++-- src/pipecat/services/azure/stt.py | 16 +++++++++++----- src/pipecat/services/gradium/stt.py | 16 +++++++++++----- .../services/heygen/api_interactive_avatar.py | 4 ++-- src/pipecat/services/mistral/stt.py | 9 ++++++--- src/pipecat/transports/heygen/transport.py | 13 ++++++++++--- 12 files changed, 71 insertions(+), 43 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index 0e54af4c7..c928acdac 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -17,18 +17,12 @@ "src/pipecat/processors/frame_processor.py", "src/pipecat/processors/frameworks/langchain.py", "src/pipecat/processors/frameworks/rtvi/observer.py", - "src/pipecat/processors/frameworks/rtvi/processor.py", "src/pipecat/processors/frameworks/strands_agents.py", "src/pipecat/services/anthropic/llm.py", - "src/pipecat/services/assemblyai/stt.py", - "src/pipecat/services/aws/agent_core.py", "src/pipecat/services/aws/llm.py", "src/pipecat/services/aws/nova_sonic/llm.py", "src/pipecat/services/aws/sagemaker/bidi_client.py", - "src/pipecat/services/aws/stt.py", - "src/pipecat/services/aws/tts.py", "src/pipecat/services/aws/utils.py", - "src/pipecat/services/azure/stt.py", "src/pipecat/services/azure/tts.py", "src/pipecat/services/deepgram/flux/base.py", "src/pipecat/services/deepgram/flux/sagemaker/stt.py", @@ -42,8 +36,6 @@ "src/pipecat/services/google/llm.py", "src/pipecat/services/google/stt.py", "src/pipecat/services/google/tts.py", - "src/pipecat/services/gradium/stt.py", - "src/pipecat/services/heygen/api_interactive_avatar.py", "src/pipecat/services/heygen/client.py", "src/pipecat/services/heygen/video.py", "src/pipecat/services/inworld/realtime/llm.py", @@ -57,7 +49,6 @@ "src/pipecat/services/openai/base_llm.py", "src/pipecat/services/openai/image.py", "src/pipecat/services/openai/realtime/llm.py", - "src/pipecat/services/openai/responses/llm.py", "src/pipecat/services/openai/stt.py", "src/pipecat/services/rime/tts.py", "src/pipecat/services/sambanova/llm.py", @@ -72,7 +63,6 @@ "src/pipecat/services/xai/realtime/llm.py", "src/pipecat/transports/base_output.py", "src/pipecat/transports/daily/transport.py", - "src/pipecat/transports/heygen/transport.py", "src/pipecat/transports/lemonslice/transport.py", "src/pipecat/transports/livekit/transport.py", "src/pipecat/transports/smallwebrtc/connection.py", diff --git a/src/pipecat/adapters/services/open_ai_responses_adapter.py b/src/pipecat/adapters/services/open_ai_responses_adapter.py index d19e7d117..6fa03d2d7 100644 --- a/src/pipecat/adapters/services/open_ai_responses_adapter.py +++ b/src/pipecat/adapters/services/open_ai_responses_adapter.py @@ -6,7 +6,7 @@ """OpenAI Responses API adapter for Pipecat.""" -from typing import Any, TypedDict, cast +from typing import Any, Required, TypedDict, cast from openai._types import NotGiven as OpenAINotGiven from openai.types.responses import FunctionToolParam, ResponseInputItemParam, ToolParam @@ -23,8 +23,10 @@ from pipecat.processors.aggregators.llm_context import ( class OpenAIResponsesLLMInvocationParams(TypedDict, total=False): """Context-based parameters for invoking OpenAI Responses API.""" - input: list[ResponseInputItemParam] - tools: list[ToolParam] | OpenAINotGiven + # `input` and `tools` are always populated by `get_llm_invocation_params`; + # `instructions` is only set when a system instruction is present. + input: Required[list[ResponseInputItemParam]] + tools: Required[list[ToolParam] | OpenAINotGiven] instructions: str diff --git a/src/pipecat/processors/frameworks/rtvi/processor.py b/src/pipecat/processors/frameworks/rtvi/processor.py index 5586ec8ae..5f5e02568 100644 --- a/src/pipecat/processors/frameworks/rtvi/processor.py +++ b/src/pipecat/processors/frameworks/rtvi/processor.py @@ -102,7 +102,7 @@ class RTVIProcessor(FrameProcessor): self._client_ready = True await self._call_event_handler("on_client_ready") - async def set_bot_ready(self, about: Mapping[str, Any] = None): + async def set_bot_ready(self, about: Mapping[str, Any] | None = None): """Mark the bot as ready and send the bot-ready message. Args: @@ -404,7 +404,7 @@ class RTVIProcessor(FrameProcessor): ) await self.push_frame(frame) - async def _send_bot_ready(self, about: Mapping[str, Any] = None): + async def _send_bot_ready(self, about: Mapping[str, Any] | None = None): """Send the bot-ready message to the client. Args: diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index f87a5a0e5..5eda4b7f6 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -233,10 +233,11 @@ class AssemblyAISTTService(WebsocketSTTService): sample_rate = connection_params.sample_rate encoding = connection_params.encoding default_settings.model = connection_params.speech_model - default_settings.formatted_finals = connection_params.formatted_finals - default_settings.word_finalization_max_wait_time = ( - connection_params.word_finalization_max_wait_time - ) + # Note: `formatted_finals` and `word_finalization_max_wait_time` + # were added to Settings after this deprecated input model + # was frozen and have no equivalent on + # AssemblyAIConnectionParams; they are only configurable via + # the canonical `settings=...` API. default_settings.end_of_turn_confidence_threshold = ( connection_params.end_of_turn_confidence_threshold ) diff --git a/src/pipecat/services/aws/agent_core.py b/src/pipecat/services/aws/agent_core.py index 2f1560b7c..d43374b4a 100644 --- a/src/pipecat/services/aws/agent_core.py +++ b/src/pipecat/services/aws/agent_core.py @@ -201,7 +201,11 @@ class AWSAgentCoreProcessor(FrameProcessor): if not payload: return - async with self._aws_session.client("bedrock-agentcore", **self._aws_params) as client: + # aioboto3's `client()` is an async context manager but its stubs don't + # advertise `__aenter__` / `__aexit__` in a way pyright can see. + async with self._aws_session.client( # pyright: ignore[reportGeneralTypeIssues] + "bedrock-agentcore", **self._aws_params + ) as client: # Invoke the AgentCore agent response = await client.invoke_agent_runtime( agentRuntimeArn=self._agentArn, payload=payload.encode() diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index aa1fdf9bf..c482b1b00 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -16,7 +16,7 @@ import random import string from collections.abc import AsyncGenerator from dataclasses import dataclass -from typing import Any +from typing import Any, cast from loguru import logger @@ -38,6 +38,7 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt try: + from websockets import Subprotocol from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State except ModuleNotFoundError as e: @@ -314,7 +315,7 @@ class AWSTranscribeSTTService(WebsocketSTTService): self._websocket = await websocket_connect( presigned_url, additional_headers=additional_headers, - subprotocols=["mqtt"], + subprotocols=[Subprotocol("mqtt")], ping_interval=None, ping_timeout=None, compression=None, @@ -534,7 +535,11 @@ class AWSTranscribeSTTService(WebsocketSTTService): is_final = not result.get("IsPartial", True) if transcript: - language = assert_given(self._settings.language) + # Technically `_settings.language` could be a raw string, but + # Language is a StrEnum so downstream handles either. + language = cast( + "Language | None", assert_given(self._settings.language) + ) if is_final: await self.push_frame( TranscriptionFrame( diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index a683a94ea..ec44202f9 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -345,7 +345,11 @@ class AWSPollyTTSService(TTSService): # Filter out None values filtered_params = {k: v for k, v in params.items() if v is not None} - async with self._aws_session.client("polly", **self._aws_params) as polly: + # aioboto3's `client()` is an async context manager but its stubs + # don't advertise `__aenter__` / `__aexit__` to pyright. + async with self._aws_session.client( # pyright: ignore[reportGeneralTypeIssues] + "polly", **self._aws_params + ) as polly: response = await polly.synthesize_speech(**filtered_params) if "AudioStream" in response: # Get the streaming body and read it @@ -353,7 +357,7 @@ class AWSPollyTTSService(TTSService): audio_data = await stream.read() else: logger.error(f"{self} No audio stream in response") - audio_data = None + return audio_data = await self._resampler.resample(audio_data, 16000, self.sample_rate) diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index e130fe980..be06e07cd 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -13,7 +13,7 @@ Speech SDK for real-time audio transcription. import asyncio from collections.abc import AsyncGenerator from dataclasses import dataclass -from typing import Any +from typing import Any, cast from loguru import logger @@ -280,8 +280,11 @@ class AzureSTTService(STTService): def _on_handle_recognized(self, event): if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or assert_given( - self._settings.language + # Technically either source could be a raw string, but Language is + # a StrEnum so downstream handles either. + language = cast( + "Language | None", + getattr(event.result, "language", None) or assert_given(self._settings.language), ) frame = TranscriptionFrame( event.result.text, @@ -297,8 +300,11 @@ class AzureSTTService(STTService): def _on_handle_recognizing(self, event): if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0: - language = getattr(event.result, "language", None) or assert_given( - self._settings.language + # Technically either source could be a raw string, but Language is + # a StrEnum so downstream handles either. + language = cast( + "Language | None", + getattr(event.result, "language", None) or assert_given(self._settings.language), ) frame = InterimTranscriptionFrame( event.result.text, diff --git a/src/pipecat/services/gradium/stt.py b/src/pipecat/services/gradium/stt.py index e58383b45..0e284d05b 100644 --- a/src/pipecat/services/gradium/stt.py +++ b/src/pipecat/services/gradium/stt.py @@ -15,7 +15,7 @@ import base64 import json from collections.abc import AsyncGenerator from dataclasses import dataclass, field -from typing import Any +from typing import Any, cast from loguru import logger from pydantic import BaseModel @@ -401,8 +401,10 @@ class GradiumSTTService(WebsocketSTTService): json_config = {} if self._json_config: json_config = json.loads(self._json_config) - language = assert_given(self._settings.language) - if language: + # Technically `_settings.language` could be a raw string, but + # Language is a StrEnum so downstream handles either. + language = cast("Language | None", assert_given(self._settings.language)) + if language is not None: gradium_language = language_to_gradium_language(language) if gradium_language: json_config["language"] = gradium_language @@ -482,12 +484,14 @@ class GradiumSTTService(WebsocketSTTService): """ self._accumulated_text.append(text) accumulated = " ".join(self._accumulated_text) + # Technically `_settings.language` could be a raw string, but Language + # is a StrEnum so downstream handles either. await self.push_frame( InterimTranscriptionFrame( text=accumulated, user_id=self._user_id, timestamp=time_now_iso8601(), - language=assert_given(self._settings.language), + language=cast("Language | None", assert_given(self._settings.language)), ) ) await self.stop_processing_metrics() @@ -519,7 +523,9 @@ class GradiumSTTService(WebsocketSTTService): text = " ".join(self._accumulated_text) self._accumulated_text.clear() logger.debug(f"Final transcription: [{text}]") - language = assert_given(self._settings.language) + # Technically `_settings.language` could be a raw string, but Language + # is a StrEnum so downstream handles either. + language = cast("Language | None", assert_given(self._settings.language)) await self.push_frame( TranscriptionFrame( text, diff --git a/src/pipecat/services/heygen/api_interactive_avatar.py b/src/pipecat/services/heygen/api_interactive_avatar.py index 4c50ceaa3..ff6a401d0 100644 --- a/src/pipecat/services/heygen/api_interactive_avatar.py +++ b/src/pipecat/services/heygen/api_interactive_avatar.py @@ -210,11 +210,11 @@ class HeyGenApi(BaseAvatarApi): "quality": request_data.quality, "avatar_id": request_data.avatar_id, "voice": { - "voice_id": request_data.voice.voiceId if request_data.voice else None, + "voice_id": request_data.voice.voice_id if request_data.voice else None, "rate": request_data.voice.rate if request_data.voice else None, "emotion": request_data.voice.emotion if request_data.voice else None, "elevenlabs_settings": ( - request_data.voice.elevenlabsSettings if request_data.voice else None + request_data.voice.elevenlabs_settings if request_data.voice else None ), }, "knowledge_id": request_data.knowledge_id, diff --git a/src/pipecat/services/mistral/stt.py b/src/pipecat/services/mistral/stt.py index 85d218628..bbbeddf04 100644 --- a/src/pipecat/services/mistral/stt.py +++ b/src/pipecat/services/mistral/stt.py @@ -12,7 +12,7 @@ Voxtral Realtime transcription API using the Mistral SDK's RealtimeConnection. from collections.abc import AsyncGenerator from dataclasses import dataclass -from typing import Any +from typing import Any, cast from loguru import logger @@ -30,6 +30,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.settings import STTSettings, assert_given from pipecat.services.stt_latency import MISTRAL_TTFS_P99 from pipecat.services.stt_service import STTService +from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt @@ -132,7 +133,7 @@ class MistralSTTService(STTService): self._connection: RealtimeConnection | None = None self._receive_task = None self._accumulated_text = "" - self._detected_language: str | None = None + self._detected_language: Language | None = None def can_generate_metrics(self) -> bool: """Check if the service can generate processing metrics. @@ -278,7 +279,9 @@ class MistralSTTService(STTService): self._accumulated_text = "" elif isinstance(event, TranscriptionStreamLanguage): - self._detected_language = event.audio_language + # Technically the SDK could emit a code we haven't added yet, + # but Language is a StrEnum so downstream handles either. + self._detected_language = cast("Language | None", event.audio_language) elif isinstance(event, RealtimeTranscriptionError): error_msg = event.error.message if event.error else "Unknown error" diff --git a/src/pipecat/transports/heygen/transport.py b/src/pipecat/transports/heygen/transport.py index 82a0ae3d0..0558445af 100644 --- a/src/pipecat/transports/heygen/transport.py +++ b/src/pipecat/transports/heygen/transport.py @@ -239,8 +239,9 @@ class HeyGenOutputTransport(BaseOutputTransport): logger.warning("self._event_id is already defined!") self._event_id = str(frame.id) elif isinstance(frame, BotStoppedSpeakingFrame): - await self._client.agent_speak_end(self._event_id) - self._event_id = None + if self._event_id is not None: + await self._client.agent_speak_end(self._event_id) + self._event_id = None await super().push_frame(frame, direction) async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -261,7 +262,8 @@ class HeyGenOutputTransport(BaseOutputTransport): """ await super().process_frame(frame, direction) if isinstance(frame, InterruptionFrame): - await self._client.interrupt(self._event_id) + if self._event_id is not None: + await self._client.interrupt(self._event_id) await self.push_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): await self._client.start_agent_listening() @@ -281,6 +283,11 @@ class HeyGenOutputTransport(BaseOutputTransport): audio = frame.audio if frame.sample_rate != HEY_GEN_SAMPLE_RATE: audio = await self._resampler.resample(audio, frame.sample_rate, HEY_GEN_SAMPLE_RATE) + if self._event_id is None: + # No active bot-speech event — drop the chunk rather than send a + # message the HeyGen API will reject. + logger.warning(f"{self}: dropping audio frame because no event_id is set") + return False await self._client.agent_speak(bytes(audio), self._event_id) return True From 26a40e2e629e90f3e213960ff8e278fed0c0eed8 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 29 Apr 2026 15:23:18 -0400 Subject: [PATCH 21/23] fix: clear 10 more services from pyright ignore list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A third pass over low-error-count files in the ignore list. Drops 10 files (67 → 57) and full-pyright errors from 555 → 525. Default pyright stays clean. Optional access guards (4 files). The same fix shape as 9e9b1f39e: a receiver typed `X | None` accessed without a guard, fixed with a local-var capture or an early return. - `mistral/stt.py`: `_connection.send_audio` could crash if `_connect()` swallowed an exception and left `_connection` unset; drop the audio chunk with a warning instead. `_receive_events` iterating `_connection.events()` got the same defensive narrowing. - `deepgram/flux/stt.py`: `_websocket_url` is set in `_connect` before `_connect_websocket` is called, but pyright doesn't track that across methods — assert at the use site. `websocket.response` is `Response | None` in the websockets stubs even though it's always populated post-handshake; guarded with a fallback. - `audio/filters/rnnoise_filter.py`: the module-level import sets `RNNoise` to `None` if `pyrnnoise` isn't installed; raise `ImportError` explicitly instead of relying on the existing try- block to catch the `None(...)` call. Also gated `filter()` with `or self._rnnoise is None` so pyright sees the narrowing. - `transports/smallwebrtc/request_handler.py`: `get_answer()` legitimately returns `None`; raise instead of crashing on three subscript accesses. `TTSService` `audio-context` API tightening. Mirroring the `append_to_audio_context` fix from the previous batch: `remove_audio_context` was typed `str` but is called with `str | None` from `get_active_audio_context_id()` results. Widened to `str | None` and the `None` handling lives in the function body (early debug log + return) — matching `append_to_audio_context`'s shape. `audio_context_available` keeps its narrow `str` signature; asking "is `None` available?" isn't a meaningful question (`_audio_contexts` is `dict[str, asyncio.Queue]`). The internal call site in `on_turn_context_completed` narrows `_turn_context_id` explicitly before passing it. Side effect: deepgram/tts.py's L307 error clears without local changes. `deepgram/tts.py` (4 errors → 0): the same `push_error(ErrorFrame(...))` latent bug we fixed in resembleai earlier in this PR — `push_error` takes a string; there's a separate `push_error_frame` for frames. Two sites switched. The Optional `_websocket.response` access is guarded the same way as deepgram/flux/stt.py. The `remove_audio_context` error was cleared by the tightening above. `aws/utils.py` (3 errors → 0): `AWSTranscribePresignedURL` declared `session_token: str` but the dict source is `str | None` (AWS supports long-term IAM creds without a session token). Same for `vocabulary_name`/`vocabulary_filter_name` on `get_request_url`, which were typed `str = ""` even though the body uses truthy checks to skip them. Widened to `str | None = None` — matches actual runtime semantics. `audio/dtmf/utils.py` (2 errors → 0): `files("...").joinpath(...)` returns a `Traversable`, but `aiofiles.open` wants a real path. For regular pip installs this worked in practice (Traversable was a `Path`), but it would fail for zipped distributions (zipapp, zipimport) where the resource isn't on disk. Wrapped in `importlib.resources.as_file(...)` — the canonical bridge that extracts to a temp file when the resource isn't already on the filesystem. Validated end-to-end: regular install still reads bytes; ad-hoc zipapp test confirmed `as_file` extracts the resource and returns a real Path. `openai/image.py` (2 errors → 0): the `size` arg to `images.generate` is `Literal[...] | None` in the SDK but our settings field is `str | None`. Mirrored the `groq/tts.py` hint-not-constraint pattern from the previous batch: defined a module-level `OpenAIImageSize = Literal[...]` alias with a comment attributing the upstream symbol and documenting the cast contract (callers can pass any string; invalid values surface as an OpenAI API error). Also guarded `image.data[0]` (response.data is `list[Image] | None`). `processors/frameworks/{langchain,strands_agents}.py` (4 + 4 → 0): both processors do `messages[-1]["content"]` on a value typed `LLMStandardMessage | LLMSpecificMessage` (the latter is a dataclass, not a dict, so `__getitem__` errors). Historically these only handled plain-text user messages, so the fix is two explicit guards (skip if the last message isn't a dict; skip if `content` isn't a string) plus a TODO noting that other shapes (multi-modal content, provider-specific messages) aren't supported yet. langchain's `__get_token_value` also got a small fix where `AIMessageChunk.content` is `str | list[parts]` but the function declares `-> str`; stringify the list case. strands_agents' surfaced two unrelated narrows: a `graph_exit_node: str | None` arg gated by an `__init__`-time assert, and `agent.stream_async` reached only when we're not in graph mode. Files dropped from the ignore list (67 → 57): audio/dtmf/utils.py, audio/filters/rnnoise_filter.py, processors/frameworks/langchain.py, processors/frameworks/strands_agents.py, services/aws/utils.py, services/deepgram/flux/stt.py, services/deepgram/tts.py, services/mistral/stt.py, services/openai/image.py, transports/smallwebrtc/request_handler.py. --- pyrightconfig.json | 10 ------ src/pipecat/audio/dtmf/utils.py | 12 ++++--- src/pipecat/audio/filters/rnnoise_filter.py | 9 ++++-- .../processors/frameworks/langchain.py | 20 ++++++++++-- .../processors/frameworks/strands_agents.py | 18 +++++++++-- src/pipecat/services/aws/utils.py | 12 ++++--- src/pipecat/services/deepgram/flux/stt.py | 13 +++++--- src/pipecat/services/deepgram/tts.py | 14 ++++---- src/pipecat/services/mistral/stt.py | 14 +++++++- src/pipecat/services/openai/image.py | 32 +++++++++++++++++-- src/pipecat/services/tts_service.py | 15 ++++++--- .../transports/smallwebrtc/request_handler.py | 2 ++ 12 files changed, 126 insertions(+), 45 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index c928acdac..db78dad8c 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -6,30 +6,23 @@ "exclude": ["**/*_pb2.py", "**/__pycache__"], "ignore": [ "tests", - "src/pipecat/audio/dtmf/utils.py", "src/pipecat/audio/filters/aic_filter.py", "src/pipecat/audio/filters/krisp_viva_filter.py", - "src/pipecat/audio/filters/rnnoise_filter.py", "src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py", "src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py", "src/pipecat/audio/vad/silero.py", "src/pipecat/processors/aggregators/llm_response_universal.py", "src/pipecat/processors/frame_processor.py", - "src/pipecat/processors/frameworks/langchain.py", "src/pipecat/processors/frameworks/rtvi/observer.py", - "src/pipecat/processors/frameworks/strands_agents.py", "src/pipecat/services/anthropic/llm.py", "src/pipecat/services/aws/llm.py", "src/pipecat/services/aws/nova_sonic/llm.py", "src/pipecat/services/aws/sagemaker/bidi_client.py", - "src/pipecat/services/aws/utils.py", "src/pipecat/services/azure/tts.py", "src/pipecat/services/deepgram/flux/base.py", "src/pipecat/services/deepgram/flux/sagemaker/stt.py", - "src/pipecat/services/deepgram/flux/stt.py", "src/pipecat/services/deepgram/sagemaker/stt.py", "src/pipecat/services/deepgram/sagemaker/tts.py", - "src/pipecat/services/deepgram/tts.py", "src/pipecat/services/elevenlabs/tts.py", "src/pipecat/services/google/gemini_live/llm.py", "src/pipecat/services/google/gemini_live/vertex/llm.py", @@ -41,13 +34,11 @@ "src/pipecat/services/inworld/realtime/llm.py", "src/pipecat/services/llm_service.py", "src/pipecat/services/mem0/memory.py", - "src/pipecat/services/mistral/stt.py", "src/pipecat/services/mistral/tts.py", "src/pipecat/services/moondream/vision.py", "src/pipecat/services/nvidia/stt.py", "src/pipecat/services/nvidia/tts.py", "src/pipecat/services/openai/base_llm.py", - "src/pipecat/services/openai/image.py", "src/pipecat/services/openai/realtime/llm.py", "src/pipecat/services/openai/stt.py", "src/pipecat/services/rime/tts.py", @@ -66,7 +57,6 @@ "src/pipecat/transports/lemonslice/transport.py", "src/pipecat/transports/livekit/transport.py", "src/pipecat/transports/smallwebrtc/connection.py", - "src/pipecat/transports/smallwebrtc/request_handler.py", "src/pipecat/transports/smallwebrtc/transport.py", "src/pipecat/transports/tavus/transport.py", "src/pipecat/transports/websocket/client.py", diff --git a/src/pipecat/audio/dtmf/utils.py b/src/pipecat/audio/dtmf/utils.py index 22026759e..dc921ea9b 100644 --- a/src/pipecat/audio/dtmf/utils.py +++ b/src/pipecat/audio/dtmf/utils.py @@ -14,7 +14,7 @@ in-memory after first load to improve performance on subsequent accesses. import asyncio import io import wave -from importlib.resources import files +from importlib.resources import as_file, files import aiofiles @@ -52,10 +52,12 @@ async def load_dtmf_audio(button: KeypadEntry, *, sample_rate: int = 8000) -> by __DTMF_RESAMPLER__ = create_file_resampler() dtmf_file_name = __DTMF_FILE_NAME.get(button, f"dtmf-{button.value}.wav") - dtmf_file_path = files("pipecat.audio.dtmf").joinpath(dtmf_file_name) - - async with aiofiles.open(dtmf_file_path, "rb") as f: - data = await f.read() + # `as_file` materialises the resource as a real filesystem `Path`, + # which aiofiles can open. (For installed packages this is just the + # bundled file; for zipped distributions it would extract to a temp.) + with as_file(files("pipecat.audio.dtmf").joinpath(dtmf_file_name)) as dtmf_file_path: + async with aiofiles.open(dtmf_file_path, "rb") as f: + data = await f.read() with io.BytesIO(data) as buffer: with wave.open(buffer, "rb") as wf: diff --git a/src/pipecat/audio/filters/rnnoise_filter.py b/src/pipecat/audio/filters/rnnoise_filter.py index 8d81d5b0f..1a316e2c5 100644 --- a/src/pipecat/audio/filters/rnnoise_filter.py +++ b/src/pipecat/audio/filters/rnnoise_filter.py @@ -60,7 +60,12 @@ class RNNoiseFilter(BaseAudioFilter): self._sample_rate = sample_rate try: - # RNNoise always requires 48kHz + # The module-level import sets `RNNoise` to `None` if pyrnnoise + # isn't installed; raise instead of calling `None(...)` so the + # except clause handles it cleanly. + if RNNoise is None: + raise ImportError("pyrnnoise is not installed") + # RNNoise always requires 48kHz. self._rnnoise = RNNoise(sample_rate=48000) self._rnnoise_ready = True except Exception as e: @@ -107,7 +112,7 @@ class RNNoiseFilter(BaseAudioFilter): Returns: Noise-suppressed audio data as bytes. """ - if not self._rnnoise_ready or not self._filtering: + if not self._rnnoise_ready or not self._filtering or self._rnnoise is None: return audio # Resample input if needed diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index 4400327fc..c5d746bb2 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -67,9 +67,20 @@ class LangchainProcessor(FrameProcessor): # The last one by the human is the one we want to send to the LLM. logger.debug(f"Got transcription frame {frame}") messages = frame.context.get_messages() - text: str = messages[-1]["content"] + # Historically this processor has only handled plain-text user + # messages; the guards below make that contract explicit for the + # type checker. TODO: maybe handle other message shapes (provider- + # specific messages, multi-modal content lists, etc.). + last_message = messages[-1] if messages else None + if not isinstance(last_message, dict): + await self.push_frame(frame, direction) + return + content = last_message.get("content") + if not isinstance(content, str): + await self.push_frame(frame, direction) + return - await self._ainvoke(text.strip()) + await self._ainvoke(content.strip()) else: await self.push_frame(frame, direction) @@ -87,7 +98,10 @@ class LangchainProcessor(FrameProcessor): case str(): return text case AIMessageChunk(): - return text.content + # `content` is `str | list[...]` (multi-modal); stringify if + # it's a list, since downstream consumers want plain text. + content = text.content + return content if isinstance(content, str) else str(content) case _: return "" diff --git a/src/pipecat/processors/frameworks/strands_agents.py b/src/pipecat/processors/frameworks/strands_agents.py index 7383cd089..ba028576e 100644 --- a/src/pipecat/processors/frameworks/strands_agents.py +++ b/src/pipecat/processors/frameworks/strands_agents.py @@ -71,9 +71,15 @@ class StrandsAgentsProcessor(FrameProcessor): await super().process_frame(frame, direction) if isinstance(frame, LLMContextFrame): messages = frame.context.get_messages() - if messages: - last_message = messages[-1] - await self._ainvoke(str(last_message["content"]).strip()) + # Historically this processor has only handled plain-text user + # messages; the guards below make that contract explicit for the + # type checker. TODO: handle other message shapes (provider- + # specific messages, multi-modal content lists, etc.). + last_message = messages[-1] if messages else None + if isinstance(last_message, dict): + content = last_message.get("content") + if isinstance(content, str): + await self._ainvoke(content.strip()) else: await self.push_frame(frame, direction) @@ -91,6 +97,9 @@ class StrandsAgentsProcessor(FrameProcessor): await self.start_ttfb_metrics() if self.graph: + # `__init__` asserts `graph_exit_node` is set whenever `graph` + # is, so this can't be None here. + assert self.graph_exit_node is not None # Graph does not stream; await full result then emit assistant text graph_result = await self.graph.invoke_async(text) if ttfb_tracking: @@ -115,6 +124,9 @@ class StrandsAgentsProcessor(FrameProcessor): except Exception as parse_err: logger.warning(f"Failed to extract messages from GraphResult: {parse_err}") else: + # `__init__` asserts at least one of `agent`/`graph` is set, + # and we're in the `not self.graph` branch. + assert self.agent is not None # Agent supports streaming events via async iterator async for event in self.agent.stream_async(text): # Push to TTS service diff --git a/src/pipecat/services/aws/utils.py b/src/pipecat/services/aws/utils.py index 2b69cf035..c14f1f20d 100644 --- a/src/pipecat/services/aws/utils.py +++ b/src/pipecat/services/aws/utils.py @@ -92,14 +92,18 @@ class AWSTranscribePresignedURL: """ def __init__( - self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1" + self, + access_key: str, + secret_key: str, + session_token: str | None, + region: str = "us-east-1", ): """Initialize the presigned URL generator. Args: access_key: AWS access key ID. secret_key: AWS secret access key. - session_token: AWS session token for temporary credentials. + session_token: AWS session token for temporary credentials (optional). region: AWS region for the service. Defaults to "us-east-1". """ self.access_key = access_key @@ -129,8 +133,8 @@ class AWSTranscribePresignedURL: sample_rate: int, language_code: str = "", media_encoding: str = "pcm", - vocabulary_name: str = "", - vocabulary_filter_name: str = "", + vocabulary_name: str | None = None, + vocabulary_filter_name: str | None = None, show_speaker_label: bool = False, enable_channel_identification: bool = False, number_of_channels: int = 1, diff --git a/src/pipecat/services/deepgram/flux/stt.py b/src/pipecat/services/deepgram/flux/stt.py index 98cf17ec2..f89b37f0b 100644 --- a/src/pipecat/services/deepgram/flux/stt.py +++ b/src/pipecat/services/deepgram/flux/stt.py @@ -299,14 +299,19 @@ class DeepgramFluxSTTService(DeepgramFluxSTTBase, WebsocketService): self._connection_established_event.clear() self._user_is_speaking = False - self._websocket = await websocket_connect( + # `_connect` sets `_websocket_url` before calling us; the assert + # narrows for pyright. + assert self._websocket_url is not None + websocket = await websocket_connect( self._websocket_url, additional_headers={"Authorization": f"Token {self._api_key}"}, ) + self._websocket = websocket - headers = { - k: v for k, v in self._websocket.response.headers.items() if k.startswith("dg-") - } + # `response` is populated after the handshake completes (which it + # has, since `websocket_connect` already returned). + response_headers = websocket.response.headers if websocket.response else {} + headers = {k: v for k, v in response_headers.items() if k.startswith("dg-")} logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}') # Creating the receiver task diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index 58b0fb81a..c1bb34083 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -232,17 +232,19 @@ class DeepgramTTSService(WebsocketTTSService): headers = {"Authorization": f"Token {self._api_key}"} - self._websocket = await websocket_connect(url, additional_headers=headers) + websocket = await websocket_connect(url, additional_headers=headers) + self._websocket = websocket - headers = { - k: v for k, v in self._websocket.response.headers.items() if k.startswith("dg-") - } + # `response` is populated after the handshake completes (which it + # has, since `websocket_connect` already returned). + response_headers = websocket.response.headers if websocket.response else {} + headers = {k: v for k, v in response_headers.items() if k.startswith("dg-")} logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}') await self._call_event_handler("on_connected") except Exception as e: logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error_frame(ErrorFrame(error=f"{self} error: {e}")) self._websocket = None await self._call_event_handler("on_connection_error", f"{e}") @@ -258,7 +260,7 @@ class DeepgramTTSService(WebsocketTTSService): await self._websocket.close() except Exception as e: logger.error(f"{self} exception: {e}") - await self.push_error(ErrorFrame(error=f"{self} error: {e}")) + await self.push_error_frame(ErrorFrame(error=f"{self} error: {e}")) finally: self._websocket = None await self._call_event_handler("on_disconnected") diff --git a/src/pipecat/services/mistral/stt.py b/src/pipecat/services/mistral/stt.py index bbbeddf04..d2f212663 100644 --- a/src/pipecat/services/mistral/stt.py +++ b/src/pipecat/services/mistral/stt.py @@ -198,6 +198,13 @@ class MistralSTTService(STTService): if not self._connection or self._connection.is_closed: await self._connect() + # `_connect` swallows exceptions and may leave `_connection` unset; + # drop the audio chunk rather than crashing if reconnect failed. + if self._connection is None: + logger.warning(f"{self}: dropping audio chunk — Mistral STT not connected") + yield None + return + await self._connection.send_audio(audio) yield None @@ -248,8 +255,13 @@ class MistralSTTService(STTService): async def _receive_events(self): """Background task: iterate connection events and handle them.""" + # `_connect` started this task only after assigning `_connection`, + # so it should not be None here; bail out defensively just in case. + connection = self._connection + if connection is None: + return try: - async for event in self._connection.events(): + async for event in connection.events(): if isinstance(event, RealtimeTranscriptionSessionCreated): logger.debug(f"{self}: Session created: {event.session}") await self._call_event_handler("on_connected") diff --git a/src/pipecat/services/openai/image.py b/src/pipecat/services/openai/image.py index 45f21a7cb..73dcf0b2b 100644 --- a/src/pipecat/services/openai/image.py +++ b/src/pipecat/services/openai/image.py @@ -13,7 +13,7 @@ for creating images from text prompts. import io from collections.abc import AsyncGenerator from dataclasses import dataclass, field -from typing import Literal +from typing import Literal, cast import aiohttp from loguru import logger @@ -28,6 +28,28 @@ from pipecat.frames.frames import ( from pipecat.services.image_service import ImageGenService from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven, assert_given +# Hint set for the `size` argument to `images.generate`. The values mirror the +# Literal that `openai.resources.images.Images.generate` accepts on its `size` +# parameter (also visible as the `size` field of +# `openai.types.image_generate_params.ImageGenerateParams`). The OpenAI SDK +# does not export this as a named alias, so we redeclare it here. +# +# We cast `_settings.image_size` (a plain `str`) to this Literal at the API +# boundary so callers can still pass any size string (e.g. a newer value the +# SDK accepts before this list is updated). Invalid values surface as an +# OpenAI API error at runtime. Keep in sync on a best-effort basis when +# bumping the openai dep. +OpenAIImageSize = Literal[ + "auto", + "1024x1024", + "1536x1024", + "1024x1536", + "256x256", + "512x512", + "1792x1024", + "1024x1792", +] + @dataclass class OpenAIImageGenSettings(ImageGenSettings): @@ -116,15 +138,19 @@ class OpenAIImageGenService(ImageGenService): """ logger.debug(f"Generating image from prompt: {prompt}") + size = cast(OpenAIImageSize | None, assert_given(self._settings.image_size)) image = await self._client.images.generate( prompt=prompt, model=assert_given(self._settings.model), n=1, - size=assert_given(self._settings.image_size), + size=size, ) - image_url = image.data[0].url + if not image.data: + yield ErrorFrame("Image generation failed: no data returned") + return + image_url = image.data[0].url if not image_url: yield ErrorFrame("Image generation failed") return diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 5cc4a4b45..2230e9948 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -612,8 +612,10 @@ class TTSService(AIService): """Handle the completion of a turn.""" # For HTTP services they emit the frames synchronously, so close the audio context here # once all frames (including TTSTextFrame above) have been enqueued. - if self._is_yielding_frames_synchronously and self.audio_context_available( - self._turn_context_id + if ( + self._is_yielding_frames_synchronously + and self._turn_context_id is not None + and self.audio_context_available(self._turn_context_id) ): if self._push_stop_frames: await self.append_to_audio_context( @@ -1206,12 +1208,17 @@ class TTSService(AIService): else: logger.debug(f"{self} unable to append audio to context {context_id}") - async def remove_audio_context(self, context_id: str): + async def remove_audio_context(self, context_id: str | None): """Remove an existing audio context. Args: - context_id: The context to remove. + context_id: The context to remove. ``None`` is accepted as a + no-op (logged) so callers can pass through values from + ``get_active_audio_context_id()`` without an explicit guard. """ + if not context_id: + logger.debug(f"{self} unable to remove audio context: no context ID provided") + return if self.audio_context_available(context_id): # We just mark the audio context for deletion by appending # None. Once we reach None while handling audio we know we can diff --git a/src/pipecat/transports/smallwebrtc/request_handler.py b/src/pipecat/transports/smallwebrtc/request_handler.py index 63c9bea14..13bf7127c 100644 --- a/src/pipecat/transports/smallwebrtc/request_handler.py +++ b/src/pipecat/transports/smallwebrtc/request_handler.py @@ -224,6 +224,8 @@ class SmallWebRTCRequestHandler: ) answer = pipecat_connection.get_answer() + if answer is None: + raise RuntimeError("SmallWebRTC connection produced no SDP answer") if self._esp32_mode: from pipecat.runner.utils import smallwebrtc_sdp_munging From 4703df8686b8a3771e4f7a54ef04ebdc30888fc0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 30 Apr 2026 11:24:50 -0400 Subject: [PATCH 22/23] fix: clear 8 more services from pyright ignore list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fourth pass over low-error-count files. Drops 8 files (57 → 49) and full-pyright errors from 525 → 496. Default pyright stays clean. Optional access on transport/client receivers (4 files). Same fix shape as #4359 — a receiver typed `X | None` accessed without a guard. For "should never happen" cases (caller's lifecycle ensures the field is non-None when the method runs), used `assert` rather than silent early-return so an invariant violation surfaces loudly: - `transports/whatsapp/client.py` (5 errors): `_validate_whatsapp_webhook_request` was typed `bytes` / `str` but called with `bytes | None` / `str | None`. Widened the helper signature and pushed the explicit None-check inside (matching its existing empty-string check). Also handled `pipecat_connection.get_answer()` returning `None` — would have crashed at `.get("sdp")` before. - `transports/websocket/client.py` (5 errors): four are the deprecated `websockets.WebSocketClientProtocol` alias (same `# pyright: ignore[reportAttributeAccessIssue]` as the `services/websocket_service.py` fix from earlier in this PR). The fifth was `async for message in self._websocket` — traced the call chain and confirmed `_client_task` is created only after `self._websocket` is assigned and cancelled before it's cleared, so the field is never None when `_client_task_handler` runs. Used `assert`. - `services/openai/stt.py` (4 errors): same pattern. `_receive_messages` is started by `_connect()` only when `self._websocket` is set, and the reconnect loop in `WebsocketService._receive_task_handler` re-establishes it before each retry. `assert` at entry. Plus L478/L483: the `try`/`except ModuleNotFoundError` import-guard makes `websocket_connect` and `State` ` | None`; `__init__` already raises `ImportError` if either is None, so an `assert` at the `_connect_websocket` use site is honest. Plus an L538 `Language | str` cast (same shape as last batch). - `services/deepgram/flux/base.py` (2 errors): `event = data.get("event")` flowed into `_handle_turn_resumed(event: str)` as `Any | None`. Tightened with an `isinstance(event, str)` guard before the `FluxEventType(event)` lookup. The other error (`average_confidence > min_confidence` where `min_confidence: float | None`) was a latent crash on missing confidence data — restored the original `not min_confidence` (which treats both `None` and `0.0` as "no filter") and added an explicit drop-on-missing-confidence-data branch. `gemini_live` Settings/InputParams (vertex). The deprecated `InputParams` declares `modalities: GeminiModalities | None` and `media_resolution: GeminiMediaResolution | None`, but their downstream usage at `services/google/gemini_live/llm.py:952,959` calls `.value` on each — `None` would crash. Rather than touching the deprecated input model, translate `None` to the canonical defaults (`GeminiModalities.AUDIO`, `GeminiMediaResolution.UNSPECIFIED`) at the assignment site in `vertex/llm.py`. Also fixed an unrelated annotation bug: `_get_credentials` was annotated `-> str` but actually returns `service_account.Credentials` (used correctly by the caller — only the annotation was wrong). `moondream/vision.py` (3 errors). `frame.format` is `str | None` but `Image.frombytes(mode, ...)` requires `str`; raise instead of crashing on missing format. The other two errors are pyright thinking the moondream2-custom `encode_image` and `query` methods are `Tensor` (rather than callables) — those are provided by the model code via `trust_remote_code=True` and aren't visible to pyright on the base `AutoModelForCausalLM` type. Scoped `# pyright: ignore[reportCallIssue]` on the two call sites. `transports/base_output.py` (3 errors). Two are `self._mixer.mix(...)` calls in `with_mixer`, a closure invoked only when `self._mixer` is truthy at the call site — captured the mixer to a local variable inside the closure with an `assert`, then used that. Third is the PIL `frombytes(mode, ...)` shape — `frame.format is None` early- return guard at the top of `resize_frame` so the main resize logic reads cleanly. `elevenlabs/tts.py` (4 errors). The payload-building dict at L1271 was typed `dict[str, str | dict[str, float | bool]]` — an aspirational shape that matched only the first two assignments. Subsequent code assigned `list[dict[...]]` (pronunciation locators) and bools, all violating the annotation. Same pattern at L926 (the WebSocket-init `msg`). Both widened to `dict[str, Any]`, which is the honest shape for a JSON request payload and what similar code uses elsewhere. Files dropped from the ignore list (57 → 49): services/deepgram/flux/base.py, services/elevenlabs/tts.py, services/google/gemini_live/vertex/llm.py, services/moondream/vision.py, services/openai/stt.py, transports/base_output.py, transports/websocket/client.py, transports/whatsapp/client.py. --- pyrightconfig.json | 10 +-------- src/pipecat/services/deepgram/flux/base.py | 10 ++++++++- src/pipecat/services/elevenlabs/tts.py | 4 ++-- .../services/google/gemini_live/vertex/llm.py | 17 +++++++++++---- src/pipecat/services/moondream/vision.py | 9 ++++++-- src/pipecat/services/openai/stt.py | 13 ++++++++++-- src/pipecat/transports/base_output.py | 12 +++++++++-- src/pipecat/transports/websocket/client.py | 21 +++++++++++++++---- src/pipecat/transports/whatsapp/client.py | 20 ++++++++++++++---- 9 files changed, 86 insertions(+), 30 deletions(-) diff --git a/pyrightconfig.json b/pyrightconfig.json index db78dad8c..528229034 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -19,13 +19,10 @@ "src/pipecat/services/aws/nova_sonic/llm.py", "src/pipecat/services/aws/sagemaker/bidi_client.py", "src/pipecat/services/azure/tts.py", - "src/pipecat/services/deepgram/flux/base.py", "src/pipecat/services/deepgram/flux/sagemaker/stt.py", "src/pipecat/services/deepgram/sagemaker/stt.py", "src/pipecat/services/deepgram/sagemaker/tts.py", - "src/pipecat/services/elevenlabs/tts.py", "src/pipecat/services/google/gemini_live/llm.py", - "src/pipecat/services/google/gemini_live/vertex/llm.py", "src/pipecat/services/google/llm.py", "src/pipecat/services/google/stt.py", "src/pipecat/services/google/tts.py", @@ -35,12 +32,10 @@ "src/pipecat/services/llm_service.py", "src/pipecat/services/mem0/memory.py", "src/pipecat/services/mistral/tts.py", - "src/pipecat/services/moondream/vision.py", "src/pipecat/services/nvidia/stt.py", "src/pipecat/services/nvidia/tts.py", "src/pipecat/services/openai/base_llm.py", "src/pipecat/services/openai/realtime/llm.py", - "src/pipecat/services/openai/stt.py", "src/pipecat/services/rime/tts.py", "src/pipecat/services/sambanova/llm.py", "src/pipecat/services/sarvam/stt.py", @@ -52,16 +47,13 @@ "src/pipecat/services/ultravox/llm.py", "src/pipecat/services/whisper/stt.py", "src/pipecat/services/xai/realtime/llm.py", - "src/pipecat/transports/base_output.py", "src/pipecat/transports/daily/transport.py", "src/pipecat/transports/lemonslice/transport.py", "src/pipecat/transports/livekit/transport.py", "src/pipecat/transports/smallwebrtc/connection.py", "src/pipecat/transports/smallwebrtc/transport.py", "src/pipecat/transports/tavus/transport.py", - "src/pipecat/transports/websocket/client.py", - "src/pipecat/transports/websocket/server.py", - "src/pipecat/transports/whatsapp/client.py" + "src/pipecat/transports/websocket/server.py" ], "reportMissingImports": false } diff --git a/src/pipecat/services/deepgram/flux/base.py b/src/pipecat/services/deepgram/flux/base.py index 173da5dc7..d2a8a24cd 100644 --- a/src/pipecat/services/deepgram/flux/base.py +++ b/src/pipecat/services/deepgram/flux/base.py @@ -536,6 +536,10 @@ class DeepgramFluxSTTBase(STTService): event = data.get("event") transcript = data.get("transcript", "") + if not isinstance(event, str): + logger.debug(f"Unhandled TurnInfo event (not a string): {event}") + return + try: flux_event_type = FluxEventType(event) except ValueError: @@ -648,7 +652,11 @@ class DeepgramFluxSTTBase(STTService): detected_language = self._primary_detected_language(data) min_confidence = assert_given(self._settings.min_confidence) - if not min_confidence or average_confidence > min_confidence: + # No threshold (None or 0.0) → accept. Otherwise require confidence + # data and compare; drop if data is missing. + if not min_confidence or ( + average_confidence is not None and average_confidence > min_confidence + ): # EndOfTurn means Flux has determined the turn is complete, # so this TranscriptionFrame is always finalized await self.push_frame( diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 239e6a64e..fe8e7ab84 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -923,7 +923,7 @@ class ElevenLabsTTSService(WebsocketTTSService): self._partial_word_start_time = 0.0 # Initialize context with voice settings and pronunciation dictionaries - msg = {"text": " ", "context_id": context_id} + msg: dict[str, Any] = {"text": " ", "context_id": context_id} if self._voice_settings: msg["voice_settings"] = self._voice_settings if self._pronunciation_dictionary_locators: @@ -1268,7 +1268,7 @@ class ElevenLabsHttpTTSService(TTSService): url = f"{self._base_url}/v1/text-to-speech/{self._settings.voice}/stream/with-timestamps" model_id = assert_given(self._settings.model) - payload: dict[str, str | dict[str, float | bool]] = { + payload: dict[str, Any] = { "text": text, "model_id": model_id, } diff --git a/src/pipecat/services/google/gemini_live/vertex/llm.py b/src/pipecat/services/google/gemini_live/vertex/llm.py index 44ded852f..b02c18a60 100644 --- a/src/pipecat/services/google/gemini_live/vertex/llm.py +++ b/src/pipecat/services/google/gemini_live/vertex/llm.py @@ -174,11 +174,17 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService): default_settings.temperature = params.temperature default_settings.top_k = params.top_k default_settings.top_p = params.top_p - default_settings.modalities = params.modalities + # `params.modalities` and `params.media_resolution` are typed + # ` | None` on the deprecated InputParams, but None isn't + # a valid setting value (downstream uses call `.value` on + # them). Fall back to the canonical defaults. + default_settings.modalities = params.modalities or GeminiModalities.AUDIO default_settings.language = ( language_to_gemini_language(params.language) if params.language else "en-US" ) - default_settings.media_resolution = params.media_resolution + default_settings.media_resolution = ( + params.media_resolution or GeminiMediaResolution.UNSPECIFIED + ) default_settings.vad = params.vad default_settings.context_window_compression = ( params.context_window_compression.model_dump() @@ -233,7 +239,9 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService): ) @staticmethod - def _get_credentials(credentials: str | None, credentials_path: str | None) -> str: + def _get_credentials( + credentials: str | None, credentials_path: str | None + ) -> service_account.Credentials: """Retrieve Credentials using Google service account credentials JSON. Supports multiple authentication methods: @@ -246,7 +254,8 @@ class GeminiLiveVertexLLMService(GeminiLiveLLMService): credentials_path: Path to the service account JSON file. Returns: - OAuth token for API authentication. + A service-account ``Credentials`` object suitable for the Vertex + AI client (with its access token refreshed). Raises: ValueError: If no valid credentials are provided or found. diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index 01482df4b..49cb0ba34 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -153,9 +153,14 @@ class MoondreamService(VisionService): logger.debug(f"Analyzing image (bytes length: {len(frame.image)})") def get_image_description(image_bytes: bytes, text: str | None) -> str: + if frame.format is None: + raise ValueError("Cannot decode image bytes without a format") image = Image.frombytes(frame.format, frame.size, image_bytes) - image_embeds = self._model.encode_image(image) - description = self._model.query(image_embeds, text)["answer"] + # `encode_image` and `query` are custom methods provided by the + # moondream2 model code (via `trust_remote_code=True`) that pyright + # can't see on `AutoModelForCausalLM`'s base type. + image_embeds = self._model.encode_image(image) # pyright: ignore[reportCallIssue] + description = self._model.query(image_embeds, text)["answer"] # pyright: ignore[reportCallIssue] return description description = await asyncio.to_thread(get_image_description, frame.image, frame.text) diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index b7dddf441..6f635639a 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -18,7 +18,7 @@ import base64 import json from collections.abc import AsyncGenerator from dataclasses import dataclass, field -from typing import Any, Literal +from typing import Any, Literal, cast from loguru import logger @@ -475,6 +475,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): async def _connect_websocket(self): """Establish the WebSocket connection to the transcription endpoint.""" try: + # `__init__` raises if websockets isn't installed, so these symbols + # are non-None by the time any method runs. + assert websocket_connect is not None and State is not None if self._websocket and self._websocket.state is State.OPEN: return @@ -534,7 +537,9 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): """Send ``session.update`` to configure the transcription session.""" transcription: dict = {"model": self._settings.model} - language = assert_given(self._settings.language) + # Technically `_settings.language` could be a raw string, but Language + # is a StrEnum so downstream handles either. + language = cast("Language | None", assert_given(self._settings.language)) language_code = self._language_to_code(language) if language else None if language_code: transcription["language"] = language_code @@ -611,6 +616,10 @@ class OpenAIRealtimeSTTService(WebsocketSTTService): Called by ``WebsocketService._receive_task_handler`` which wraps this method with automatic reconnection on connection errors. """ + # `_connect` only starts the receive task after `_websocket` is set, + # and reconnects re-establish it before the next iteration, so this + # invariant should always hold when this method runs. + assert self._websocket is not None async for message in self._websocket: try: evt = json.loads(message) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index d485129f4..3f1427627 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -771,13 +771,16 @@ class BaseOutputTransport(FrameProcessor): await self._bot_stopped_speaking() async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]: + # Caller below only invokes this when `self._mixer` is set. + mixer = self._mixer + assert mixer is not None last_frame_time = 0 silence = b"\x00" * self._audio_chunk_size while True: try: frame = self._audio_queue.get_nowait() if isinstance(frame, OutputAudioRawFrame): - frame.audio = await self._mixer.mix(frame.audio) + frame.audio = await mixer.mix(frame.audio) last_frame_time = time.time() yield frame self._audio_queue.task_done() @@ -788,7 +791,7 @@ class BaseOutputTransport(FrameProcessor): await self._bot_stopped_speaking() # Generate an audio frame with only the mixer's part. frame = OutputAudioRawFrame( - audio=await self._mixer.mix(silence), + audio=await mixer.mix(silence), sample_rate=self._sample_rate, num_channels=self._params.audio_out_channels, ) @@ -927,6 +930,11 @@ class BaseOutputTransport(FrameProcessor): """ def resize_frame(frame: OutputImageRawFrame) -> OutputImageRawFrame: + # Without a format we can't decode the bytes, so leave the + # frame as-is and let the transport pass it through unchanged. + if frame.format is None: + return frame + desired_size = (self._params.video_out_width, self._params.video_out_height) # TODO: we should refactor in the future to support dynamic resolutions diff --git a/src/pipecat/transports/websocket/client.py b/src/pipecat/transports/websocket/client.py index 5665dfd23..9f2a43dbc 100644 --- a/src/pipecat/transports/websocket/client.py +++ b/src/pipecat/transports/websocket/client.py @@ -64,9 +64,18 @@ class WebsocketClientCallbacks(BaseModel): on_message: Called when a message is received from the WebSocket. """ - on_connected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]] - on_disconnected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]] - on_message: Callable[[websockets.WebSocketClientProtocol, websockets.Data], Awaitable[None]] + on_connected: Callable[ + [websockets.WebSocketClientProtocol], # pyright: ignore[reportAttributeAccessIssue] + Awaitable[None], + ] + on_disconnected: Callable[ + [websockets.WebSocketClientProtocol], # pyright: ignore[reportAttributeAccessIssue] + Awaitable[None], + ] + on_message: Callable[ + [websockets.WebSocketClientProtocol, websockets.Data], # pyright: ignore[reportAttributeAccessIssue] + Awaitable[None], + ] class WebsocketClientSession: @@ -98,7 +107,7 @@ class WebsocketClientSession: self._leave_counter = 0 self._task_manager: BaseTaskManager | None = None - self._websocket: websockets.WebSocketClientProtocol | None = None + self._websocket: websockets.WebSocketClientProtocol | None = None # pyright: ignore[reportAttributeAccessIssue] @property def task_manager(self) -> BaseTaskManager: @@ -192,6 +201,10 @@ class WebsocketClientSession: async def _client_task_handler(self): """Handle incoming messages from the WebSocket connection.""" + # `connect()` only starts this task after `_websocket` is assigned, and + # `disconnect()` cancels the task before clearing `_websocket`, so this + # invariant should always hold when this method runs. + assert self._websocket is not None try: # Handle incoming messages async for message in self._websocket: diff --git a/src/pipecat/transports/whatsapp/client.py b/src/pipecat/transports/whatsapp/client.py index 8f479520f..171a79247 100644 --- a/src/pipecat/transports/whatsapp/client.py +++ b/src/pipecat/transports/whatsapp/client.py @@ -154,8 +154,17 @@ class WhatsAppClient: return int(challenge) - async def _validate_whatsapp_webhook_request(self, raw_body: bytes, sha256_signature: str): + async def _validate_whatsapp_webhook_request( + self, raw_body: bytes | None, sha256_signature: str | None + ): """Common handler for both /start and /connect endpoints.""" + # Callers gate on `self._whatsapp_secret`, so the assert holds. + assert self._whatsapp_secret is not None + if raw_body is None: + raise Exception("Missing raw request body") + if not sha256_signature: + raise Exception("Missing X-Hub-Signature-256 header") + # Compute HMAC SHA256 using your App Secret expected_signature = hmac.new( key=self._whatsapp_secret.encode("utf-8"), @@ -164,8 +173,6 @@ class WhatsAppClient: ).hexdigest() # Extract signature from header (strip 'sha256=' prefix) - if not sha256_signature: - raise Exception("Missing X-Hub-Signature-256 header") received_signature = sha256_signature.split("sha256=")[-1] # Compare signatures securely @@ -306,7 +313,12 @@ class WhatsAppClient: # Create and initialize WebRTC connection pipecat_connection = SmallWebRTCConnection(self._ice_servers) await pipecat_connection.initialize(sdp=call.session.sdp, type=call.session.sdp_type) - sdp_answer = pipecat_connection.get_answer().get("sdp") + answer = pipecat_connection.get_answer() + if answer is None: + raise RuntimeError("SmallWebRTC connection produced no SDP answer") + sdp_answer = answer.get("sdp") + if sdp_answer is None: + raise RuntimeError("SmallWebRTC SDP answer missing 'sdp' field") sdp_answer = self._filter_sdp_for_whatsapp(sdp_answer) logger.debug(f"SDP answer generated for call {call.id}") From 2730e47e611c00ac2e2da9c10e1a3fb5a492a222 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 30 Apr 2026 11:52:51 -0400 Subject: [PATCH 23/23] ci: install all extras for the pyright type-check job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pyright job in `format.yaml` previously installed only `--extra daily --extra tracing`. That was sufficient when most optional-dep- using files were in the pyright ignore list, but as this PR has cleared dozens of files, those files now reference symbols from optional-dep modules (`aiortc.RTCIceServer` via `IceServer`, `google.genai.types.HttpOptions`, etc.). `reportMissingImports: false` tolerates the failed imports themselves, but the imported names become `Unknown` and using them as type expressions trips `reportInvalidTypeForm` / `reportAttributeAccessIssue` — errors that aren't gated by that flag. Switch to `--all-extras --no-extra gstreamer --no-extra local` (matching the dev setup in README.md), so pyright sees the same dependency set the code is intended to be type-checked against and the install-set scales naturally as more files leave the ignore list. Also reconcile CLAUDE.md's setup command, which only excluded `gstreamer`. README.md is canonical and additionally excludes `local` (pyaudio requires `portaudio` native libs that aren't installed by default on a clean Ubuntu CI runner). --- .github/workflows/format.yaml | 4 +++- CLAUDE.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index 7ac009bdc..5ce0b746f 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -32,7 +32,9 @@ jobs: run: uv python install 3.12 - name: Install development dependencies - run: uv sync --group dev --extra daily --extra tracing + # `--all-extras` (matching the dev setup in README.md) so pyright can + # resolve types from various optional dependencies. + run: uv sync --group dev --all-extras --no-extra gstreamer --no-extra local - name: Ruff formatter id: ruff-format diff --git a/CLAUDE.md b/CLAUDE.md index 5dc0e9295..123307489 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ Pipecat is an open-source Python framework for building real-time voice and mult ```bash # Setup development environment -uv sync --group dev --all-extras --no-extra gstreamer +uv sync --group dev --all-extras --no-extra gstreamer --no-extra local # Install pre-commit hooks uv run pre-commit install