776a3526f9
This wasn't really an issue before, when folks were *knowingly* migrating from `OpenAILLMContext` to `LLMContext`. But in the latest AWS Nova Sonic change, we're swapping it out from under folks, so this kind of compatibility is more important. For context, the reason we *didn't* offer the `messages` property earlier was to aid in the development of `LLMContext`—we wanted to draw attention to all the places where messages were being read from context, so we could find the places where we might need to pass an argument to the read.
316 lines
11 KiB
Python
316 lines
11 KiB
Python
#
|
|
# Copyright (c) 2025, Daily
|
|
#
|
|
# SPDX-License-Identifier: BSD 2-Clause License
|
|
#
|
|
|
|
"""Universal LLM context management for LLM services in Pipecat.
|
|
|
|
Context contents are represented in a universal format (based on OpenAI)
|
|
that supports a union of known Pipecat LLM service functionality.
|
|
|
|
Whenever an LLM service needs to access context, it does a just-in-time
|
|
translation from this universal context into whatever format it needs, using a
|
|
service-specific adapter.
|
|
"""
|
|
|
|
import base64
|
|
import copy
|
|
import io
|
|
from dataclasses import dataclass
|
|
from typing import TYPE_CHECKING, Any, List, Optional, TypeAlias, Union
|
|
|
|
from loguru import logger
|
|
from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
|
|
from openai._types import NotGiven as OpenAINotGiven
|
|
from openai.types.chat import (
|
|
ChatCompletionMessageParam,
|
|
ChatCompletionToolChoiceOptionParam,
|
|
)
|
|
from PIL import Image
|
|
|
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|
from pipecat.frames.frames import AudioRawFrame
|
|
|
|
if TYPE_CHECKING:
|
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
|
|
|
# "Re-export" types from OpenAI that we're using as universal context types.
|
|
# NOTE: if universal message types need to someday diverge from OpenAI's, we
|
|
# should consider managing our own definitions. But we should do so carefully,
|
|
# as the OpenAI messages are somewhat of a standard and we want to continue
|
|
# supporting them.
|
|
LLMStandardMessage = ChatCompletionMessageParam
|
|
LLMContextToolChoice = ChatCompletionToolChoiceOptionParam
|
|
NOT_GIVEN = OPEN_AI_NOT_GIVEN
|
|
NotGiven = OpenAINotGiven
|
|
|
|
|
|
@dataclass
|
|
class LLMSpecificMessage:
|
|
"""A container for a context message that is specific to a particular LLM service.
|
|
|
|
Enables the use of service-specific message types while maintaining
|
|
compatibility with the universal LLM context format.
|
|
"""
|
|
|
|
llm: str
|
|
message: Any
|
|
|
|
|
|
LLMContextMessage: TypeAlias = Union[LLMStandardMessage, LLMSpecificMessage]
|
|
|
|
|
|
class LLMContext:
|
|
"""Manages conversation context for LLM interactions.
|
|
|
|
Handles message history, tool definitions, tool choices, and multimedia
|
|
content for LLM conversations. Provides methods for message manipulation,
|
|
and content formatting.
|
|
"""
|
|
|
|
@staticmethod
|
|
def from_openai_context(openai_context: "OpenAILLMContext") -> "LLMContext":
|
|
"""Create a universal LLM context from an OpenAI-specific context.
|
|
|
|
NOTE: this should only be used internally, for facilitating migration
|
|
from OpenAILLMContext to LLMContext. New user code should use
|
|
LLMContext directly.
|
|
|
|
Args:
|
|
openai_context: The OpenAI LLM context to convert.
|
|
|
|
Returns:
|
|
New LLMContext instance with converted messages and settings.
|
|
"""
|
|
return LLMContext(
|
|
messages=openai_context.get_messages(),
|
|
tools=openai_context.tools,
|
|
tool_choice=openai_context.tool_choice,
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
messages: Optional[List[LLMContextMessage]] = None,
|
|
tools: ToolsSchema | NotGiven = NOT_GIVEN,
|
|
tool_choice: LLMContextToolChoice | NotGiven = NOT_GIVEN,
|
|
):
|
|
"""Initialize the LLM context.
|
|
|
|
Args:
|
|
messages: Initial list of conversation messages.
|
|
tools: Available tools for the LLM to use.
|
|
tool_choice: Tool selection strategy for the LLM.
|
|
"""
|
|
self._messages: List[LLMContextMessage] = messages if messages else []
|
|
self._tools: ToolsSchema | NotGiven = LLMContext._normalize_and_validate_tools(tools)
|
|
self._tool_choice: LLMContextToolChoice | NotGiven = tool_choice
|
|
|
|
@property
|
|
def messages(self) -> List[LLMContextMessage]:
|
|
"""Get the current messages list.
|
|
|
|
NOTE: This is equivalent to calling `get_messages()` with no filter. If
|
|
you want to filter out LLM-specific messages that don't pertain to your
|
|
LLM, use `get_messages()` directly.
|
|
|
|
Returns:
|
|
List of conversation messages.
|
|
"""
|
|
return self.get_messages()
|
|
|
|
def get_messages(self, llm_specific_filter: Optional[str] = None) -> List[LLMContextMessage]:
|
|
"""Get the current messages list.
|
|
|
|
Args:
|
|
llm_specific_filter: Optional filter to return LLM-specific
|
|
messages for the given LLM, in addition to the standard
|
|
messages. If messages end up being filtered, an error will be
|
|
logged; this is intended to catch accidental use of
|
|
incompatible LLM-specific messages.
|
|
|
|
Returns:
|
|
List of conversation messages.
|
|
"""
|
|
if llm_specific_filter is None:
|
|
return self._messages
|
|
filtered_messages = [
|
|
msg
|
|
for msg in self._messages
|
|
if not isinstance(msg, LLMSpecificMessage) or msg.llm == llm_specific_filter
|
|
]
|
|
if len(filtered_messages) < len(self._messages):
|
|
logger.error(
|
|
f"Attempted to use incompatible LLMSpecificMessages with LLM '{llm_specific_filter}'."
|
|
)
|
|
return filtered_messages
|
|
|
|
@property
|
|
def tools(self) -> ToolsSchema | NotGiven:
|
|
"""Get the tools list.
|
|
|
|
Returns:
|
|
Tools list.
|
|
"""
|
|
return self._tools
|
|
|
|
@property
|
|
def tool_choice(self) -> LLMContextToolChoice | NotGiven:
|
|
"""Get the current tool choice setting.
|
|
|
|
Returns:
|
|
The tool choice configuration.
|
|
"""
|
|
return self._tool_choice
|
|
|
|
def add_message(self, message: LLMContextMessage):
|
|
"""Add a single message to the context.
|
|
|
|
Args:
|
|
message: The message to add to the conversation history.
|
|
"""
|
|
self._messages.append(message)
|
|
|
|
def add_messages(self, messages: List[LLMContextMessage]):
|
|
"""Add multiple messages to the context.
|
|
|
|
Args:
|
|
messages: List of messages to add to the conversation history.
|
|
"""
|
|
self._messages.extend(messages)
|
|
|
|
def set_messages(self, messages: List[LLMContextMessage]):
|
|
"""Replace all messages in the context.
|
|
|
|
Args:
|
|
messages: New list of messages to replace the current history.
|
|
"""
|
|
self._messages[:] = messages
|
|
|
|
def set_tools(self, tools: ToolsSchema | NotGiven = NOT_GIVEN):
|
|
"""Set the available tools for the LLM.
|
|
|
|
Args:
|
|
tools: A ToolsSchema or NOT_GIVEN to disable tools.
|
|
"""
|
|
self._tools = LLMContext._normalize_and_validate_tools(tools)
|
|
|
|
def set_tool_choice(self, tool_choice: LLMContextToolChoice | NotGiven):
|
|
"""Set the tool choice configuration.
|
|
|
|
Args:
|
|
tool_choice: Tool selection strategy for the LLM.
|
|
"""
|
|
self._tool_choice = tool_choice
|
|
|
|
def add_image_frame_message(
|
|
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
|
|
):
|
|
"""Add a message containing an image frame.
|
|
|
|
Args:
|
|
format: Image format (e.g., 'RGB', 'RGBA').
|
|
size: Image dimensions as (width, height) tuple.
|
|
image: Raw image bytes.
|
|
text: Optional text to include with the image.
|
|
"""
|
|
buffer = io.BytesIO()
|
|
Image.frombytes(format, size, image).save(buffer, format="JPEG")
|
|
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
|
|
|
|
content = []
|
|
if text:
|
|
content.append({"type": "text", "text": text})
|
|
content.append(
|
|
{"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}},
|
|
)
|
|
self.add_message({"role": "user", "content": content})
|
|
|
|
def add_audio_frames_message(
|
|
self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows"
|
|
):
|
|
"""Add a message containing audio frames.
|
|
|
|
Args:
|
|
audio_frames: List of audio frame objects to include.
|
|
text: Optional text to include with the audio.
|
|
"""
|
|
if not audio_frames:
|
|
return
|
|
|
|
sample_rate = audio_frames[0].sample_rate
|
|
num_channels = audio_frames[0].num_channels
|
|
|
|
content = []
|
|
content.append({"type": "text", "text": text})
|
|
data = b"".join(frame.audio for frame in audio_frames)
|
|
data = bytes(
|
|
self._create_wav_header(
|
|
sample_rate,
|
|
num_channels,
|
|
16,
|
|
len(data),
|
|
)
|
|
+ data
|
|
)
|
|
encoded_audio = base64.b64encode(data).decode("utf-8")
|
|
content.append(
|
|
{
|
|
"type": "input_audio",
|
|
"input_audio": {"data": encoded_audio, "format": "wav"},
|
|
}
|
|
)
|
|
self.add_message({"role": "user", "content": content})
|
|
|
|
def _create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size):
|
|
"""Create a WAV file header for audio data.
|
|
|
|
Args:
|
|
sample_rate: Audio sample rate in Hz.
|
|
num_channels: Number of audio channels.
|
|
bits_per_sample: Bits per audio sample.
|
|
data_size: Size of audio data in bytes.
|
|
|
|
Returns:
|
|
WAV header as a bytearray.
|
|
"""
|
|
# RIFF chunk descriptor
|
|
header = bytearray()
|
|
header.extend(b"RIFF") # ChunkID
|
|
header.extend((data_size + 36).to_bytes(4, "little")) # ChunkSize: total size - 8
|
|
header.extend(b"WAVE") # Format
|
|
# "fmt " sub-chunk
|
|
header.extend(b"fmt ") # Subchunk1ID
|
|
header.extend((16).to_bytes(4, "little")) # Subchunk1Size (16 for PCM)
|
|
header.extend((1).to_bytes(2, "little")) # AudioFormat (1 for PCM)
|
|
header.extend(num_channels.to_bytes(2, "little")) # NumChannels
|
|
header.extend(sample_rate.to_bytes(4, "little")) # SampleRate
|
|
# Calculate byte rate and block align
|
|
byte_rate = sample_rate * num_channels * (bits_per_sample // 8)
|
|
block_align = num_channels * (bits_per_sample // 8)
|
|
header.extend(byte_rate.to_bytes(4, "little")) # ByteRate
|
|
header.extend(block_align.to_bytes(2, "little")) # BlockAlign
|
|
header.extend(bits_per_sample.to_bytes(2, "little")) # BitsPerSample
|
|
# "data" sub-chunk
|
|
header.extend(b"data") # Subchunk2ID
|
|
header.extend(data_size.to_bytes(4, "little")) # Subchunk2Size
|
|
return header
|
|
|
|
@staticmethod
|
|
def _normalize_and_validate_tools(tools: ToolsSchema | NotGiven) -> ToolsSchema | NotGiven:
|
|
"""Normalize and validate the given tools.
|
|
|
|
Raises:
|
|
TypeError: If tools are not a ToolsSchema or NotGiven.
|
|
"""
|
|
if isinstance(tools, ToolsSchema):
|
|
if not tools.standard_tools and not tools.custom_tools:
|
|
return NOT_GIVEN
|
|
return tools
|
|
elif tools is NOT_GIVEN:
|
|
return NOT_GIVEN
|
|
else:
|
|
raise TypeError(
|
|
f"In LLMContext, tools must be a ToolsSchema object or NOT_GIVEN. Got type: {type(tools)}",
|
|
)
|