wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Base OpenAI LLM service

Browse Source
This commit is contained in:
Moishe Lettvin
2024-03-08 11:09:16 -05:00
parent c75a3fb0d0
commit d9378e23ba
15 changed files with 558 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/dailyai/services/ai_services.py
View File

@@ -67,49 +67,9 @@ class AIService(FrameProcessor):
class LLMService(AIService):
def __init__(self, messages=None, tools=None):
""" This class is a no-op but serves as a base class for LLM services. """
def __init__(self):
super().__init__()
self._tools = tools
self._messages = messages
@abstractmethod
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
yield ""
@abstractmethod
async def run_llm(self, messages) -> str:
pass
async def process_frame(self, frame: Frame, tool_choice: str = None) -> AsyncGenerator[Frame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
function_name = ""
arguments = ""
if isinstance(frame, LLMMessagesQueueFrame):
yield LLMResponseStartFrame()
async for text_chunk in self.run_llm_async(frame.messages, tool_choice):
# We're streaming the LLM response and returning individual TextFrames for each chunk because
# we want to enable quick TTS. But if the LLM response is a function call, we don't need to yield
# each chunk because the function call is only useful as a single frame. Instead, we'll emit a
# LLMFunctionStartFrame to let downstream services know a function call is coming, then we'll
# collect the function arguments and return the entire call in a single LLMFunctionCallFrame.
if isinstance(text_chunk, str):
yield TextFrame(text_chunk)
elif text_chunk.function:
if text_chunk.function.name:
function_name += text_chunk.function.name
yield LLMFunctionStartFrame(function_name=text_chunk.function.name)
if text_chunk.function.arguments:
# Keep iterating through the response to collect all the argument fragments and
# yield a complete LLMFunctionCallFrame after run_llm_async completes
arguments += text_chunk.function.arguments
if (function_name and arguments):
yield LLMFunctionCallFrame(function_name=function_name, arguments=arguments)
function_name = ""
arguments = ""
yield LLMResponseEndFrame()
else:
yield frame
class TTSService(AIService):

95
src/dailyai/services/azure_ai_services.py
View File

@@ -14,7 +14,14 @@ from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
from PIL import Image
# See .env.example for Azure configuration needed
from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
from azure.cognitiveservices.speech import (
SpeechSynthesizer,
SpeechConfig,
ResultReason,
CancellationReason,
)
from dailyai.services.openai_api_llm_service import BaseOpenAILLMService
class AzureTTSService(TTSService):
@@ -23,18 +30,21 @@ class AzureTTSService(TTSService):
self.speech_config = SpeechConfig(subscription=api_key, region=region)
self.speech_synthesizer = SpeechSynthesizer(
speech_config=self.speech_config, audio_config=None)
speech_config=self.speech_config, audio_config=None
)
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
self.logger.info("Running azure tts")
ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
"xmlns:mstts='http://www.w3.org/2001/mstts'>" \
"<voice name='en-US-SaraNeural'>" \
"<mstts:silence type='Sentenceboundary' value='20ms' />" \
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \
"<prosody rate='1.05'>" \
f"{sentence}" \
ssml = (
"<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' "
"xmlns:mstts='http://www.w3.org/2001/mstts'>"
"<voice name='en-US-SaraNeural'>"
"<mstts:silence type='Sentenceboundary' value='20ms' />"
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>"
"<prosody rate='1.05'>"
f"{sentence}"
"</prosody></mstts:express-as></voice></speak> "
)
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
self.logger.info("Got azure tts result")
if result.reason == ResultReason.SynthesizingAudioCompleted:
@@ -43,62 +53,39 @@ class AzureTTSService(TTSService):
yield result.audio_data[44:]
elif result.reason == ResultReason.Canceled:
cancellation_details = result.cancellation_details
self.logger.info("Speech synthesis canceled: {}".format(cancellation_details.reason))
self.logger.info(
"Speech synthesis canceled: {}".format(cancellation_details.reason)
)
if cancellation_details.reason == CancellationReason.Error:
self.logger.info("Error details: {}".format(cancellation_details.error_details))
self.logger.info(
"Error details: {}".format(cancellation_details.error_details)
)
class AzureLLMService(LLMService):
def __init__(self, *, api_key, endpoint, api_version="2023-12-01-preview", model, tools=None, messages=None):
super().__init__(tools=tools, messages=messages)
self._model: str = model
class AzureLLMService(BaseOpenAILLMService):
def __init__(self, *, api_key, endpoint, api_version="2023-12-01-preview", model):
super().__init__(model)
# This overrides the client created by the super class init
self._client = AsyncAzureOpenAI(
api_key=api_key,
azure_endpoint=endpoint,
api_version=api_version,
)
async def run_llm_async(self, messages, tool_choice=None) -> AsyncGenerator[str, None]:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}")
if self._tools:
tools = self._tools
else:
tools = None
start_time = time.time()
chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages, tools=tools, tool_choice=tool_choice)
self.logger.info(f"=== Azure OpenAI LLM TTFB: {time.time() - start_time}")
async for chunk in chunks:
if len(chunk.choices) == 0:
continue
if chunk.choices[0].delta.tool_calls:
yield chunk.choices[0].delta.tool_calls[0]
elif chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
async def run_llm(self, messages) -> str | None:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}")
response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
return None
class AzureImageGenServiceREST(ImageGenService):
def __init__(
self,
*,
api_version="2023-06-01-preview",
image_size: str,
aiohttp_session: aiohttp.ClientSession,
api_key,
endpoint,
model):
self,
*,
api_version="2023-06-01-preview",
image_size: str,
aiohttp_session: aiohttp.ClientSession,
api_key,
endpoint,
model,
):
super().__init__(image_size=image_size)
self._api_key = api_key
@@ -121,7 +108,7 @@ class AzureImageGenServiceREST(ImageGenService):
) as submission:
# We never get past this line, because this header isn't
# defined on a 429 response, but something is eating our exceptions!
operation_location = submission.headers['operation-location']
operation_location = submission.headers["operation-location"]
status = ""
attempts_left = 120
json_response = None
@@ -137,7 +124,9 @@ class AzureImageGenServiceREST(ImageGenService):
json_response = await response.json()
status = json_response["status"]
image_url = json_response["result"]["data"][0]["url"] if json_response else None
image_url = (
json_response["result"]["data"][0]["url"] if json_response else None
)
if not image_url:
raise Exception("Image generation failed")
# Load the image from the url

2
src/dailyai/services/base_transport_service.py
View File

@@ -380,7 +380,6 @@ class BaseTransportService():
b = bytearray()
smallest_write_size = 3200
largest_write_size = 8000
all_audio_frames = bytearray()
while True:
try:
frames_or_frame: Frame | list[Frame] = (
@@ -414,7 +413,6 @@ class BaseTransportService():
if frame:
if isinstance(frame, AudioFrame):
chunk = frame.data
all_audio_frames.extend(chunk)
b.extend(chunk)
truncated_length: int = len(b) - (

43
src/dailyai/services/ollama_ai_services.py
View File

@@ -1,42 +1,7 @@
from openai import AsyncOpenAI
import json
from collections.abc import AsyncGenerator
from dailyai.services.ai_services import LLMService
from dailyai.services.openai_api_llm_service import BaseOpenAILLMService
class OLLamaLLMService(LLMService):
def __init__(self, model="llama2", base_url='http://localhost:11434/v1'):
super().__init__()
self._model = model
self._client = AsyncOpenAI(api_key="ollama", base_url=base_url)
class OLLamaLLMService(BaseOpenAILLMService):
async def get_response(self, messages, stream):
return await self._client.chat.completions.create(
stream=stream,
messages=messages,
model=self._model
)
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages)
async for chunk in chunks:
if len(chunk.choices) == 0:
continue
if chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
async def run_llm(self, messages) -> str | None:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
return None
def __init__(self, model="llama2", base_url="http://localhost:11434/v1"):
super().__init__(model=model, base_url=base_url, api_key="ollama")

44
src/dailyai/services/open_ai_services.py
View File

@@ -8,49 +8,13 @@ import json
from collections.abc import AsyncGenerator
from dailyai.services.ai_services import LLMService, ImageGenService
from dailyai.services.openai_api_llm_service import BaseOpenAILLMService
class OpenAILLMService(LLMService):
def __init__(self, *, api_key, model="gpt-4", tools=None, messages=None):
super().__init__(tools=tools, messages=messages)
self._model = model
self._client = AsyncOpenAI(api_key=api_key)
class OpenAILLMService(BaseOpenAILLMService):
async def get_response(self, messages, stream):
return await self._client.chat.completions.create(
stream=stream,
messages=messages,
model=self._model,
tools=self._tools
)
async def run_llm_async(self, messages, tool_choice=None) -> AsyncGenerator[str, None]:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
if self._tools:
tools = self._tools
else:
tools = None
start_time = time.time()
chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages, tools=tools, tool_choice=tool_choice)
self.logger.info(f"=== OpenAI LLM TTFB: {time.time() - start_time}")
async for chunk in chunks:
if len(chunk.choices) == 0:
continue
if chunk.choices[0].delta.tool_calls:
yield chunk.choices[0].delta.tool_calls[0]
elif chunk.choices[0].delta.content:
yield chunk.choices[0].delta.content
async def run_llm(self, messages) -> str | None:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
return None
def __init__(self, model="gpt-4", * args, **kwargs):
super().__init__(model, *args, **kwargs)
class OpenAIImageGenService(ImageGenService):

120
src/dailyai/services/openai_api_llm_service.py Normal file
View File

@@ -0,0 +1,120 @@
import json
import time
from typing import AsyncGenerator, List
from openai import AsyncOpenAI, AsyncStream
from dailyai.pipeline.frames import (
Frame,
LLMFunctionCallFrame,
LLMFunctionStartFrame,
LLMMessagesQueueFrame,
LLMResponseEndFrame,
LLMResponseStartFrame,
OpenAILLMContextFrame,
TextFrame,
)
from dailyai.services.ai_services import LLMService
from dailyai.services.openai_llm_context import OpenAILLMContext
from openai.types.chat import (
ChatCompletion,
ChatCompletionChunk,
ChatCompletionMessageParam,
)
class BaseOpenAILLMService(LLMService):
"""This is the base for all services that use the AsyncOpenAI client.
This service consumes OpenAILLMContextFrame frames, which contain a reference
to an OpenAILLMContext frame. The OpenAILLMContext object defines the context
sent to the LLM for a completion. This includes user, assistant and system messages
as well as tool choices and the tool, which is used if requesting function
calls from the LLM.
"""
def __init__(self, model: str, api_key=None, base_url=None):
super().__init__()
self._model: str = model
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
async def _stream_chat_completions(
self, context: OpenAILLMContext
) -> AsyncStream[ChatCompletionChunk]:
messages: List[ChatCompletionMessageParam] = context.get_messages()
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
start_time = time.time()
chunks: AsyncStream[ChatCompletionChunk] = (
await self._client.chat.completions.create(
model=self._model,
stream=True,
messages=messages,
tools=context.tools,
tool_choice=context.tool_choice,
)
)
self.logger.info(f"=== OpenAI LLM TTFB: {time.time() - start_time}")
return chunks
async def _chat_completions(self, messages) -> str | None:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response: ChatCompletion = await self._client.chat.completions.create(
model=self._model, stream=False, messages=messages
)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
return None
async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]:
if isinstance(frame, OpenAILLMContextFrame):
context: OpenAILLMContext = frame.context
elif isinstance(frame, LLMMessagesQueueFrame):
context = OpenAILLMContext.from_messages(frame.messages)
else:
yield frame
return
function_name = ""
arguments = ""
yield LLMResponseStartFrame()
chunk_stream: AsyncStream[ChatCompletionChunk] = (
await self._stream_chat_completions(context)
)
async for chunk in chunk_stream:
if len(chunk.choices) == 0:
continue
if chunk.choices[0].delta.tool_calls:
# We're streaming the LLM response to enable the fastest response times.
# For text, we just yield each chunk as we receive it and count on consumers
# to do whatever coalescing they need (eg. to pass full sentences to TTS)
#
# If the LLM is a function call, we'll do some coalescing here.
# If the response contains a function name, we'll yield a frame to tell consumers
# that they can start preparing to call the function with that name.
# We accumulate all the arguments for the rest of the streamed response, then when
# the response is done, we package up all the arguments and the function name and
# yield a frame containing the function name and the arguments.
tool_call = chunk.choices[0].delta.tool_calls[0]
if tool_call.function and tool_call.function.name:
function_name += tool_call.function.name
yield LLMFunctionStartFrame(function_name=tool_call.function.name)
if tool_call.function and tool_call.function.arguments:
# Keep iterating through the response to collect all the argument fragments and
# yield a complete LLMFunctionCallFrame after run_llm_async completes
arguments += tool_call.function.arguments
elif chunk.choices[0].delta.content:
yield TextFrame(chunk.choices[0].delta.content)
# if we got a function name and arguments, yield the frame with all the info so
# frame consumers can take action based on the function call.
if function_name and arguments:
yield LLMFunctionCallFrame(function_name=function_name, arguments=arguments)
yield LLMResponseEndFrame()

52
src/dailyai/services/openai_llm_context.py Normal file
View File

@@ -0,0 +1,52 @@
from typing import List
from openai._types import NOT_GIVEN, NotGiven
from openai.types.chat import (
ChatCompletionToolParam,
ChatCompletionToolChoiceOptionParam,
ChatCompletionMessageParam,
)
class OpenAILLMContext:
def __init__(
self,
messages: List[ChatCompletionMessageParam] | None = None,
tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN,
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN
):
self.messages: List[ChatCompletionMessageParam] = messages if messages else []
self.tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
self.tools: List[ChatCompletionToolParam] | NotGiven = tools
@staticmethod
def from_messages(messages: List[dict]) -> "OpenAILLMContext":
context = OpenAILLMContext()
for message in messages:
context.add_message({
"content":message["content"],
"role":message["role"],
"name":message["name"] if "name" in message else message["role"]
})
return context
#def __deepcopy__(self, memo):
def add_message(self, message: ChatCompletionMessageParam):
self.messages.append(message)
def get_messages(self) -> List[ChatCompletionMessageParam]:
return self.messages
def set_tool_choice(
self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven
):
self.tool_choice = tool_choice
def set_tools(self, tools:List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN):
if tools != NOT_GIVEN and len(tools) == 0:
tools = NOT_GIVEN
self.tools = tools