working 19-openai-realtime-beta.py example
This commit is contained in:
Kwindla Hultman Kramer
@@ -15,7 +15,7 @@ from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.logger import FrameLogger
|
||||
from pipecat.services.openai import OpenAILLMServiceRealtimeBeta
|
||||
from pipecat.services.openai_realtime_beta import OpenAILLMServiceRealtimeBeta
|
||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||
from pipecat.vad.silero import SileroVADAnalyzer
|
||||
|
||||
@@ -40,10 +40,14 @@ async def main():
|
||||
token,
|
||||
"Respond bot",
|
||||
DailyParams(
|
||||
audio_in_enabled=True,
|
||||
audio_in_sample_rate=24000,
|
||||
audio_out_enabled=True,
|
||||
audio_out_sample_rate=24000,
|
||||
transcription_enabled=True,
|
||||
vad_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
vad_audio_passthrough=True,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -61,9 +65,9 @@ async def main():
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(), # Transport user input
|
||||
fl1,
|
||||
# fl1,
|
||||
llm, # LLM
|
||||
fl2,
|
||||
# fl2,
|
||||
transport.output(), # Transport bot output
|
||||
]
|
||||
)
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
@@ -18,8 +17,6 @@ from PIL import Image
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
EndFrame,
|
||||
ErrorFrame,
|
||||
Frame,
|
||||
FunctionCallInProgressFrame,
|
||||
@@ -28,7 +25,6 @@ from pipecat.frames.frames import (
|
||||
LLMFullResponseStartFrame,
|
||||
LLMMessagesFrame,
|
||||
LLMUpdateSettingsFrame,
|
||||
StartFrame,
|
||||
StartInterruptionFrame,
|
||||
TextFrame,
|
||||
TTSAudioRawFrame,
|
||||
@@ -60,7 +56,6 @@ try:
|
||||
DefaultAsyncHttpxClient,
|
||||
)
|
||||
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
|
||||
import websockets
|
||||
except ModuleNotFoundError as e:
|
||||
logger.error(f"Exception: {e}")
|
||||
logger.error(
|
||||
@@ -68,14 +63,6 @@ except ModuleNotFoundError as e:
|
||||
)
|
||||
raise Exception(f"Missing module: {e}")
|
||||
|
||||
# websocket logger
|
||||
import logging
|
||||
|
||||
logging.basicConfig(
|
||||
format="%(message)s",
|
||||
level=logging.DEBUG,
|
||||
)
|
||||
|
||||
|
||||
ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"]
|
||||
|
||||
@@ -587,91 +574,3 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing frame: {e}")
|
||||
|
||||
|
||||
class OpenAILLMServiceRealtimeBeta(LLMService):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
base_url="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(base_url=base_url, **kwargs)
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
self._websocket = None
|
||||
self._receive_task = None
|
||||
|
||||
async def start(self, frame: StartFrame):
|
||||
await super().start(frame)
|
||||
await self._connect()
|
||||
|
||||
async def stop(self, frame: EndFrame):
|
||||
await super().stop(frame)
|
||||
await self._disconnect()
|
||||
|
||||
async def cancel(self, frame: CancelFrame):
|
||||
await super().cancel(frame)
|
||||
await self._disconnect()
|
||||
|
||||
async def _connect(self):
|
||||
try:
|
||||
logger.debug(f"connecting to {self.base_url} with api_key {self.api_key}")
|
||||
self._websocket = await websockets.connect(
|
||||
uri=self.base_url,
|
||||
extra_headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"OpenAI-Beta": "realtime=v1",
|
||||
},
|
||||
)
|
||||
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
|
||||
except Exception as e:
|
||||
logger.error(f"{self} initialization error: {e}")
|
||||
self._websocket = None
|
||||
|
||||
async def _disconnect(self):
|
||||
pass
|
||||
|
||||
async def _receive_task_handler(self):
|
||||
try:
|
||||
async for message in self._get_websocket():
|
||||
msg = json.loads(message)
|
||||
logger.debug(f"Received message: {msg}")
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"{self} exception: {e}")
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
# if isinstance(frame, TranscriptionFrame):
|
||||
# self._websocket.send(
|
||||
# json.dumps(
|
||||
# {
|
||||
# {
|
||||
# "type": "response.create",
|
||||
# "response": {
|
||||
# "modalities": ["text"],
|
||||
# "instructions": frame.text,
|
||||
# },
|
||||
# }
|
||||
# }
|
||||
# )
|
||||
# )
|
||||
|
||||
# async def get_chat_completions(
|
||||
# self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
|
||||
# ) -> AsyncStream[ChatCompletionChunk]:
|
||||
# async def _empty_async_generator() -> AsyncGenerator[str, None]:
|
||||
# try:
|
||||
# if False:
|
||||
# yield ""
|
||||
# except asyncio.CancelledError:
|
||||
# return
|
||||
# except Exception as e:
|
||||
# logger.error(f"{self} exception: {e}")
|
||||
|
||||
# return _empty_async_generator()
|
||||
|
||||
205
src/pipecat/services/openai_realtime_beta.py
Normal file
205
src/pipecat/services/openai_realtime_beta.py
Normal file
@@ -0,0 +1,205 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import websockets
|
||||
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
Frame,
|
||||
EndFrame,
|
||||
InputAudioRawFrame,
|
||||
StartFrame,
|
||||
TextFrame,
|
||||
TranscriptionFrame,
|
||||
TTSAudioRawFrame,
|
||||
)
|
||||
from pipecat.processors.frame_processor import FrameDirection
|
||||
from pipecat.services.ai_services import LLMService
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
|
||||
from loguru import logger
|
||||
|
||||
# temp: websocket logger
|
||||
# import logging
|
||||
|
||||
# logging.basicConfig(
|
||||
# format="%(message)s",
|
||||
# level=logging.DEBUG,
|
||||
# )
|
||||
|
||||
|
||||
class OpenAILLMServiceRealtimeBeta(LLMService):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_key: str,
|
||||
base_url="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01",
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(base_url=base_url, **kwargs)
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url
|
||||
self._websocket = None
|
||||
self._receive_task = None
|
||||
|
||||
self._session_properties = None
|
||||
self._responses_in_flight = {}
|
||||
|
||||
async def start(self, frame: StartFrame):
|
||||
await super().start(frame)
|
||||
await self._connect()
|
||||
|
||||
async def stop(self, frame: EndFrame):
|
||||
await super().stop(frame)
|
||||
await self._disconnect()
|
||||
|
||||
async def cancel(self, frame: CancelFrame):
|
||||
await super().cancel(frame)
|
||||
await self._disconnect()
|
||||
|
||||
async def _connect(self):
|
||||
try:
|
||||
logger.debug(f"connecting to {self.base_url} with api_key {self.api_key}")
|
||||
self._websocket = await websockets.connect(
|
||||
uri=self.base_url,
|
||||
extra_headers={
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"OpenAI-Beta": "realtime=v1",
|
||||
},
|
||||
)
|
||||
self._receive_task = self.get_event_loop().create_task(self._receive_task_handler())
|
||||
except Exception as e:
|
||||
logger.error(f"{self} initialization error: {e}")
|
||||
self._websocket = None
|
||||
|
||||
async def _disconnect(self):
|
||||
try:
|
||||
await self.stop_all_metrics()
|
||||
|
||||
if self._websocket:
|
||||
await self._websocket.close()
|
||||
self._websocket = None
|
||||
|
||||
if self._receive_task:
|
||||
self._receive_task.cancel()
|
||||
await self._receive_task
|
||||
self._receive_task = None
|
||||
|
||||
self._context_id = None
|
||||
except Exception as e:
|
||||
logger.error(f"{self} error closing websocket: {e}")
|
||||
|
||||
def _get_websocket(self):
|
||||
if self._websocket:
|
||||
return self._websocket
|
||||
raise Exception("Websocket not connected")
|
||||
|
||||
async def _receive_task_handler(self):
|
||||
try:
|
||||
async for message in self._get_websocket():
|
||||
msg = json.loads(message)
|
||||
logger.debug(f"Received message: {msg}")
|
||||
if not msg:
|
||||
continue
|
||||
if msg["type"] == "session.created":
|
||||
self._session_properties = msg["session"]
|
||||
elif msg["type"] == "response.created":
|
||||
pass
|
||||
elif msg["type"] == "response.output_item.added":
|
||||
pass
|
||||
elif msg["type"] == "response.audio.delta":
|
||||
frame = TTSAudioRawFrame(
|
||||
audio=base64.b64decode(msg["delta"]),
|
||||
sample_rate=24000,
|
||||
num_channels=1,
|
||||
)
|
||||
await self.push_frame(frame)
|
||||
elif msg["type"] == "response.text.delta":
|
||||
logger.debug(f"!!! {msg['delta']}")
|
||||
elif msg["type"] == "response.output_item.done":
|
||||
if msg["item"]["type"] == "message":
|
||||
for item in msg["item"]["content"]:
|
||||
if item["type"] == "text":
|
||||
await self.push_frame(TextFrame(item["text"]))
|
||||
elif msg["type"] == "response.done":
|
||||
await self.stop_processing_metrics()
|
||||
await self.push_frame(LLMFullResponseEndFrame())
|
||||
elif msg["type"] == "response.error":
|
||||
logger.error(f"Error: {msg}")
|
||||
raise Exception(f"Error: {msg}")
|
||||
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"{self} exception: {e}")
|
||||
|
||||
async def _create_response(self, context: OpenAILLMContext, messages: list):
|
||||
try:
|
||||
await self.push_frame(LLMFullResponseStartFrame())
|
||||
await self.start_processing_metrics()
|
||||
await self._websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"status": "completed",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": messages[0]["content"]}],
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
await self._websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "response.create",
|
||||
"response": {
|
||||
"modalities": ["audio", "text"],
|
||||
"instructions": "Your knowledge cutoff is 2023-10. You are a helpful, witty, and friendly AI. You are a participant in a voice chat. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and playful tone. If interacting in a non-English language, start by using the standard accent or dialect familiar to the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, even if you're asked about them.",
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"{self} exception: {e}")
|
||||
|
||||
async def _send_user_audio(self, frame):
|
||||
payload = base64.b64encode(frame.audio).decode("utf-8")
|
||||
await self._websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": payload,
|
||||
},
|
||||
)
|
||||
)
|
||||
# await self._websocket.send(json.dumps(({"type": "input_audio_buffer.commit"})))
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, TranscriptionFrame):
|
||||
messages = [{"role": "user", "content": frame.text}]
|
||||
context = OpenAILLMContext(messages)
|
||||
await self._create_response(context, messages)
|
||||
if isinstance(frame, InputAudioRawFrame):
|
||||
await self._send_user_audio(frame)
|
||||
|
||||
# async def get_chat_completions(
|
||||
# self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
|
||||
# ) -> AsyncStream[ChatCompletionChunk]:
|
||||
# async def _empty_async_generator() -> AsyncGenerator[str, None]:
|
||||
# try:
|
||||
# if False:
|
||||
# yield ""
|
||||
# except asyncio.CancelledError:
|
||||
# return
|
||||
# except Exception as e:
|
||||
# logger.error(f"{self} exception: {e}")
|
||||
|
||||
# return _empty_async_generator()
|
||||
Reference in New Issue
Block a user