wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

processors(rtvi): renamed realtime-ai to rtvi

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2024-07-21 14:53:49 -07:00
parent dfbc11300c
commit be6995cfdf
3 changed files with 114 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
examples/realtime-ai/bot.py → examples/rtvi/bot.py
View File

@@ -11,12 +11,12 @@ import os
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.pipeline.runner import PipelineRunner
from pipecat.processors.frameworks.realtimeai import (
RealtimeAIConfig,
RealtimeAILLMConfig,
RealtimeAIProcessor,
RealtimeAISetup,
RealtimeAITTSConfig)
from pipecat.processors.frameworks.rtvi import (
RTVIConfig,
RTVILLMConfig,
RTVIProcessor,
RTVISetup,
RTVITTSConfig)
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.vad.silero import SileroVADAnalyzer
@@ -43,14 +43,14 @@ async def main(room_url, token):
vad_analyzer=SileroVADAnalyzer()
))
llm = RealtimeAILLMConfig(
llm = RTVILLMConfig(
model="llama3-70b-8192",
messages=[{"role": "system", "content": "You are a helpful assistant named Gary. Briefly say hello!"}]
)
tts = RealtimeAITTSConfig(voice="79a125e8-cd45-4c13-8a67-188112f4dd22")
setup = RealtimeAISetup(config=RealtimeAIConfig(llm=llm, tts=tts))
tts = RTVITTSConfig(voice="79a125e8-cd45-4c13-8a67-188112f4dd22")
setup = RTVISetup(config=RTVIConfig(llm=llm, tts=tts))
rtai = RealtimeAIProcessor(
rtai = RTVIProcessor(
transport=transport,
setup=setup,
llm_api_key=os.getenv("OPENAI_API_KEY"),

58
examples/rtvi/runner.py Normal file
View File

@@ -0,0 +1,58 @@
import argparse
import os
import time
import urllib
import requests
def configure():
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument(
"-u",
"--url",
type=str,
required=False,
help="URL of the Daily room to join")
parser.add_argument(
"-k",
"--apikey",
type=str,
required=False,
help="Daily API Key (needed to create an owner token for the room)",
)
args, unknown = parser.parse_known_args()
url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
key = args.apikey or os.getenv("DAILY_API_KEY")
if not url:
raise Exception(
"No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.")
if not key:
raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
# Create a meeting token for the given room with an expiration 1 hour in
# the future.
room_name: str = urllib.parse.urlparse(url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={
"Authorization": f"Bearer {key}"},
json={
"properties": {
"room_name": room_name,
"is_owner": True,
"exp": expiration}},
)
if res.status_code != 200:
raise Exception(
f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
return (url, token)

92
src/pipecat/processors/frameworks/realtimeai.py → src/pipecat/processors/frameworks/rtvi.py
View File

@@ -46,103 +46,103 @@ DEFAULT_MODEL = "llama3-70b-8192"
DEFAULT_VOICE = "79a125e8-cd45-4c13-8a67-188112f4dd22"
class RealtimeAILLMConfig(BaseModel):
class RTVILLMConfig(BaseModel):
model: Optional[str] = None
messages: Optional[List[dict]] = None
class RealtimeAITTSConfig(BaseModel):
class RTVITTSConfig(BaseModel):
voice: Optional[str] = None
class RealtimeAIConfig(BaseModel):
llm: Optional[RealtimeAILLMConfig] = None
tts: Optional[RealtimeAITTSConfig] = None
class RTVIConfig(BaseModel):
llm: Optional[RTVILLMConfig] = None
tts: Optional[RTVITTSConfig] = None
class RealtimeAISetup(BaseModel):
config: Optional[RealtimeAIConfig] = None
class RTVISetup(BaseModel):
config: Optional[RTVIConfig] = None
class RealtimeAILLMMessageData(BaseModel):
class RTVILLMMessageData(BaseModel):
messages: List[dict]
class RealtimeAITTSMessageData(BaseModel):
class RTVITTSMessageData(BaseModel):
text: str
interrupt: Optional[bool] = False
class RealtimeAIMessageData(BaseModel):
setup: Optional[RealtimeAISetup] = None
config: Optional[RealtimeAIConfig] = None
llm: Optional[RealtimeAILLMMessageData] = None
tts: Optional[RealtimeAITTSMessageData] = None
class RTVIMessageData(BaseModel):
setup: Optional[RTVISetup] = None
config: Optional[RTVIConfig] = None
llm: Optional[RTVILLMMessageData] = None
tts: Optional[RTVITTSMessageData] = None
class RealtimeAIMessage(BaseModel):
class RTVIMessage(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: str
data: Optional[RealtimeAIMessageData] = None
data: Optional[RTVIMessageData] = None
class RealtimeAIBasicResponse(BaseModel):
class RTVIBasicResponse(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: str
success: bool
error: Optional[str] = None
class RealtimeAILLMContextMessageData(BaseModel):
class RTVILLMContextMessageData(BaseModel):
messages: List[dict]
class RealtimeAIBotReady(BaseModel):
class RTVIBotReady(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: Literal["bot-ready"] = "bot-ready"
class RealtimeAILLMContextMessage(BaseModel):
class RTVILLMContextMessage(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: Literal["llm-context"] = "llm-context"
data: RealtimeAILLMContextMessageData
data: RTVILLMContextMessageData
class RealtimeAITranscriptionMessageData(BaseModel):
class RTVITranscriptionMessageData(BaseModel):
text: str
user_id: str
timestamp: str
class RealtimeAITranscriptionMessage(BaseModel):
class RTVITranscriptionMessage(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: Literal["user-transcription"] = "user-transcription"
data: RealtimeAITranscriptionMessageData
data: RTVITranscriptionMessageData
class RealtimeAIInterimTranscriptionMessage(BaseModel):
class RTVIInterimTranscriptionMessage(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: Literal["user-interim-transcription"] = "user-interim-transcription"
data: RealtimeAITranscriptionMessageData
data: RTVITranscriptionMessageData
class RealtimeAIUserStartedSpeakingMessage(BaseModel):
class RTVIUserStartedSpeakingMessage(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: Literal["user-started-speaking"] = "user-started-speaking"
class RealtimeAIUserStoppedSpeakingMessage(BaseModel):
class RTVIUserStoppedSpeakingMessage(BaseModel):
label: Literal["realtime-ai"] = "realtime-ai"
type: Literal["user-stopped-speaking"] = "user-stopped-speaking"
class RealtimeAIProcessor(FrameProcessor):
class RTVIProcessor(FrameProcessor):
def __init__(
self,
*,
transport: BaseTransport,
setup: RealtimeAISetup | None = None,
setup: RTVISetup | None = None,
llm_api_key: str = "",
llm_base_url: str = "https://api.groq.com/openai/v1",
tts_api_key: str = "",
@@ -206,14 +206,14 @@ class RealtimeAIProcessor(FrameProcessor):
message = None
if isinstance(frame, TranscriptionFrame):
message = RealtimeAITranscriptionMessage(
data=RealtimeAITranscriptionMessageData(
message = RTVITranscriptionMessage(
data=RTVITranscriptionMessageData(
text=frame.text,
user_id=frame.user_id,
timestamp=frame.timestamp))
elif isinstance(frame, InterimTranscriptionFrame):
message = RealtimeAIInterimTranscriptionMessage(
data=RealtimeAITranscriptionMessageData(
message = RTVIInterimTranscriptionMessage(
data=RTVITranscriptionMessageData(
text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp))
if message:
@@ -223,9 +223,9 @@ class RealtimeAIProcessor(FrameProcessor):
async def _handle_interruptions(self, frame: Frame):
message = None
if isinstance(frame, UserStartedSpeakingFrame):
message = RealtimeAIUserStartedSpeakingMessage()
message = RTVIUserStartedSpeakingMessage()
elif isinstance(frame, UserStoppedSpeakingFrame):
message = RealtimeAIUserStoppedSpeakingMessage()
message = RTVIUserStoppedSpeakingMessage()
if message:
frame = TransportMessageFrame(message=message.model_dump(exclude_none=True))
@@ -233,7 +233,7 @@ class RealtimeAIProcessor(FrameProcessor):
async def _handle_message(self, frame: TransportMessageFrame):
try:
message = RealtimeAIMessage.model_validate(frame.message)
message = RTVIMessage.model_validate(frame.message)
except ValidationError as e:
await self._send_response("setup", False, f"invalid message: {e}")
return
@@ -263,7 +263,7 @@ class RealtimeAIProcessor(FrameProcessor):
except ValidationError as e:
await self._send_response(message.type, False, f"invalid message: {e}")
async def _handle_setup(self, setup: RealtimeAISetup | None):
async def _handle_setup(self, setup: RTVISetup | None):
try:
model = DEFAULT_MODEL
if setup and setup.config and setup.config.llm and setup.config.llm.model:
@@ -305,13 +305,13 @@ class RealtimeAIProcessor(FrameProcessor):
start_frame = dataclasses.replace(self._start_frame)
await self.push_frame(start_frame)
message = RealtimeAIBotReady()
message = RTVIBotReady()
frame = TransportMessageFrame(message=message.model_dump(exclude_none=True))
await self.push_frame(frame)
except Exception as e:
await self._send_response("setup", False, f"unable to create pipeline: {e}")
async def _handle_config_update(self, config: RealtimeAIConfig):
async def _handle_config_update(self, config: RTVIConfig):
if config.llm and config.llm.model:
frame = LLMModelUpdateFrame(config.llm.model)
await self.push_frame(frame)
@@ -323,22 +323,22 @@ class RealtimeAIProcessor(FrameProcessor):
await self.push_frame(frame)
async def _handle_llm_get_context(self):
data = RealtimeAILLMContextMessageData(messages=self._tma_in.messages)
message = RealtimeAILLMContextMessage(data=data)
data = RTVILLMContextMessageData(messages=self._tma_in.messages)
message = RTVILLMContextMessage(data=data)
frame = TransportMessageFrame(message=message.model_dump(exclude_none=True))
await self.push_frame(frame)
async def _handle_llm_append_context(self, data: RealtimeAILLMMessageData):
async def _handle_llm_append_context(self, data: RTVILLMMessageData):
if data and data.messages:
frame = LLMMessagesAppendFrame(data.messages)
await self.push_frame(frame)
async def _handle_llm_update_context(self, data: RealtimeAILLMMessageData):
async def _handle_llm_update_context(self, data: RTVILLMMessageData):
if data and data.messages:
frame = LLMMessagesUpdateFrame(data.messages)
await self.push_frame(frame)
async def _handle_tts_speak(self, data: RealtimeAITTSMessageData):
async def _handle_tts_speak(self, data: RTVITTSMessageData):
if data and data.text:
if data.interrupt:
await self._handle_tts_interrupt()
@@ -363,6 +363,6 @@ class RealtimeAIProcessor(FrameProcessor):
if parent and self._start_frame:
parent.link(pipeline)
message = RealtimeAIBasicResponse(type=type, success=success, error=error)
message = RTVIBasicResponse(type=type, success=success, error=error)
frame = TransportMessageFrame(message=message.model_dump(exclude_none=True))
await self.push_frame(frame)