From c37552de70fe2f0253fa5663014d6af013fb7645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 9 Aug 2024 18:12:37 -0700 Subject: [PATCH] processors(rtvi): add support for action responses --- src/pipecat/processors/frameworks/rtvi.py | 29 ++++++++++------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 49ebe759f..98f114dd9 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -6,7 +6,7 @@ import asyncio -from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional +from typing import Any, Awaitable, Callable, Dict, List, Literal, Optional, Union from pydantic import BaseModel, Field, PrivateAttr, ValidationError from pipecat.frames.frames import ( @@ -26,6 +26,7 @@ from loguru import logger RTVI_PROTOCOL_VERSION = "0.1" +ActionResult = Union[bool,int,float,str,list,dict] class RTVIServiceOption(BaseModel): name: str @@ -60,7 +61,7 @@ class RTVIAction(BaseModel): service: str action: str arguments: List[RTVIActionArgument] = [] - handler: Callable[["RTVIProcessor", str, Dict[str, Any]], Awaitable[None]] = Field(exclude=True) + handler: Callable[["RTVIProcessor", str, Dict[str, Any]], Awaitable[ActionResult]] = Field(exclude=True) _arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={}) def model_post_init(self, __context: Any) -> None: @@ -150,14 +151,15 @@ class RTVIConfigResponse(BaseModel): data: RTVIConfig -class RTVILLMContextMessageData(BaseModel): - messages: List[dict] +class RTVIActionResponseData(BaseModel): + result: ActionResult -class RTVILLMContextMessage(BaseModel): +class RTVIActionResponse(BaseModel): label: Literal["rtvi-ai"] = "rtvi-ai" - type: Literal["llm-context"] = "llm-context" - data: RTVILLMContextMessageData + type: Literal["action-response"] = "action-response" + id: str + data: RTVIActionResponseData class RTVIBotReadyData(BaseModel): @@ -343,8 +345,6 @@ class RTVIProcessor(FrameProcessor): case "action": action = RTVIActionRun.model_validate(message.data) await self._handle_action(message.id, action) - # case "llm-get-context": - # await self._handle_llm_get_context() case _: await self._send_error_response(message.id, f"Unsupported type {message.type}") @@ -399,13 +399,10 @@ class RTVIProcessor(FrameProcessor): if data.arguments: for arg in data.arguments: arguments[arg.name] = arg.value - await action.handler(self, action.service, arguments) - - # async def _handle_llm_get_context(self): - # data = RTVILLMContextMessageData(messages=self._tma_in.messages) - # message = RTVILLMContextMessage(data=data) - # frame = TransportMessageFrame(message=message.model_dump(exclude_none=True)) - # await self.push_frame(frame) + result = await action.handler(self, action.service, arguments) + message = RTVIActionResponse(id=request_id, data=RTVIActionResponseData(result=result)) + frame = TransportMessageFrame(message=message.model_dump(exclude_none=True)) + await self.push_frame(frame) async def _send_bot_ready(self): message = RTVIBotReady(