Compare commits
7 Commits
mb/update-
...
hush/rtviS
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd6379cb6a | ||
|
|
7618d7511a | ||
|
|
9ca7bad978 | ||
|
|
10289c1f1c | ||
|
|
213d5d6abc | ||
|
|
01f37f769d | ||
|
|
52b393537a |
@@ -1,17 +1,18 @@
|
|||||||
import {
|
import {
|
||||||
RTVIClientAudio,
|
RTVIClientAudio,
|
||||||
RTVIClientVideo,
|
RTVIClientVideo,
|
||||||
|
useRTVIClient,
|
||||||
useRTVIClientTransportState,
|
useRTVIClientTransportState,
|
||||||
} from '@pipecat-ai/client-react';
|
} from "@pipecat-ai/client-react";
|
||||||
import { RTVIProvider } from './providers/RTVIProvider';
|
import { RTVIProvider } from "./providers/RTVIProvider";
|
||||||
import { ConnectButton } from './components/ConnectButton';
|
import { ConnectButton } from "./components/ConnectButton";
|
||||||
import { StatusDisplay } from './components/StatusDisplay';
|
import { StatusDisplay } from "./components/StatusDisplay";
|
||||||
import { DebugDisplay } from './components/DebugDisplay';
|
import { DebugDisplay } from "./components/DebugDisplay";
|
||||||
import './App.css';
|
import "./App.css";
|
||||||
|
|
||||||
function BotVideo() {
|
function BotVideo() {
|
||||||
const transportState = useRTVIClientTransportState();
|
const transportState = useRTVIClientTransportState();
|
||||||
const isConnected = transportState !== 'disconnected';
|
const isConnected = transportState !== "disconnected";
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="bot-container">
|
<div className="bot-container">
|
||||||
@@ -23,11 +24,31 @@ function BotVideo() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function AppContent() {
|
function AppContent() {
|
||||||
|
const client = useRTVIClient();
|
||||||
return (
|
return (
|
||||||
<div className="app">
|
<div className="app">
|
||||||
<div className="status-bar">
|
<div className="status-bar">
|
||||||
<StatusDisplay />
|
<StatusDisplay />
|
||||||
<ConnectButton />
|
<ConnectButton />
|
||||||
|
<div
|
||||||
|
className="controls"
|
||||||
|
onClick={async () => {
|
||||||
|
if (!client) {
|
||||||
|
console.error("RTVI client is not initialized");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
client.action({
|
||||||
|
service: "tts",
|
||||||
|
action: "say",
|
||||||
|
arguments: [
|
||||||
|
{ name: "text", value: "Hello, world!" },
|
||||||
|
{ name: "interrupt", value: false },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<button className="connect-btn">Say something</button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="main-content">
|
<div className="main-content">
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ the conversation flow.
|
|||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
@@ -32,15 +33,24 @@ from pipecat.frames.frames import (
|
|||||||
BotStartedSpeakingFrame,
|
BotStartedSpeakingFrame,
|
||||||
BotStoppedSpeakingFrame,
|
BotStoppedSpeakingFrame,
|
||||||
Frame,
|
Frame,
|
||||||
|
LLMMessagesAppendFrame,
|
||||||
OutputImageRawFrame,
|
OutputImageRawFrame,
|
||||||
SpriteFrame,
|
SpriteFrame,
|
||||||
|
TTSSpeakFrame,
|
||||||
)
|
)
|
||||||
from pipecat.pipeline.pipeline import Pipeline
|
from pipecat.pipeline.pipeline import Pipeline
|
||||||
from pipecat.pipeline.runner import PipelineRunner
|
from pipecat.pipeline.runner import PipelineRunner
|
||||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||||
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
|
from pipecat.processors.frameworks.rtvi import (
|
||||||
|
ActionResult,
|
||||||
|
RTVIAction,
|
||||||
|
RTVIActionArgument,
|
||||||
|
RTVIObserver,
|
||||||
|
RTVIProcessor,
|
||||||
|
RTVIService,
|
||||||
|
)
|
||||||
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
from pipecat.services.elevenlabs.tts import ElevenLabsTTSService
|
||||||
from pipecat.services.openai.llm import OpenAILLMService
|
from pipecat.services.openai.llm import OpenAILLMService
|
||||||
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
from pipecat.transports.services.daily import DailyParams, DailyTransport
|
||||||
@@ -181,8 +191,57 @@ async def main():
|
|||||||
#
|
#
|
||||||
# RTVI events for Pipecat client UI
|
# RTVI events for Pipecat client UI
|
||||||
#
|
#
|
||||||
|
|
||||||
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
|
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
|
||||||
|
|
||||||
|
rtvi_tts = RTVIService(
|
||||||
|
name="tts",
|
||||||
|
options=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
async def action_tts_say_handler(
|
||||||
|
rtvi: RTVIProcessor, service: str, arguments: Dict[str, Any]
|
||||||
|
) -> ActionResult:
|
||||||
|
if "interrupt" in arguments and arguments["interrupt"]:
|
||||||
|
# interrupting breaks function handling
|
||||||
|
await rtvi.interrupt_bot()
|
||||||
|
if "text" in arguments:
|
||||||
|
save = arguments["save"] if "save" in arguments else False
|
||||||
|
frame = TTSSpeakFrame(text=arguments["text"])
|
||||||
|
await rtvi.push_frame(frame)
|
||||||
|
if save:
|
||||||
|
llm_frame = LLMMessagesAppendFrame(
|
||||||
|
messages=[{"role": "assistant", "content": arguments["text"]}]
|
||||||
|
)
|
||||||
|
await rtvi.push_frame(llm_frame)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
action_tts_say = RTVIAction(
|
||||||
|
service="tts",
|
||||||
|
action="say",
|
||||||
|
result="bool",
|
||||||
|
arguments=[
|
||||||
|
RTVIActionArgument(name="text", type="string"),
|
||||||
|
RTVIActionArgument(name="save_in_context", type="bool"),
|
||||||
|
],
|
||||||
|
handler=action_tts_say_handler,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def action_tts_interrupt_handler(
|
||||||
|
rtvi: RTVIProcessor, service: str, arguments: Dict[str, Any]
|
||||||
|
) -> ActionResult:
|
||||||
|
await rtvi.interrupt_bot()
|
||||||
|
return True
|
||||||
|
|
||||||
|
action_tts_interrupt = RTVIAction(
|
||||||
|
service="tts", action="interrupt", result="bool", handler=action_tts_interrupt_handler
|
||||||
|
)
|
||||||
|
|
||||||
|
rtvi.register_service(rtvi_tts)
|
||||||
|
rtvi.register_action(action_tts_say)
|
||||||
|
rtvi.register_action(action_tts_interrupt)
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
[
|
[
|
||||||
transport.input(),
|
transport.input(),
|
||||||
|
|||||||
Reference in New Issue
Block a user