update CHANGELOG and create websocker-server instructions
This commit is contained in:
Aleix Conchillo Flaqué
27
examples/websocket-server/README.md
Normal file
27
examples/websocket-server/README.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# Websocket Server
|
||||
|
||||
This is an example that shows how to use `WebsocketServerTransport` to communicate with a web client.
|
||||
|
||||
## Get started
|
||||
|
||||
```python
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
## Run the server
|
||||
|
||||
```bash
|
||||
python server.py
|
||||
```
|
||||
|
||||
## Run the HTTP server
|
||||
|
||||
This will host the static web client:
|
||||
|
||||
```bash
|
||||
python -m http.server
|
||||
```
|
||||
|
||||
Then, visit `http://localhost:8000` in your browser to start a session.
|
||||
43
examples/websocket-server/frames.proto
Normal file
43
examples/websocket-server/frames.proto
Normal file
@@ -0,0 +1,43 @@
|
||||
//
|
||||
// Copyright (c) 2024, Daily
|
||||
//
|
||||
// SPDX-License-Identifier: BSD 2-Clause License
|
||||
//
|
||||
|
||||
// Generate frames_pb2.py with:
|
||||
//
|
||||
// python -m grpc_tools.protoc --proto_path=./ --python_out=./protobufs frames.proto
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package pipecat;
|
||||
|
||||
message TextFrame {
|
||||
uint64 id = 1;
|
||||
string name = 2;
|
||||
string text = 3;
|
||||
}
|
||||
|
||||
message AudioRawFrame {
|
||||
uint64 id = 1;
|
||||
string name = 2;
|
||||
bytes audio = 3;
|
||||
uint32 sample_rate = 4;
|
||||
uint32 num_channels = 5;
|
||||
}
|
||||
|
||||
message TranscriptionFrame {
|
||||
uint64 id = 1;
|
||||
string name = 2;
|
||||
string text = 3;
|
||||
string user_id = 4;
|
||||
string timestamp = 5;
|
||||
}
|
||||
|
||||
message Frame {
|
||||
oneof frame {
|
||||
TextFrame text = 1;
|
||||
AudioRawFrame audio = 2;
|
||||
TranscriptionFrame transcription = 3;
|
||||
}
|
||||
}
|
||||
204
examples/websocket-server/index.html
Normal file
204
examples/websocket-server/index.html
Normal file
@@ -0,0 +1,204 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<script src="https://cdn.jsdelivr.net/npm/protobufjs@7.X.X/dist/protobuf.min.js"></script>
|
||||
<title>Pipecat WebSocket Client Example</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<h1>Pipecat WebSocket Client Example</h1>
|
||||
<h3><div id="progressText">Loading, wait...</div></h2>
|
||||
<button id="startAudioBtn">Start Audio</button>
|
||||
<button id="stopAudioBtn">Stop Audio</button>
|
||||
<script>
|
||||
const SAMPLE_RATE = 16000;
|
||||
const NUM_CHANNELS = 1;
|
||||
const PLAY_TIME_RESET_THRESHOLD_MS = 1.0;
|
||||
|
||||
// The protobuf type. We will load it later.
|
||||
let Frame = null;
|
||||
|
||||
// The websocket connection.
|
||||
let ws = null;
|
||||
|
||||
// The audio context
|
||||
let audioContext = null;
|
||||
|
||||
// The audio context media stream source
|
||||
let source = null;
|
||||
|
||||
// The microphone stream from getUserMedia. SHould be sampled to the
|
||||
// proper sample rate.
|
||||
let microphoneStream = null;
|
||||
|
||||
// Script processor to get data from microphone.
|
||||
let scriptProcessor = null;
|
||||
|
||||
// AudioContext play time.
|
||||
let playTime = 0;
|
||||
|
||||
// Last time we received a websocket message.
|
||||
let lastMessageTime = 0;
|
||||
|
||||
// Whether we should be playing audio.
|
||||
let isPlaying = false;
|
||||
|
||||
let startBtn = document.getElementById('startAudioBtn');
|
||||
let stopBtn = document.getElementById('stopAudioBtn');
|
||||
|
||||
const proto = protobuf.load("frames.proto", (err, root) => {
|
||||
if (err) {
|
||||
throw err;
|
||||
}
|
||||
Frame = root.lookupType("pipecat.Frame");
|
||||
const progressText = document.getElementById("progressText");
|
||||
progressText.textContent = "We are ready! Make sure to run the server and then click `Start Audio`.";
|
||||
|
||||
startBtn.disabled = false;
|
||||
stopBtn.disabled = true;
|
||||
});
|
||||
|
||||
function initWebSocket() {
|
||||
ws = new WebSocket('ws://localhost:8765');
|
||||
|
||||
ws.addEventListener('open', () => console.log('WebSocket connection established.'));
|
||||
ws.addEventListener('message', handleWebSocketMessage);
|
||||
ws.addEventListener('close', (event) => {
|
||||
console.log("WebSocket connection closed.", event.code, event.reason);
|
||||
stopAudio(false);
|
||||
});
|
||||
ws.addEventListener('error', (event) => console.error('WebSocket error:', event));
|
||||
}
|
||||
|
||||
async function handleWebSocketMessage(event) {
|
||||
const arrayBuffer = await event.data.arrayBuffer();
|
||||
if (isPlaying) {
|
||||
enqueueAudioFromProto(arrayBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueAudioFromProto(arrayBuffer) {
|
||||
const parsedFrame = Frame.decode(new Uint8Array(arrayBuffer));
|
||||
if (!parsedFrame?.audio) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Reset play time if it's been a while we haven't played anything.
|
||||
const diffTime = audioContext.currentTime - lastMessageTime;
|
||||
if ((playTime == 0) || (diffTime > PLAY_TIME_RESET_THRESHOLD_MS)) {
|
||||
playTime = audioContext.currentTime;
|
||||
}
|
||||
lastMessageTime = audioContext.currentTime;
|
||||
|
||||
// We should be able to use parsedFrame.audio.audio.buffer but for
|
||||
// some reason that contains all the bytes from the protobuf message.
|
||||
const audioVector = Array.from(parsedFrame.audio.audio);
|
||||
const audioArray = new Uint8Array(audioVector);
|
||||
|
||||
audioContext.decodeAudioData(audioArray.buffer, function(buffer) {
|
||||
const source = new AudioBufferSourceNode(audioContext);
|
||||
source.buffer = buffer;
|
||||
source.start(playTime);
|
||||
source.connect(audioContext.destination);
|
||||
playTime = playTime + buffer.duration;
|
||||
});
|
||||
}
|
||||
|
||||
function convertFloat32ToS16PCM(float32Array) {
|
||||
let int16Array = new Int16Array(float32Array.length);
|
||||
|
||||
for (let i = 0; i < float32Array.length; i++) {
|
||||
let clampedValue = Math.max(-1, Math.min(1, float32Array[i]));
|
||||
int16Array[i] = clampedValue < 0 ? clampedValue * 32768 : clampedValue * 32767;
|
||||
}
|
||||
return int16Array;
|
||||
}
|
||||
|
||||
function startAudioBtnHandler() {
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
alert('getUserMedia is not supported in your browser.');
|
||||
return;
|
||||
}
|
||||
|
||||
startBtn.disabled = true;
|
||||
stopBtn.disabled = false;
|
||||
|
||||
audioContext = new (window.AudioContext || window.webkitAudioContext)({
|
||||
latencyHint: "interactive",
|
||||
sampleRate: SAMPLE_RATE
|
||||
});
|
||||
|
||||
isPlaying = true;
|
||||
|
||||
initWebSocket();
|
||||
|
||||
navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
sampleRate: SAMPLE_RATE,
|
||||
channelCount: NUM_CHANNELS,
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
}
|
||||
}).then((stream) => {
|
||||
microphoneStream = stream;
|
||||
// 512 is closest thing to 200ms.
|
||||
scriptProcessor = audioContext.createScriptProcessor(512, 1, 1);
|
||||
source = audioContext.createMediaStreamSource(stream);
|
||||
source.connect(scriptProcessor);
|
||||
scriptProcessor.connect(audioContext.destination);
|
||||
|
||||
scriptProcessor.onaudioprocess = (event) => {
|
||||
if (!ws) {
|
||||
return;
|
||||
}
|
||||
|
||||
const audioData = event.inputBuffer.getChannelData(0);
|
||||
const pcmS16Array = convertFloat32ToS16PCM(audioData);
|
||||
const pcmByteArray = new Uint8Array(pcmS16Array.buffer);
|
||||
const frame = Frame.create({
|
||||
audio: {
|
||||
audio: Array.from(pcmByteArray),
|
||||
sampleRate: SAMPLE_RATE,
|
||||
numChannels: NUM_CHANNELS
|
||||
}
|
||||
});
|
||||
const encodedFrame = new Uint8Array(Frame.encode(frame).finish());
|
||||
ws.send(encodedFrame);
|
||||
};
|
||||
}).catch((error) => console.error('Error accessing microphone:', error));
|
||||
}
|
||||
|
||||
function stopAudio(closeWebsocket) {
|
||||
isPlaying = false;
|
||||
startBtn.disabled = false;
|
||||
stopBtn.disabled = true;
|
||||
|
||||
if (ws && closeWebsocket) {
|
||||
ws.close();
|
||||
ws = null;
|
||||
}
|
||||
|
||||
if (scriptProcessor) {
|
||||
scriptProcessor.disconnect();
|
||||
}
|
||||
if (source) {
|
||||
source.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
function stopAudioBtnHandler() {
|
||||
stopAudio(true);
|
||||
}
|
||||
|
||||
startBtn.addEventListener('click', startAudioBtnHandler);
|
||||
stopBtn.addEventListener('click', stopAudioBtnHandler);
|
||||
startBtn.disabled = true;
|
||||
stopBtn.disabled = true;
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
2
examples/websocket-server/requirements.txt
Normal file
2
examples/websocket-server/requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
python-dotenv
|
||||
pipecat-ai[openai,silero,websocket,whisper]
|
||||
85
examples/websocket-server/server.py
Normal file
85
examples/websocket-server/server.py
Normal file
@@ -0,0 +1,85 @@
|
||||
#
|
||||
# Copyright (c) 2024, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
from pipecat.frames.frames import LLMMessagesFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineTask
|
||||
from pipecat.processors.aggregators.llm_response import LLMAssistantResponseAggregator, LLMUserResponseAggregator
|
||||
from pipecat.services.elevenlabs import ElevenLabsTTSService
|
||||
from pipecat.services.openai import OpenAILLMService
|
||||
from pipecat.services.whisper import WhisperSTTService
|
||||
from pipecat.transports.network.websocket_server import WebsocketServerParams, WebsocketServerTransport
|
||||
from pipecat.vad.silero import SileroVAD
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv(override=True)
|
||||
|
||||
logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
|
||||
async def main():
|
||||
async with aiohttp.ClientSession() as session:
|
||||
transport = WebsocketServerTransport(params=WebsocketServerParams(add_wav_header=True))
|
||||
|
||||
vad = SileroVAD(audio_passthrough=True)
|
||||
|
||||
llm = OpenAILLMService(
|
||||
api_key=os.getenv("OPENAI_API_KEY"),
|
||||
model="gpt-4o")
|
||||
|
||||
stt = WhisperSTTService()
|
||||
|
||||
tts = ElevenLabsTTSService(
|
||||
aiohttp_session=session,
|
||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
|
||||
)
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
|
||||
},
|
||||
]
|
||||
|
||||
tma_in = LLMUserResponseAggregator(messages)
|
||||
tma_out = LLMAssistantResponseAggregator(messages)
|
||||
|
||||
pipeline = Pipeline([
|
||||
transport.input(), # Websocket input from client
|
||||
vad, # VAD to detect user speech
|
||||
stt, # Speech-To-Text
|
||||
tma_in, # User responses
|
||||
llm, # LLM
|
||||
tts, # Text-To-Speech
|
||||
transport.output(), # Websocket output to client
|
||||
tma_out # LLM responses
|
||||
])
|
||||
|
||||
task = PipelineTask(pipeline)
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(transport, client):
|
||||
# Kick off the conversation.
|
||||
messages.append(
|
||||
{"role": "system", "content": "Please introduce yourself to the user."})
|
||||
await task.queue_frames([LLMMessagesFrame(messages)])
|
||||
|
||||
runner = PipelineRunner()
|
||||
|
||||
await runner.run(task)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user