Full video transformation example using SmallWebRTCTransport.
This commit is contained in:
Filipi Fuchter
43
examples/aiortc/video-transform/README.rst
Normal file
43
examples/aiortc/video-transform/README.rst
Normal file
@@ -0,0 +1,43 @@
|
||||
# Video transform
|
||||
|
||||
A Pipecat example demonstrating how to send and receive audio and video using SmallWebRTCTransport.
|
||||
It also performs some image processing on the video frames using OpenCV.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### First, start the bot server:
|
||||
|
||||
1. Navigate to the server directory:
|
||||
```bash
|
||||
cd server
|
||||
```
|
||||
2. Create and activate a virtual environment:
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # On Windows: venv\Scripts\activate
|
||||
```
|
||||
3. Install requirements:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
4. Copy env.example to .env and configure:
|
||||
- Add your API keys
|
||||
5. Start the server:
|
||||
```bash
|
||||
python server.py
|
||||
```
|
||||
|
||||
### Next, connect using the client app:
|
||||
|
||||
For client-side setup, refer to the [JavaScript Guide](client/typescript/README.md).
|
||||
|
||||
## Important Note
|
||||
|
||||
Ensure the bot server is running before using any client implementations.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.10+
|
||||
- Node.js 16+ (for JavaScript)
|
||||
- Google API key
|
||||
- Modern web browser with WebRTC support
|
||||
27
examples/aiortc/video-transform/client/typescript/README.md
Normal file
27
examples/aiortc/video-transform/client/typescript/README.md
Normal file
@@ -0,0 +1,27 @@
|
||||
# JavaScript Implementation
|
||||
|
||||
Basic implementation using the [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/js/introduction).
|
||||
|
||||
## Setup
|
||||
|
||||
1. Run the bot server. See the [server README](../../README).
|
||||
|
||||
2. Navigate to the `client/typescript` directory:
|
||||
|
||||
```bash
|
||||
cd client/typescript
|
||||
```
|
||||
|
||||
3. Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
4. Run the client app:
|
||||
|
||||
```
|
||||
npm run dev
|
||||
```
|
||||
|
||||
5. Visit http://localhost:5173 in your browser.
|
||||
68
examples/aiortc/video-transform/client/typescript/index.html
Normal file
68
examples/aiortc/video-transform/client/typescript/index.html
Normal file
@@ -0,0 +1,68 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>WebRTC demo</title>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="container">
|
||||
<!-- Settings Bar -->
|
||||
<div class="status-bar">
|
||||
<div class="option">
|
||||
<label>Audio</label>
|
||||
<select id="audio-input">
|
||||
<option value="" selected>Default device</option>
|
||||
</select>
|
||||
<select id="audio-codec">
|
||||
<option value="default" selected>Default codecs</option>
|
||||
<option value="opus/48000/2">Opus</option>
|
||||
<option value="PCMU/8000">PCMU</option>
|
||||
<option value="PCMA/8000">PCMA</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="option">
|
||||
<label>Video</label>
|
||||
<select id="video-input">
|
||||
<option value="" selected>Default device</option>
|
||||
</select>
|
||||
<select id="video-codec">
|
||||
<option value="default" selected>Default codecs</option>
|
||||
<option value="VP8/90000">VP8</option>
|
||||
<option value="H264/90000">H264</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Status Bar -->
|
||||
<div class="status-bar">
|
||||
<div class="status">
|
||||
Status: <span id="connection-status">Disconnected</span>
|
||||
</div>
|
||||
<div class="controls">
|
||||
<button id="connect-btn">Connect</button>
|
||||
<button id="disconnect-btn" disabled>Disconnect</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Main Content -->
|
||||
<div class="main-content">
|
||||
<div class="bot-container">
|
||||
<div id="bot-video-container">
|
||||
<video id="bot-video" autoplay="true" playsinline="true"></video>
|
||||
</div>
|
||||
<audio id="bot-audio" autoplay></audio>
|
||||
</div>
|
||||
<!-- Debug Panel -->
|
||||
<div class="debug-panel">
|
||||
<div id="debug-log"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="module" src="/src/app.ts"></script>
|
||||
<link rel="stylesheet" href="/src/style.css">
|
||||
|
||||
</body>
|
||||
</html>
|
||||
530
examples/aiortc/video-transform/client/typescript/package-lock.json
generated
Normal file
530
examples/aiortc/video-transform/client/typescript/package-lock.json
generated
Normal file
@@ -0,0 +1,530 @@
|
||||
{
|
||||
"name": "client",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "client",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@pipecat-ai/client-js": "^0.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.1",
|
||||
"@vitejs/plugin-react-swc": "^3.7.2",
|
||||
"typescript": "^5.7.3",
|
||||
"vite": "^6.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-arm64": {
|
||||
"version": "0.24.0",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz",
|
||||
"integrity": "sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@pipecat-ai/client-js": {
|
||||
"version": "0.3.2",
|
||||
"resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.2.tgz",
|
||||
"integrity": "sha512-psunOVrJjPka2SWlq53vxVWCA0Vt8pSXsXtn8pOLC0YTKFsUx+b7Z6quYUJcDZjCe1aAg9cKETek3Xal3Co8Tg==",
|
||||
"license": "BSD-2-Clause",
|
||||
"dependencies": {
|
||||
"@types/events": "^3.0.3",
|
||||
"clone-deep": "^4.0.1",
|
||||
"events": "^3.3.0",
|
||||
"typed-emitter": "^2.1.0",
|
||||
"uuid": "^10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@rollup/rollup-darwin-arm64": {
|
||||
"version": "4.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.28.0.tgz",
|
||||
"integrity": "sha512-lmKx9yHsppblnLQZOGxdO66gT77bvdBtr/0P+TPOseowE7D9AJoBw8ZDULRasXRWf1Z86/gcOdpBrV6VDUY36Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@swc/core": {
|
||||
"version": "1.10.14",
|
||||
"resolved": "https://registry.npmjs.org/@swc/core/-/core-1.10.14.tgz",
|
||||
"integrity": "sha512-WSrnE6JRnH20ZYjOOgSS4aOaPv9gxlkI2KRkN24kagbZnPZMnN8bZZyzw1rrLvwgpuRGv17Uz+hflosbR+SP6w==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@swc/counter": "^0.1.3",
|
||||
"@swc/types": "^0.1.17"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/swc"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@swc/core-darwin-arm64": "1.10.14",
|
||||
"@swc/core-darwin-x64": "1.10.14",
|
||||
"@swc/core-linux-arm-gnueabihf": "1.10.14",
|
||||
"@swc/core-linux-arm64-gnu": "1.10.14",
|
||||
"@swc/core-linux-arm64-musl": "1.10.14",
|
||||
"@swc/core-linux-x64-gnu": "1.10.14",
|
||||
"@swc/core-linux-x64-musl": "1.10.14",
|
||||
"@swc/core-win32-arm64-msvc": "1.10.14",
|
||||
"@swc/core-win32-ia32-msvc": "1.10.14",
|
||||
"@swc/core-win32-x64-msvc": "1.10.14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@swc/helpers": "*"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@swc/helpers": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@swc/core-darwin-arm64": {
|
||||
"version": "1.10.14",
|
||||
"resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.10.14.tgz",
|
||||
"integrity": "sha512-Dh4VyrhDDb05tdRmqJ/MucOPMTnrB4pRJol18HVyLlqu1HOT5EzonUniNTCdQbUXjgdv5UVJSTE1lYTzrp+myA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"license": "Apache-2.0 AND MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/@swc/counter": {
|
||||
"version": "0.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz",
|
||||
"integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/@swc/types": {
|
||||
"version": "0.1.17",
|
||||
"resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.17.tgz",
|
||||
"integrity": "sha512-V5gRru+aD8YVyCOMAjMpWR1Ui577DD5KSJsHP8RAxopAH22jFz6GZd/qxqjO6MJHQhcsjvjOFXyDhyLQUnMveQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@swc/counter": "^0.1.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
|
||||
"integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/events": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz",
|
||||
"integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "22.13.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz",
|
||||
"integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~6.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@vitejs/plugin-react-swc": {
|
||||
"version": "3.7.2",
|
||||
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-3.7.2.tgz",
|
||||
"integrity": "sha512-y0byko2b2tSVVf5Gpng1eEhX1OvPC7x8yns1Fx8jDzlJp4LS6CMkCPfLw47cjyoMrshQDoQw4qcgjsU9VvlCew==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@swc/core": "^1.7.26"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"vite": "^4 || ^5 || ^6"
|
||||
}
|
||||
},
|
||||
"node_modules/clone-deep": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
|
||||
"integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"is-plain-object": "^2.0.4",
|
||||
"kind-of": "^6.0.2",
|
||||
"shallow-clone": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.24.0",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.0.tgz",
|
||||
"integrity": "sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"esbuild": "bin/esbuild"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@esbuild/aix-ppc64": "0.24.0",
|
||||
"@esbuild/android-arm": "0.24.0",
|
||||
"@esbuild/android-arm64": "0.24.0",
|
||||
"@esbuild/android-x64": "0.24.0",
|
||||
"@esbuild/darwin-arm64": "0.24.0",
|
||||
"@esbuild/darwin-x64": "0.24.0",
|
||||
"@esbuild/freebsd-arm64": "0.24.0",
|
||||
"@esbuild/freebsd-x64": "0.24.0",
|
||||
"@esbuild/linux-arm": "0.24.0",
|
||||
"@esbuild/linux-arm64": "0.24.0",
|
||||
"@esbuild/linux-ia32": "0.24.0",
|
||||
"@esbuild/linux-loong64": "0.24.0",
|
||||
"@esbuild/linux-mips64el": "0.24.0",
|
||||
"@esbuild/linux-ppc64": "0.24.0",
|
||||
"@esbuild/linux-riscv64": "0.24.0",
|
||||
"@esbuild/linux-s390x": "0.24.0",
|
||||
"@esbuild/linux-x64": "0.24.0",
|
||||
"@esbuild/netbsd-x64": "0.24.0",
|
||||
"@esbuild/openbsd-arm64": "0.24.0",
|
||||
"@esbuild/openbsd-x64": "0.24.0",
|
||||
"@esbuild/sunos-x64": "0.24.0",
|
||||
"@esbuild/win32-arm64": "0.24.0",
|
||||
"@esbuild/win32-ia32": "0.24.0",
|
||||
"@esbuild/win32-x64": "0.24.0"
|
||||
}
|
||||
},
|
||||
"node_modules/events": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
|
||||
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.8.x"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
|
||||
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/is-plain-object": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
|
||||
"integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"isobject": "^3.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/isobject": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
|
||||
"integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/kind-of": {
|
||||
"version": "6.0.3",
|
||||
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/nanoid": {
|
||||
"version": "3.3.8",
|
||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz",
|
||||
"integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/ai"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"nanoid": "bin/nanoid.cjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
"integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/postcss": {
|
||||
"version": "8.4.49",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz",
|
||||
"integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/postcss/"
|
||||
},
|
||||
{
|
||||
"type": "tidelift",
|
||||
"url": "https://tidelift.com/funding/github/npm/postcss"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/ai"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nanoid": "^3.3.7",
|
||||
"picocolors": "^1.1.1",
|
||||
"source-map-js": "^1.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^10 || ^12 || >=14"
|
||||
}
|
||||
},
|
||||
"node_modules/rollup": {
|
||||
"version": "4.28.0",
|
||||
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.28.0.tgz",
|
||||
"integrity": "sha512-G9GOrmgWHBma4YfCcX8PjH0qhXSdH8B4HDE2o4/jaxj93S4DPCIDoLcXz99eWMji4hB29UFCEd7B2gwGJDR9cQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/estree": "1.0.6"
|
||||
},
|
||||
"bin": {
|
||||
"rollup": "dist/bin/rollup"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0",
|
||||
"npm": ">=8.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@rollup/rollup-android-arm-eabi": "4.28.0",
|
||||
"@rollup/rollup-android-arm64": "4.28.0",
|
||||
"@rollup/rollup-darwin-arm64": "4.28.0",
|
||||
"@rollup/rollup-darwin-x64": "4.28.0",
|
||||
"@rollup/rollup-freebsd-arm64": "4.28.0",
|
||||
"@rollup/rollup-freebsd-x64": "4.28.0",
|
||||
"@rollup/rollup-linux-arm-gnueabihf": "4.28.0",
|
||||
"@rollup/rollup-linux-arm-musleabihf": "4.28.0",
|
||||
"@rollup/rollup-linux-arm64-gnu": "4.28.0",
|
||||
"@rollup/rollup-linux-arm64-musl": "4.28.0",
|
||||
"@rollup/rollup-linux-powerpc64le-gnu": "4.28.0",
|
||||
"@rollup/rollup-linux-riscv64-gnu": "4.28.0",
|
||||
"@rollup/rollup-linux-s390x-gnu": "4.28.0",
|
||||
"@rollup/rollup-linux-x64-gnu": "4.28.0",
|
||||
"@rollup/rollup-linux-x64-musl": "4.28.0",
|
||||
"@rollup/rollup-win32-arm64-msvc": "4.28.0",
|
||||
"@rollup/rollup-win32-ia32-msvc": "4.28.0",
|
||||
"@rollup/rollup-win32-x64-msvc": "4.28.0",
|
||||
"fsevents": "~2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/rxjs": {
|
||||
"version": "7.8.1",
|
||||
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz",
|
||||
"integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==",
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/shallow-clone": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz",
|
||||
"integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"kind-of": "^6.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/source-map-js": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz",
|
||||
"integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/typed-emitter": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz",
|
||||
"integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==",
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"rxjs": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.7.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz",
|
||||
"integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.20.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
|
||||
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/uuid": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
|
||||
"integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
|
||||
"funding": [
|
||||
"https://github.com/sponsors/broofa",
|
||||
"https://github.com/sponsors/ctavan"
|
||||
],
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
},
|
||||
"node_modules/vite": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-6.0.2.tgz",
|
||||
"integrity": "sha512-XdQ+VsY2tJpBsKGs0wf3U/+azx8BBpYRHFAyKm5VeEZNOJZRB63q7Sc8Iup3k0TrN3KO6QgyzFf+opSbfY1y0g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"esbuild": "^0.24.0",
|
||||
"postcss": "^8.4.49",
|
||||
"rollup": "^4.23.0"
|
||||
},
|
||||
"bin": {
|
||||
"vite": "bin/vite.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^18.0.0 || ^20.0.0 || >=22.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/vitejs/vite?sponsor=1"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "~2.3.3"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0",
|
||||
"jiti": ">=1.21.0",
|
||||
"less": "*",
|
||||
"lightningcss": "^1.21.0",
|
||||
"sass": "*",
|
||||
"sass-embedded": "*",
|
||||
"stylus": "*",
|
||||
"sugarss": "*",
|
||||
"terser": "^5.16.0",
|
||||
"tsx": "^4.8.1",
|
||||
"yaml": "^2.4.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@types/node": {
|
||||
"optional": true
|
||||
},
|
||||
"jiti": {
|
||||
"optional": true
|
||||
},
|
||||
"less": {
|
||||
"optional": true
|
||||
},
|
||||
"lightningcss": {
|
||||
"optional": true
|
||||
},
|
||||
"sass": {
|
||||
"optional": true
|
||||
},
|
||||
"sass-embedded": {
|
||||
"optional": true
|
||||
},
|
||||
"stylus": {
|
||||
"optional": true
|
||||
},
|
||||
"sugarss": {
|
||||
"optional": true
|
||||
},
|
||||
"terser": {
|
||||
"optional": true
|
||||
},
|
||||
"tsx": {
|
||||
"optional": true
|
||||
},
|
||||
"yaml": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "client",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"dev": "node_modules/.bin/vite",
|
||||
"build": "node_modules/.bin/tsc && vite build",
|
||||
"preview": "node_modules/.bin/vite preview"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.13.1",
|
||||
"@vitejs/plugin-react-swc": "^3.7.2",
|
||||
"typescript": "^5.7.3",
|
||||
"vite": "^6.0.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pipecat-ai/client-js": "^0.3.2"
|
||||
}
|
||||
}
|
||||
145
examples/aiortc/video-transform/client/typescript/src/app.ts
Normal file
145
examples/aiortc/video-transform/client/typescript/src/app.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
import {SmallWebRTCTransport} from "./smallWebRTCTransport";
|
||||
|
||||
class WebRTCApp {
|
||||
|
||||
private declare connectBtn: HTMLButtonElement;
|
||||
private declare disconnectBtn: HTMLButtonElement;
|
||||
|
||||
private declare audioInput: HTMLSelectElement;
|
||||
private declare videoInput: HTMLSelectElement;
|
||||
private declare audioCodec: HTMLSelectElement;
|
||||
private declare videoCodec: HTMLSelectElement;
|
||||
|
||||
private declare videoElement: HTMLVideoElement;
|
||||
private declare audioElement: HTMLAudioElement;
|
||||
|
||||
private debugLog: HTMLElement | null = null;
|
||||
private statusSpan: HTMLElement | null = null;
|
||||
|
||||
private smallWebRTCTransport: SmallWebRTCTransport;
|
||||
|
||||
constructor() {
|
||||
this.setupDOMElements();
|
||||
this.setupDOMEventListeners();
|
||||
|
||||
this.smallWebRTCTransport = new SmallWebRTCTransport({
|
||||
onConnected: () => {
|
||||
this.onConnectedHandler()
|
||||
},
|
||||
onDisconnected: () => {
|
||||
this.onDisconnectedHandler()
|
||||
},
|
||||
onLog: (message: string) => {
|
||||
this.log(message)
|
||||
},
|
||||
onTrackStarted: (track: MediaStreamTrack) => {
|
||||
this.onTrackStarted(track)
|
||||
}
|
||||
});
|
||||
void this.populateDevices();
|
||||
}
|
||||
|
||||
private setupDOMElements(): void {
|
||||
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
|
||||
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
|
||||
|
||||
this.audioInput = document.getElementById('audio-input') as HTMLSelectElement;
|
||||
this.videoInput = document.getElementById('video-input') as HTMLSelectElement;
|
||||
this.audioCodec = document.getElementById('audio-codec') as HTMLSelectElement;
|
||||
this.videoCodec = document.getElementById('video-codec') as HTMLSelectElement;
|
||||
|
||||
this.videoElement = document.getElementById('bot-video') as HTMLVideoElement;
|
||||
this.audioElement = document.getElementById('bot-audio') as HTMLAudioElement;
|
||||
|
||||
this.debugLog = document.getElementById('debug-log');
|
||||
this.statusSpan = document.getElementById('connection-status');
|
||||
}
|
||||
|
||||
private setupDOMEventListeners(): void {
|
||||
this.connectBtn.addEventListener("click", () => this.start());
|
||||
this.disconnectBtn.addEventListener("click", () => this.stop());
|
||||
}
|
||||
|
||||
private log(message: string): void {
|
||||
if (!this.debugLog) return;
|
||||
const entry = document.createElement('div');
|
||||
entry.textContent = `${new Date().toISOString()} - ${message}`;
|
||||
if (message.startsWith('User: ')) {
|
||||
entry.style.color = '#2196F3';
|
||||
} else if (message.startsWith('Bot: ')) {
|
||||
entry.style.color = '#4CAF50';
|
||||
}
|
||||
this.debugLog.appendChild(entry);
|
||||
this.debugLog.scrollTop = this.debugLog.scrollHeight;
|
||||
}
|
||||
|
||||
private clearAllLogs() {
|
||||
this.debugLog!.innerText = ''
|
||||
}
|
||||
|
||||
private updateStatus(status: string): void {
|
||||
if (this.statusSpan) {
|
||||
this.statusSpan.textContent = status;
|
||||
}
|
||||
this.log(`Status: ${status}`);
|
||||
}
|
||||
|
||||
private onConnectedHandler() {
|
||||
this.updateStatus('Connected');
|
||||
if (this.connectBtn) this.connectBtn.disabled = true;
|
||||
if (this.disconnectBtn) this.disconnectBtn.disabled = false;
|
||||
}
|
||||
|
||||
private onDisconnectedHandler() {
|
||||
this.updateStatus('Disconnected');
|
||||
if (this.connectBtn) this.connectBtn.disabled = false;
|
||||
if (this.disconnectBtn) this.disconnectBtn.disabled = true;
|
||||
}
|
||||
|
||||
private onTrackStarted(track: MediaStreamTrack) {
|
||||
if (track.kind === 'video') {
|
||||
this.videoElement.srcObject = new MediaStream([track]);
|
||||
} else {
|
||||
this.audioElement.srcObject = new MediaStream([track]);
|
||||
}
|
||||
}
|
||||
|
||||
private async populateDevices(): Promise<void> {
|
||||
const populateSelect = (select: HTMLSelectElement, devices: MediaDeviceInfo[]): void => {
|
||||
let counter = 1;
|
||||
devices.forEach((device) => {
|
||||
const option = document.createElement('option');
|
||||
option.value = device.deviceId;
|
||||
option.text = device.label || ('Device #' + counter);
|
||||
select.appendChild(option);
|
||||
counter += 1;
|
||||
});
|
||||
};
|
||||
|
||||
try {
|
||||
const audioDevices = await this.smallWebRTCTransport.getAllMics();
|
||||
populateSelect(this.audioInput, audioDevices);
|
||||
const videoDevices = await this.smallWebRTCTransport.getAllCams();
|
||||
populateSelect(this.videoInput, videoDevices);
|
||||
} catch (e) {
|
||||
alert(e);
|
||||
}
|
||||
}
|
||||
|
||||
private async start(): Promise<void> {
|
||||
this.clearAllLogs()
|
||||
|
||||
const audioDevice = this.audioInput.value;
|
||||
const audioCodec = this.audioCodec.value;
|
||||
const videoDevice = this.videoInput.value;
|
||||
const videoCodec = this.videoCodec.value;
|
||||
await this.smallWebRTCTransport.start(audioDevice, audioCodec, videoCodec, videoDevice)
|
||||
}
|
||||
|
||||
private stop(): void {
|
||||
this.smallWebRTCTransport.stop()
|
||||
}
|
||||
}
|
||||
|
||||
// Create the WebRTCConnection instance
|
||||
const webRTCConnection = new WebRTCApp();
|
||||
@@ -0,0 +1,372 @@
|
||||
// TODO: we should refactor everything that is inside this class,
|
||||
// and create a Pipecat client transport here
|
||||
// https://github.com/pipecat-ai/pipecat-client-web-transports
|
||||
|
||||
const SIGNALLING_TYPE = "signalling";
|
||||
|
||||
enum SignallingMessage {
|
||||
RENEGOTIATE = "renegotiate",
|
||||
}
|
||||
|
||||
// Interface for the structure of the signalling message
|
||||
interface SignallingMessageObject {
|
||||
type: string;
|
||||
message: SignallingMessage;
|
||||
}
|
||||
|
||||
export type SmallWebRTCTransportCallbacks = {
|
||||
onLog(message: string): void;
|
||||
onTrackStarted(track: MediaStreamTrack): void;
|
||||
onConnected(): void;
|
||||
onDisconnected(): void;
|
||||
}
|
||||
|
||||
export class SmallWebRTCTransport {
|
||||
|
||||
private _callbacks: SmallWebRTCTransportCallbacks;
|
||||
|
||||
private pc: RTCPeerConnection | null = null;
|
||||
private dc: RTCDataChannel | null = null;
|
||||
|
||||
constructor(callbacks: SmallWebRTCTransportCallbacks) {
|
||||
this._callbacks = callbacks
|
||||
}
|
||||
|
||||
private log(message: string): void {
|
||||
console.log(message);
|
||||
this._callbacks.onLog(message)
|
||||
}
|
||||
|
||||
private createPeerConnection(): RTCPeerConnection {
|
||||
const config: RTCConfiguration = {
|
||||
//iceServers: [{ urls: ['stun:stun.l.google.com:19302'] }]
|
||||
};
|
||||
|
||||
let pc = new RTCPeerConnection(config);
|
||||
|
||||
pc.addEventListener('icegatheringstatechange', () => {
|
||||
this.log(`iceGatheringState: ${this.pc!.iceGatheringState}`)
|
||||
});
|
||||
this.log(`iceGatheringState: ${pc.iceGatheringState}`)
|
||||
|
||||
pc.addEventListener('iceconnectionstatechange', () => {
|
||||
let connectionState = this.pc!.iceConnectionState
|
||||
this.log(`iceConnectionState: ${connectionState}`)
|
||||
});
|
||||
this.log(`iceConnectionState: ${pc.iceConnectionState}`)
|
||||
|
||||
pc.addEventListener('signalingstatechange', () => {
|
||||
this.log(`signalingState: ${this.pc!.signalingState}`)
|
||||
});
|
||||
this.log(`signalingState: ${pc.signalingState}`)
|
||||
|
||||
pc.addEventListener('track', (evt: RTCTrackEvent) => {
|
||||
this.log(`Received new track ${evt.track.kind}`)
|
||||
this._callbacks.onTrackStarted(evt.track)
|
||||
});
|
||||
|
||||
pc.onconnectionstatechange = () => {
|
||||
let connectionState = this.pc?.connectionState
|
||||
this.log(`connectionState: ${connectionState}`)
|
||||
if (connectionState == 'connected') {
|
||||
this._callbacks.onConnected()
|
||||
} else if (connectionState == 'disconnected') {
|
||||
this._callbacks.onDisconnected()
|
||||
}
|
||||
}
|
||||
|
||||
return pc;
|
||||
}
|
||||
|
||||
private async negotiate(audioCodec: string, videoCodec: string): Promise<void> {
|
||||
if (!this.pc) {
|
||||
return Promise.reject('Peer connection is not initialized');
|
||||
}
|
||||
|
||||
try {
|
||||
// Create offer
|
||||
const offer = await this.pc.createOffer();
|
||||
await this.pc.setLocalDescription(offer);
|
||||
|
||||
// Wait for ICE gathering to complete
|
||||
await new Promise<void>((resolve) => {
|
||||
if (this.pc!.iceGatheringState === 'complete') {
|
||||
resolve();
|
||||
} else {
|
||||
const checkState = () => {
|
||||
if (this.pc!.iceGatheringState === 'complete') {
|
||||
this.pc!.removeEventListener('icegatheringstatechange', checkState);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
this.pc!.addEventListener('icegatheringstatechange', checkState);
|
||||
}
|
||||
});
|
||||
|
||||
let offerSdp = this.pc!.localDescription!;
|
||||
let codec: string;
|
||||
|
||||
// Filter audio codec
|
||||
if (audioCodec !== 'default') {
|
||||
// @ts-ignore
|
||||
offerSdp.sdp = this.sdpFilterCodec('audio', audioCodec, offerSdp.sdp);
|
||||
}
|
||||
|
||||
// Filter video codec
|
||||
if (videoCodec !== 'default') {
|
||||
// @ts-ignore
|
||||
offerSdp.sdp = this.sdpFilterCodec('video', videoCodec, offerSdp.sdp);
|
||||
}
|
||||
|
||||
// Send offer to server
|
||||
const response = await fetch('/api/offer', {
|
||||
body: JSON.stringify({
|
||||
sdp: offerSdp.sdp,
|
||||
type: offerSdp.type,
|
||||
}),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
method: 'POST',
|
||||
});
|
||||
|
||||
const answer: RTCSessionDescriptionInit = await response.json();
|
||||
// @ts-ignore
|
||||
this.log(`Received answer for peer connection id ${answer.pc_id}`)
|
||||
await this.pc!.setRemoteDescription(answer);
|
||||
} catch (e) {
|
||||
alert(e);
|
||||
}
|
||||
}
|
||||
|
||||
private addInitialTransceivers() {
|
||||
// Transceivers always appear in creation-order for both peers
|
||||
// For now we are only considering that we are going to have 02 transceivers,
|
||||
// one for audio and one for video
|
||||
this.pc!.addTransceiver('audio', { direction: 'sendrecv' });
|
||||
this.pc!.addTransceiver('video', { direction: 'sendrecv' });
|
||||
}
|
||||
|
||||
private getAudioTransceiver() {
|
||||
// Transceivers always appear in creation-order for both peers
|
||||
// Look at addInitialTransceivers
|
||||
return this.pc!.getTransceivers()[0];
|
||||
}
|
||||
|
||||
private getVideoTransceiver() {
|
||||
// Transceivers always appear in creation-order for both peers
|
||||
// Look at addInitialTransceivers
|
||||
return this.pc!.getTransceivers()[1];
|
||||
}
|
||||
|
||||
async start(audioDevice: string | undefined, audioCodec: string, videoCodec: string, videoDevice: string | undefined): Promise<void> {
|
||||
this.pc = this.createPeerConnection();
|
||||
this.addInitialTransceivers();
|
||||
this.dc = this.createDataChannel('chat', { ordered: true });
|
||||
await this.addUserMedias(audioDevice, videoDevice);
|
||||
await this.negotiate(audioCodec, videoCodec);
|
||||
}
|
||||
|
||||
private async addUserMedias(audioDevice: string|undefined, videoDevice:string|undefined): Promise<void> {
|
||||
this.log("Will send the audio and video");
|
||||
const constraints = this.createMediaConstraints(audioDevice, videoDevice);
|
||||
|
||||
if (constraints.audio || constraints.video) {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
|
||||
let videoTrack = stream.getVideoTracks()[0]
|
||||
await this.getVideoTransceiver().sender.replaceTrack(videoTrack);
|
||||
|
||||
let audioTrack = stream.getAudioTracks()[0]
|
||||
await this.getAudioTransceiver().sender.replaceTrack(audioTrack);
|
||||
} catch (err) {
|
||||
alert(`Could not acquire media: ${err}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Method to handle a general message (this can be expanded for other types of messages)
|
||||
handleMessage(message: string): void {
|
||||
try {
|
||||
this.log(message)
|
||||
|
||||
const messageObj = JSON.parse(message); // Type is `any` initially
|
||||
|
||||
// Check if it's a signalling message
|
||||
if (messageObj.type === SIGNALLING_TYPE) {
|
||||
this.handleSignallingMessage(messageObj as SignallingMessageObject); // Delegate to handleSignallingMessage
|
||||
} else {
|
||||
// implement to handle the other messages in the future
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to parse JSON message:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Method to handle signalling messages specifically
|
||||
handleSignallingMessage(messageObj: SignallingMessageObject): void {
|
||||
// Cast the object to the correct type after verification
|
||||
const signallingMessage = messageObj as SignallingMessageObject;
|
||||
|
||||
// Handle different signalling message types
|
||||
switch (signallingMessage.message) {
|
||||
case SignallingMessage.RENEGOTIATE:
|
||||
this.log("Handling renegotiation...");
|
||||
// TODO: implement it
|
||||
break;
|
||||
|
||||
default:
|
||||
console.warn("Unknown signalling message:", signallingMessage.message);
|
||||
}
|
||||
}
|
||||
|
||||
private createDataChannel(label: string, options: RTCDataChannelInit): RTCDataChannel {
|
||||
const dc = this.pc!.createDataChannel(label, options);
|
||||
let timeStart: number | null = null;
|
||||
|
||||
const getCurrentTimestamp = (): number => {
|
||||
if (timeStart === null) {
|
||||
timeStart = Date.now();
|
||||
return 0;
|
||||
}
|
||||
return Date.now() - timeStart;
|
||||
};
|
||||
|
||||
dc.addEventListener('close', () => {
|
||||
this.log("datachannel closed")
|
||||
});
|
||||
|
||||
dc.addEventListener('open', () => {
|
||||
this.log("datachannel opened")
|
||||
// Sending message that the client is ready, just for testing
|
||||
dc.send(JSON.stringify({id: 'clientReady', label: 'rtvi-ai', type:'client-ready'}))
|
||||
});
|
||||
|
||||
dc.addEventListener('message', (evt: MessageEvent) => {
|
||||
let message = evt.data
|
||||
this.handleMessage(message)
|
||||
});
|
||||
|
||||
return dc;
|
||||
}
|
||||
|
||||
private createMediaConstraints(audioDevice: string|undefined, videoDevice:string|undefined): MediaStreamConstraints {
|
||||
const constraints: MediaStreamConstraints = { audio: false, video: false };
|
||||
|
||||
const audioConstraints: MediaTrackConstraints = {};
|
||||
if (audioDevice) audioConstraints.deviceId = { exact: audioDevice };
|
||||
|
||||
constraints.audio = Object.keys(audioConstraints).length ? audioConstraints : true;
|
||||
|
||||
const videoConstraints: MediaTrackConstraints = {};
|
||||
if (videoDevice) videoConstraints.deviceId = { exact: videoDevice };
|
||||
|
||||
constraints.video = Object.keys(videoConstraints).length ? videoConstraints : true;
|
||||
|
||||
return constraints;
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.pc) {
|
||||
this.log("Peer connection is already closed or null.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.dc) {
|
||||
this.dc.close();
|
||||
}
|
||||
|
||||
this.pc.getTransceivers().forEach((transceiver) => {
|
||||
if (transceiver.stop) {
|
||||
transceiver.stop();
|
||||
}
|
||||
});
|
||||
|
||||
this.pc.getSenders().forEach((sender) => {
|
||||
sender.track?.stop();
|
||||
});
|
||||
|
||||
this.pc.close();
|
||||
|
||||
// For some reason after we close the peer connection, it is not triggering the listeners
|
||||
this._callbacks.onDisconnected()
|
||||
}
|
||||
|
||||
private async getAllDevices() {
|
||||
return await navigator.mediaDevices.enumerateDevices();
|
||||
}
|
||||
|
||||
async getAllCams() {
|
||||
const devices = await this.getAllDevices();
|
||||
return devices.filter((d) => d.kind === "videoinput");
|
||||
}
|
||||
|
||||
async getAllMics() {
|
||||
const devices = await this.getAllDevices();
|
||||
return devices.filter((d) => d.kind === "audioinput");
|
||||
}
|
||||
|
||||
private sdpFilterCodec(kind: string, codec: string, realSdp: string): string {
|
||||
const allowed: number[] = [];
|
||||
const rtxRegex = new RegExp('a=fmtp:(\\d+) apt=(\\d+)\\r$');
|
||||
const codecRegex = new RegExp('a=rtpmap:([0-9]+) ' + this.escapeRegExp(codec));
|
||||
const videoRegex = new RegExp('(m=' + kind + ' .*?)( ([0-9]+))*\\s*$');
|
||||
|
||||
const lines = realSdp.split('\n');
|
||||
|
||||
let isKind = false;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('m=' + kind + ' ')) {
|
||||
isKind = true;
|
||||
} else if (lines[i].startsWith('m=')) {
|
||||
isKind = false;
|
||||
}
|
||||
|
||||
if (isKind) {
|
||||
const match = lines[i].match(codecRegex);
|
||||
if (match) {
|
||||
allowed.push(parseInt(match[1]));
|
||||
}
|
||||
|
||||
const matchRtx = lines[i].match(rtxRegex);
|
||||
if (matchRtx && allowed.includes(parseInt(matchRtx[2]))) {
|
||||
allowed.push(parseInt(matchRtx[1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const skipRegex = 'a=(fmtp|rtcp-fb|rtpmap):([0-9]+)';
|
||||
let sdp = '';
|
||||
|
||||
isKind = false;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].startsWith('m=' + kind + ' ')) {
|
||||
isKind = true;
|
||||
} else if (lines[i].startsWith('m=')) {
|
||||
isKind = false;
|
||||
}
|
||||
|
||||
if (isKind) {
|
||||
const skipMatch = lines[i].match(skipRegex);
|
||||
if (skipMatch && !allowed.includes(parseInt(skipMatch[2]))) {
|
||||
continue;
|
||||
} else if (lines[i].match(videoRegex)) {
|
||||
sdp += lines[i].replace(videoRegex, '$1 ' + allowed.join(' ')) + '\n';
|
||||
} else {
|
||||
sdp += lines[i] + '\n';
|
||||
}
|
||||
} else {
|
||||
sdp += lines[i] + '\n';
|
||||
}
|
||||
}
|
||||
|
||||
return sdp;
|
||||
}
|
||||
|
||||
private escapeRegExp(string: string): string {
|
||||
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
120
examples/aiortc/video-transform/client/typescript/src/style.css
Normal file
120
examples/aiortc/video-transform/client/typescript/src/style.css
Normal file
@@ -0,0 +1,120 @@
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f0f0f0;
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.container {
|
||||
margin: 0 auto;
|
||||
width: 90%;
|
||||
}
|
||||
|
||||
.option {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
label {
|
||||
margin: 5px;
|
||||
}
|
||||
|
||||
select {
|
||||
padding: 8px;
|
||||
margin: 10px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #ccc;
|
||||
}
|
||||
|
||||
.status-bar {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 10px;
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.controls button {
|
||||
padding: 8px 16px;
|
||||
margin-left: 10px;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
#connect-btn {
|
||||
background-color: #4caf50;
|
||||
color: white;
|
||||
}
|
||||
|
||||
#disconnect-btn {
|
||||
background-color: #f44336;
|
||||
color: white;
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.main-content {
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 20px;
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.bot-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
width: 50%;
|
||||
}
|
||||
|
||||
#bot-video-container {
|
||||
width: 640px;
|
||||
height: 360px;
|
||||
background-color: #e0e0e0;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
#bot-video-container video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.debug-panel {
|
||||
background-color: #fff;
|
||||
border-radius: 8px;
|
||||
padding-left: 20px;
|
||||
width: 50%;
|
||||
}
|
||||
|
||||
.debug-panel h3 {
|
||||
margin: 0 0 10px 0;
|
||||
font-size: 16px;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
#debug-log {
|
||||
height: 500px;
|
||||
overflow-y: auto;
|
||||
background-color: #f8f8f8;
|
||||
padding: 10px;
|
||||
border-radius: 4px;
|
||||
font-family: monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
111
examples/aiortc/video-transform/client/typescript/tsconfig.json
Normal file
111
examples/aiortc/video-transform/client/typescript/tsconfig.json
Normal file
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
/* Visit https://aka.ms/tsconfig to read more about this file */
|
||||
|
||||
/* Projects */
|
||||
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
|
||||
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
|
||||
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
|
||||
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
|
||||
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
|
||||
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
|
||||
|
||||
/* Language and Environment */
|
||||
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
|
||||
// "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
|
||||
// "jsx": "preserve", /* Specify what JSX code is generated. */
|
||||
// "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
|
||||
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
|
||||
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
|
||||
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
|
||||
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
|
||||
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
|
||||
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
|
||||
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
|
||||
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
|
||||
|
||||
/* Modules */
|
||||
"module": "commonjs", /* Specify what module code is generated. */
|
||||
// "rootDir": "./", /* Specify the root folder within your source files. */
|
||||
// "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
|
||||
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
|
||||
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
|
||||
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
|
||||
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
|
||||
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
|
||||
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
|
||||
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
|
||||
// "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
|
||||
// "rewriteRelativeImportExtensions": true, /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */
|
||||
// "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
|
||||
// "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
|
||||
// "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
|
||||
// "noUncheckedSideEffectImports": true, /* Check side effect imports. */
|
||||
// "resolveJsonModule": true, /* Enable importing .json files. */
|
||||
// "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
|
||||
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
|
||||
|
||||
/* JavaScript Support */
|
||||
// "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
|
||||
// "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
|
||||
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
|
||||
|
||||
/* Emit */
|
||||
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
|
||||
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
|
||||
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
|
||||
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
|
||||
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
|
||||
// "noEmit": true, /* Disable emitting files from a compilation. */
|
||||
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
|
||||
// "outDir": "./", /* Specify an output folder for all emitted files. */
|
||||
// "removeComments": true, /* Disable emitting comments. */
|
||||
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
|
||||
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
|
||||
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
|
||||
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
|
||||
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
|
||||
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
|
||||
// "newLine": "crlf", /* Set the newline character for emitting files. */
|
||||
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
|
||||
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
|
||||
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
|
||||
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
|
||||
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
|
||||
|
||||
/* Interop Constraints */
|
||||
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
|
||||
// "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
|
||||
// "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
|
||||
// "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
|
||||
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
|
||||
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
|
||||
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
|
||||
|
||||
/* Type Checking */
|
||||
"strict": true, /* Enable all strict type-checking options. */
|
||||
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
|
||||
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
|
||||
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
|
||||
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
|
||||
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
|
||||
// "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
|
||||
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
|
||||
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
|
||||
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
|
||||
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
|
||||
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
|
||||
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
|
||||
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
|
||||
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
|
||||
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
|
||||
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
|
||||
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
|
||||
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
|
||||
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
|
||||
|
||||
/* Completeness */
|
||||
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
|
||||
"skipLibCheck": true /* Skip type checking all .d.ts files. */
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { defineConfig } from 'vite';
|
||||
import react from '@vitejs/plugin-react-swc';
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
proxy: {
|
||||
// Proxy /api requests to the backend server
|
||||
'/api': {
|
||||
target: 'http://0.0.0.0:7860', // Replace with your backend URL
|
||||
changeOrigin: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
155
examples/aiortc/video-transform/server/aiortc_bot.py
Normal file
155
examples/aiortc/video-transform/server/aiortc_bot.py
Normal file
@@ -0,0 +1,155 @@
|
||||
#
|
||||
# Copyright (c) 2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
||||
from pipecat.frames.frames import Frame, InputImageRawFrame, OutputImageRawFrame
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineParams, PipelineTask
|
||||
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIObserver, RTVIProcessor
|
||||
from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService
|
||||
from pipecat.transports.base_transport import TransportParams
|
||||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
logger.remove(0)
|
||||
logger.add(sys.stderr, level="DEBUG")
|
||||
|
||||
|
||||
class EdgeDetectionProcessor(FrameProcessor):
|
||||
def __init__(self, camera_out_width, camera_out_height: int):
|
||||
super().__init__()
|
||||
self._camera_out_width = camera_out_width
|
||||
self._camera_out_height = camera_out_height
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, InputImageRawFrame):
|
||||
# Convert bytes to NumPy array
|
||||
img = np.frombuffer(frame.image, dtype=np.uint8).reshape(
|
||||
(frame.size[1], frame.size[0], 3)
|
||||
)
|
||||
|
||||
# perform edge detection
|
||||
img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
|
||||
|
||||
# convert the size if needed
|
||||
desired_size = (self._camera_out_width, self._camera_out_height)
|
||||
if frame.size != desired_size:
|
||||
resized_image = cv2.resize(img, desired_size)
|
||||
frame = OutputImageRawFrame(resized_image.tobytes(), desired_size, frame.format)
|
||||
await self.push_frame(frame)
|
||||
else:
|
||||
await self.push_frame(
|
||||
OutputImageRawFrame(image=img.tobytes(), size=frame.size, format=frame.format)
|
||||
)
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
SYSTEM_INSTRUCTION = f"""
|
||||
"You are Gemini Chatbot, a friendly, helpful robot.
|
||||
|
||||
Your goal is to demonstrate your capabilities in a succinct way.
|
||||
|
||||
Your output will be converted to audio so don't include special characters in your answers.
|
||||
|
||||
Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most.
|
||||
"""
|
||||
|
||||
|
||||
async def run_bot(webrtc_connection):
|
||||
transport_params = TransportParams(
|
||||
camera_in_enabled=True,
|
||||
camera_out_enabled=True,
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
vad_enabled=True,
|
||||
vad_analyzer=SileroVADAnalyzer(),
|
||||
vad_audio_passthrough=True,
|
||||
)
|
||||
|
||||
pipecat_transport = SmallWebRTCTransport(
|
||||
webrtc_connection=webrtc_connection, params=transport_params
|
||||
)
|
||||
|
||||
llm = GeminiMultimodalLiveLLMService(
|
||||
api_key=os.getenv("GOOGLE_API_KEY"),
|
||||
voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck
|
||||
transcribe_user_audio=True,
|
||||
transcribe_model_audio=True,
|
||||
system_instruction=SYSTEM_INSTRUCTION,
|
||||
)
|
||||
|
||||
context = OpenAILLMContext(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Start by greeting the user warmly and introducing yourself.",
|
||||
}
|
||||
],
|
||||
)
|
||||
context_aggregator = llm.create_context_aggregator(context)
|
||||
|
||||
# RTVI events for Pipecat client UI
|
||||
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
pipecat_transport.input(),
|
||||
context_aggregator.user(),
|
||||
rtvi,
|
||||
llm, # LLM
|
||||
EdgeDetectionProcessor(
|
||||
transport_params.camera_out_width, transport_params.camera_out_height
|
||||
), # Sending the video back to the user
|
||||
pipecat_transport.output(),
|
||||
context_aggregator.assistant(),
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
params=PipelineParams(
|
||||
allow_interruptions=True,
|
||||
observers=[RTVIObserver(rtvi)],
|
||||
),
|
||||
)
|
||||
|
||||
@rtvi.event_handler("on_client_ready")
|
||||
async def on_client_ready(rtvi):
|
||||
logger.info("Pipecat client ready.")
|
||||
await rtvi.set_bot_ready()
|
||||
|
||||
@pipecat_transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(transport, client):
|
||||
logger.info("Pipecat Client connected")
|
||||
# Kick off the conversation.
|
||||
await task.queue_frames([context_aggregator.user().get_context_frame()])
|
||||
|
||||
@pipecat_transport.event_handler("on_client_disconnected")
|
||||
async def on_client_disconnected(transport, client):
|
||||
logger.info("Pipecat Client disconnected")
|
||||
|
||||
@pipecat_transport.event_handler("on_client_closed")
|
||||
async def on_client_closed(transport, client):
|
||||
logger.info("Pipecat Client closed")
|
||||
await task.cancel()
|
||||
|
||||
runner = PipelineRunner(handle_sigint=False)
|
||||
|
||||
await runner.run(task)
|
||||
1
examples/aiortc/video-transform/server/env.example
Normal file
1
examples/aiortc/video-transform/server/env.example
Normal file
@@ -0,0 +1 @@
|
||||
GOOGLE_API_KEY=
|
||||
6
examples/aiortc/video-transform/server/requirements.txt
Normal file
6
examples/aiortc/video-transform/server/requirements.txt
Normal file
@@ -0,0 +1,6 @@
|
||||
python-dotenv
|
||||
fastapi[all]
|
||||
uvicorn
|
||||
aiortc
|
||||
opencv-python
|
||||
pipecat-ai[google,silero]
|
||||
63
examples/aiortc/video-transform/server/server.py
Normal file
63
examples/aiortc/video-transform/server/server.py
Normal file
@@ -0,0 +1,63 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import uvicorn
|
||||
from aiortc_bot import run_bot
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import BackgroundTasks, FastAPI
|
||||
|
||||
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||
|
||||
# Load environment variables
|
||||
load_dotenv(override=True)
|
||||
|
||||
logger = logging.getLogger("pc")
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
pcs = set()
|
||||
|
||||
|
||||
@app.post("/api/offer")
|
||||
async def offer(request: dict, background_tasks: BackgroundTasks):
|
||||
pipecat_connection = SmallWebRTCConnection()
|
||||
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
|
||||
|
||||
pcs.add(pipecat_connection)
|
||||
|
||||
@pipecat_connection.on("closed")
|
||||
async def handle_disconnected():
|
||||
logger.info("Discarding the peer connection.")
|
||||
pcs.discard(pipecat_connection)
|
||||
|
||||
background_tasks.add_task(run_bot, pipecat_connection)
|
||||
|
||||
return pipecat_connection.get_answer()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
yield # Run app
|
||||
coros = [pc.close() for pc in pcs]
|
||||
await asyncio.gather(*coros)
|
||||
pcs.clear()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="WebRTC demo")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Host for HTTP server (default: 0.0.0.0)")
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=7860, help="Port for HTTP server (default: 7860)"
|
||||
)
|
||||
parser.add_argument("--verbose", "-v", action="count")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
uvicorn.run(app, host=args.host, port=args.port)
|
||||
Reference in New Issue
Block a user