Compare commits

...

30 Commits

Author SHA1 Message Date
vipyne
115c45b5fc mcp_run POC 2025-04-10 15:21:29 -05:00
vipyne
1908e7cd97 WIP getting mcp.run to work 2025-04-10 13:51:00 -05:00
Aleix Conchillo Flaqué
5dbb5f176b Merge pull request #1551 from pipecat-ai/aleix/daily-python-0.17.0
pyproject: update daily-python to 0.17.0
2025-04-10 09:06:55 -07:00
Filipi da Silva Fuchter
b89f2611f7 Merge pull request #1539 from pipecat-ai/small_wbertc_mute_state
SmallWebRTC mute state
2025-04-10 11:26:53 -03:00
Filipi Fuchter
db0f783c55 Updating the video-transform demo to use the latest version of the SmallWebRTCTransport. 2025-04-10 11:23:28 -03:00
Filipi Fuchter
20ec323647 Refactoring the video-transform demo to be able to enable or disable the cam. 2025-04-10 11:23:05 -03:00
Filipi Fuchter
f71c09a4fd Added support in SmallWebRTCTransport to detect when remote tracks are muted. 2025-04-10 11:22:37 -03:00
Mark Backman
cba4ebfcf9 Merge pull request #1555 from pipecat-ai/mb/gemini-beta-base 2025-04-10 09:01:16 -04:00
Mark Backman
3b9a8946f9 Update GeminiMultimodalLiveLLMService base_url 2025-04-10 08:17:52 -04:00
Mark Backman
db3620c4be Merge pull request #1553 from balaji-atoa/main
feat: change default model name on live api
2025-04-10 08:10:35 -04:00
Mark Backman
11338ea92d Merge pull request #1552 from pipecat-ai/mb/p2p-capture-image
Add image capture to SmallWebRTCTransport
2025-04-10 07:52:13 -04:00
Filipi da Silva Fuchter
90563a4091 Merge pull request #1542 from pipecat-ai/small_webrtc_prebuilt_ui
Using the small-webrtc-prebuilt-ui
2025-04-10 07:39:26 -03:00
Filipi da Silva Fuchter
937f5f7cb7 Update examples/p2p-webrtc/video-transform/server/requirements.txt
Co-authored-by: Mark Backman <mark@daily.co>
2025-04-10 07:37:23 -03:00
Filipi da Silva Fuchter
4f221b817a Update examples/p2p-webrtc/video-transform/README.md
Co-authored-by: Mark Backman <mark@daily.co>
2025-04-10 07:37:07 -03:00
balaji-atoa
c79c1f65fc feat: change default model name on live api 2025-04-10 11:59:11 +05:30
Mark Backman
8ad2ad0e59 Add image capture to SmallWebRTCTransport 2025-04-09 23:01:06 -04:00
Aleix Conchillo Flaqué
499b258bf9 pyproject: update daily-python to 0.17.0 2025-04-09 18:59:10 -07:00
Filipi Fuchter
05b6a5ae4b Improving the video-transform readme 2025-04-09 15:55:13 -03:00
Filipi Fuchter
65fcea28ce Using the small-webrtc-prebuilt-ui 2025-04-09 15:45:30 -03:00
Kwindla Hultman Kramer
005c0b55b6 Merge pull request #1545 from pipecat-ai/khk/gem-live-0408
Gemini Multimodal Live API base_url format fix
2025-04-08 21:46:30 -07:00
Kwindla Hultman Kramer
1828127f41 small fix to wss base_url 2025-04-08 20:22:26 -07:00
Filipi da Silva Fuchter
77ab841cab Merge pull request #1532 from pipecat-ai/p2p_ios_demo
iOS demo for the p2p-webrtc video-transform example
2025-04-07 16:58:06 -03:00
Filipi Fuchter
3bbc75110a Mentioning the iOS client inside the changelog and fixing the readme. 2025-04-07 16:54:26 -03:00
Filipi Fuchter
b2ce1d9378 Merge branch 'main' into p2p_ios_demo 2025-04-07 16:50:58 -03:00
Filipi Fuchter
58714865df Using the public version of pipecat-client-ios-small-webrtc 2025-04-07 16:48:18 -03:00
Mark Backman
03b3635b0a Merge pull request #1521 from pipecat-ai/mb/increase-bot-vad-stop-secs
Increase BOT_VAD_STOP_SECS for services with slower speech patterns
2025-04-07 14:44:31 -04:00
Mark Backman
aaa7b5e626 Merge pull request #1524 from pipecat-ai/mb/tts-generate-with-text
TTS: Skip generation when there is no text
2025-04-07 14:44:18 -04:00
Filipi Fuchter
a1578bd67a iOS demo for the p2p-webrtc video-transform example 2025-04-04 16:40:52 -03:00
Mark Backman
63146d6f85 TTS: Skip generation when there is no text 2025-04-03 16:15:58 -04:00
Mark Backman
a21be058e2 Increase BOT_VAD_STOP_SECS for services with slower speech patterns 2025-04-03 15:25:48 -04:00
52 changed files with 2399 additions and 72 deletions

View File

@@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Added support in `SmallWebRTCTransport` to detect when remote tracks are
muted.
- Added support for image capture from a video stream to the
`SmallWebRTCTransport`.
- Added a new iOS client option to the `SmallWebRTCTransport`
**video-transform** example.
- Added new processors `ProducerProcessor` and `ConsumerProcessor`. The
producer processor processes frames from the pipeline and decides whether the
consumers should consume it or not. If so, the same frame that is received by
@@ -23,8 +32,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
type was incorrectly handled as a codec retransmission.
- Avoid initial video delays.
### Changed
- Updated `GeminiMultimodalLiveLLMService`s default `model` to
`models/gemini-2.0-flash-live-001` and `base_url` to the `v1beta` websocket
URL.
### Fixed
- Updated `daily-python` to 0.17.0 to fix an issue that was preventing to run on
older platforms.
- Fixed an issue in the Azure TTS services where the language was being set
incorrectly.

View File

@@ -0,0 +1,110 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import aiohttp
import asyncio
import os
import sys
from dotenv import load_dotenv
from loguru import logger
from runner import configure
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
from pipecat.services.cartesia.tts import CartesiaTTSService
from pipecat.transports.services.daily import DailyParams, DailyTransport
from pipecat.services.mcp_run.mcp_run import MCPRun
from pipecat.services.anthropic.llm import AnthropicLLMService
from pipecat.services.google.llm import GoogleLLMService
from pipecat.services.openai.llm import OpenAILLMService
load_dotenv(override=True)
logger.remove()
logger.add(sys.stderr, level="DEBUG")
async def main():
async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session)
transport = DailyTransport(
room_url,
token,
"Bot with MCP tools",
DailyParams(
audio_out_enabled=True,
transcription_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
),
)
tts = CartesiaTTSService(
api_key=os.getenv("CARTESIA_API_KEY"),
voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady
)
llm = AnthropicLLMService(api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest")
# llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001")
# llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o")
mcp_run = MCPRun(llm)
tools = mcp_run.register_mcp_tools(llm)
system = """
You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities
in a succinct way. You have access to various tools provided by mcp.run that you can use to help users.
Your output will be converted to audio so don't include special characters in your answers.
Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing.
Just respond with short sentences when you are carrying out tool calls.
"""
messages = [{"role": "system","content": system}]
context = OpenAILLMContext(messages, tools)
context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline(
[
transport.input(), # Transport user input
context_aggregator.user(), # User spoken responses
llm, # LLM
tts, # TTS
transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses and tool context
]
)
task = PipelineTask(
pipeline,
params=PipelineParams(
allow_interruptions=True,
enable_metrics=True,
),
)
@transport.event_handler("on_first_participant_joined")
async def on_first_participant_joined(transport, participant):
logger.info("First participant joined: {}", participant["id"])
await transport.capture_participant_transcription(participant["id"])
# Kick off the conversation.
await task.queue_frames([context_aggregator.user().get_context_frame()])
runner = PipelineRunner()
await runner.run(task)
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -35,9 +35,19 @@ cd server
python server.py
```
### 2Connect Using the Client App
### 2Test with SmallWebRTC Prebuilt UI
For client-side setup, refer to the [JavaScript Guide](client/typescript/README.md).
You can quickly test your bot using the `SmallWebRTCPrebuiltUI`:
- Open your browser and navigate to:
👉 http://localhost:7860
- (Or use your custom port, if configured)
### 3⃣ Connect Using a Custom Client App
For client-side setup, refer to the:
- [Typescript Guide](client/typescript/README.md).
- [iOS Guide](client/ios/README.md).
## ⚠️ Important Note
Ensure the bot server is running before using any client implementations.

View File

@@ -0,0 +1,2 @@
/SimpleChatbot.xcodeproj/xcuserdata/
/SimpleChatbot.xcodeproj/project.xcworkspace/xcuserdata/

View File

@@ -0,0 +1,18 @@
# iOS implementation
Basic implementation using the [Pipecat iOS SDK](https://docs.pipecat.ai/client/ios/introduction).
## Prerequisites
1. Run the bot server. See the [server README](../../server).
2. Install [Xcode](https://developer.apple.com/xcode/), and set up your device [to run your own applications](https://developer.apple.com/documentation/xcode/distributing-your-app-to-registered-devices).
## Running locally
1. Clone this repository locally.
2. Open the SimpleChatbot.xcodeproj in Xcode.
3. Tell Xcode to update its Package Dependencies by clicking File -> Packages -> Update to Latest Package Versions.
4. Build the project.
5. Run the project on your device.
6. Connect to the URL you are testing.

View File

@@ -0,0 +1,727 @@
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 56;
objects = {
/* Begin PBXBuildFile section */
90031FA72C616EE700408370 /* SimpleChatbotApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FA62C616EE700408370 /* SimpleChatbotApp.swift */; };
90031FA92C616EE700408370 /* PreJoinView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FA82C616EE700408370 /* PreJoinView.swift */; };
90031FAB2C616EE800408370 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 90031FAA2C616EE800408370 /* Assets.xcassets */; };
90031FAE2C616EE800408370 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 90031FAD2C616EE800408370 /* Preview Assets.xcassets */; };
90031FB82C616EE900408370 /* SimpleChatbotTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FB72C616EE900408370 /* SimpleChatbotTests.swift */; };
90031FC22C616EE900408370 /* SimpleChatbotUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FC12C616EE900408370 /* SimpleChatbotUITests.swift */; };
90031FC42C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FC32C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift */; };
90031FDC2C6D5DD700408370 /* ToastModifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90031FDB2C6D5DD700408370 /* ToastModifier.swift */; };
90383A912D9C357F00D0DDA3 /* PipecatClientIOSSmallWebrtc in Frameworks */ = {isa = PBXBuildFile; productRef = 90383A902D9C357F00D0DDA3 /* PipecatClientIOSSmallWebrtc */; };
90383A932D9C35B300D0DDA3 /* ChatView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90383A922D9C35B300D0DDA3 /* ChatView.swift */; };
90383A962D9C35BD00D0DDA3 /* LiveMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90383A942D9C35BD00D0DDA3 /* LiveMessage.swift */; };
90383A982D9D85E700D0DDA3 /* CameraButtonView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90383A972D9D85E700D0DDA3 /* CameraButtonView.swift */; };
90383A9B2DA4620800D0DDA3 /* PipecatClientIOSSmallWebrtc in Frameworks */ = {isa = PBXBuildFile; productRef = 90383A9A2DA4620800D0DDA3 /* PipecatClientIOSSmallWebrtc */; };
90ABB98E2C735ED6000D9CC7 /* MeetingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB98D2C735ED6000D9CC7 /* MeetingView.swift */; };
90ABB9932C73820D000D9CC7 /* MicrophoneView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9922C73820D000D9CC7 /* MicrophoneView.swift */; };
90ABB9982C738356000D9CC7 /* CustomColors.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9972C738356000D9CC7 /* CustomColors.swift */; };
90ABB99A2C73A6A9000D9CC7 /* MockCallContainerModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9992C73A6A9000D9CC7 /* MockCallContainerModel.swift */; };
90ABB99D2C73C2D1000D9CC7 /* CallContainerModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB99C2C73C2D1000D9CC7 /* CallContainerModel.swift */; };
90ABB9A32C74E1CE000D9CC7 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A22C74E1CE000D9CC7 /* SettingsView.swift */; };
90ABB9A62C74EA8A000D9CC7 /* SettingsPreference.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A52C74EA8A000D9CC7 /* SettingsPreference.swift */; };
90ABB9A82C74EAB1000D9CC7 /* SettingsManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 90ABB9A72C74EAB1000D9CC7 /* SettingsManager.swift */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
90031FB42C616EE900408370 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 90031F9B2C616EE700408370 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 90031FA22C616EE700408370;
remoteInfo = SimpleChatbot;
};
90031FBE2C616EE900408370 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 90031F9B2C616EE700408370 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 90031FA22C616EE700408370;
remoteInfo = SimpleChatbot;
};
/* End PBXContainerItemProxy section */
/* Begin PBXFileReference section */
90031FA32C616EE700408370 /* SimpleChatbot.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SimpleChatbot.app; sourceTree = BUILT_PRODUCTS_DIR; };
90031FA62C616EE700408370 /* SimpleChatbotApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SimpleChatbotApp.swift; sourceTree = "<group>"; };
90031FA82C616EE700408370 /* PreJoinView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PreJoinView.swift; sourceTree = "<group>"; };
90031FAA2C616EE800408370 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
90031FAD2C616EE800408370 /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = "<group>"; };
90031FB32C616EE900408370 /* SimpleChatbotTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = SimpleChatbotTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
90031FB72C616EE900408370 /* SimpleChatbotTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SimpleChatbotTests.swift; sourceTree = "<group>"; };
90031FBD2C616EE900408370 /* SimpleChatbotUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = SimpleChatbotUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
90031FC12C616EE900408370 /* SimpleChatbotUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SimpleChatbotUITests.swift; sourceTree = "<group>"; };
90031FC32C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SimpleChatbotUITestsLaunchTests.swift; sourceTree = "<group>"; };
90031FD62C63FD6A00408370 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = "<group>"; };
90031FDB2C6D5DD700408370 /* ToastModifier.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ToastModifier.swift; sourceTree = "<group>"; };
90383A922D9C35B300D0DDA3 /* ChatView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ChatView.swift; sourceTree = "<group>"; };
90383A942D9C35BD00D0DDA3 /* LiveMessage.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = LiveMessage.swift; sourceTree = "<group>"; };
90383A972D9D85E700D0DDA3 /* CameraButtonView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraButtonView.swift; sourceTree = "<group>"; };
90ABB98D2C735ED6000D9CC7 /* MeetingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingView.swift; sourceTree = "<group>"; };
90ABB9922C73820D000D9CC7 /* MicrophoneView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MicrophoneView.swift; sourceTree = "<group>"; };
90ABB9972C738356000D9CC7 /* CustomColors.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CustomColors.swift; sourceTree = "<group>"; };
90ABB9992C73A6A9000D9CC7 /* MockCallContainerModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MockCallContainerModel.swift; sourceTree = "<group>"; };
90ABB99C2C73C2D1000D9CC7 /* CallContainerModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CallContainerModel.swift; sourceTree = "<group>"; };
90ABB9A22C74E1CE000D9CC7 /* SettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsView.swift; sourceTree = "<group>"; };
90ABB9A52C74EA8A000D9CC7 /* SettingsPreference.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsPreference.swift; sourceTree = "<group>"; };
90ABB9A72C74EAB1000D9CC7 /* SettingsManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsManager.swift; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
90031FA02C616EE700408370 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
90383A912D9C357F00D0DDA3 /* PipecatClientIOSSmallWebrtc in Frameworks */,
90383A9B2DA4620800D0DDA3 /* PipecatClientIOSSmallWebrtc in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
90031FB02C616EE900408370 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
90031FBA2C616EE900408370 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
90031F9A2C616EE700408370 = {
isa = PBXGroup;
children = (
90031FA52C616EE700408370 /* SimpleChatbot */,
90031FB62C616EE900408370 /* SimpleChatbotTests */,
90031FC02C616EE900408370 /* SimpleChatbotUITests */,
90031FA42C616EE700408370 /* Products */,
);
sourceTree = "<group>";
};
90031FA42C616EE700408370 /* Products */ = {
isa = PBXGroup;
children = (
90031FA32C616EE700408370 /* SimpleChatbot.app */,
90031FB32C616EE900408370 /* SimpleChatbotTests.xctest */,
90031FBD2C616EE900408370 /* SimpleChatbotUITests.xctest */,
);
name = Products;
sourceTree = "<group>";
};
90031FA52C616EE700408370 /* SimpleChatbot */ = {
isa = PBXGroup;
children = (
90383A952D9C35BD00D0DDA3 /* types */,
90ABB99B2C73C2C5000D9CC7 /* model */,
90031FDD2C6D61E000408370 /* views */,
90031FD62C63FD6A00408370 /* Info.plist */,
90031FA62C616EE700408370 /* SimpleChatbotApp.swift */,
90031FAA2C616EE800408370 /* Assets.xcassets */,
90031FAC2C616EE800408370 /* Preview Content */,
);
path = SimpleChatbot;
sourceTree = "<group>";
};
90031FAC2C616EE800408370 /* Preview Content */ = {
isa = PBXGroup;
children = (
90031FAD2C616EE800408370 /* Preview Assets.xcassets */,
);
path = "Preview Content";
sourceTree = "<group>";
};
90031FB62C616EE900408370 /* SimpleChatbotTests */ = {
isa = PBXGroup;
children = (
90031FB72C616EE900408370 /* SimpleChatbotTests.swift */,
);
path = SimpleChatbotTests;
sourceTree = "<group>";
};
90031FC02C616EE900408370 /* SimpleChatbotUITests */ = {
isa = PBXGroup;
children = (
90031FC12C616EE900408370 /* SimpleChatbotUITests.swift */,
90031FC32C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift */,
);
path = SimpleChatbotUITests;
sourceTree = "<group>";
};
90031FDD2C6D61E000408370 /* views */ = {
isa = PBXGroup;
children = (
90ABB99E2C73C3A9000D9CC7 /* components */,
90ABB9962C738346000D9CC7 /* extensions */,
90ABB9A42C74EA52000D9CC7 /* settings */,
90031FA82C616EE700408370 /* PreJoinView.swift */,
90ABB98D2C735ED6000D9CC7 /* MeetingView.swift */,
);
path = views;
sourceTree = "<group>";
};
90383A952D9C35BD00D0DDA3 /* types */ = {
isa = PBXGroup;
children = (
90383A942D9C35BD00D0DDA3 /* LiveMessage.swift */,
);
path = types;
sourceTree = "<group>";
};
90ABB9962C738346000D9CC7 /* extensions */ = {
isa = PBXGroup;
children = (
90ABB9972C738356000D9CC7 /* CustomColors.swift */,
);
path = extensions;
sourceTree = "<group>";
};
90ABB99B2C73C2C5000D9CC7 /* model */ = {
isa = PBXGroup;
children = (
90ABB9992C73A6A9000D9CC7 /* MockCallContainerModel.swift */,
90ABB99C2C73C2D1000D9CC7 /* CallContainerModel.swift */,
);
path = model;
sourceTree = "<group>";
};
90ABB99E2C73C3A9000D9CC7 /* components */ = {
isa = PBXGroup;
children = (
90383A972D9D85E700D0DDA3 /* CameraButtonView.swift */,
90383A922D9C35B300D0DDA3 /* ChatView.swift */,
90ABB9922C73820D000D9CC7 /* MicrophoneView.swift */,
90031FDB2C6D5DD700408370 /* ToastModifier.swift */,
);
path = components;
sourceTree = "<group>";
};
90ABB9A42C74EA52000D9CC7 /* settings */ = {
isa = PBXGroup;
children = (
90ABB9A52C74EA8A000D9CC7 /* SettingsPreference.swift */,
90ABB9A72C74EAB1000D9CC7 /* SettingsManager.swift */,
90ABB9A22C74E1CE000D9CC7 /* SettingsView.swift */,
);
path = settings;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
90031FA22C616EE700408370 /* SimpleChatbot */ = {
isa = PBXNativeTarget;
buildConfigurationList = 90031FC72C616EE900408370 /* Build configuration list for PBXNativeTarget "SimpleChatbot" */;
buildPhases = (
90031F9F2C616EE700408370 /* Sources */,
90031FA02C616EE700408370 /* Frameworks */,
90031FA12C616EE700408370 /* Resources */,
);
buildRules = (
);
dependencies = (
);
name = SimpleChatbot;
packageProductDependencies = (
90383A902D9C357F00D0DDA3 /* PipecatClientIOSSmallWebrtc */,
90383A9A2DA4620800D0DDA3 /* PipecatClientIOSSmallWebrtc */,
);
productName = SimpleChatbot;
productReference = 90031FA32C616EE700408370 /* SimpleChatbot.app */;
productType = "com.apple.product-type.application";
};
90031FB22C616EE900408370 /* SimpleChatbotTests */ = {
isa = PBXNativeTarget;
buildConfigurationList = 90031FCA2C616EE900408370 /* Build configuration list for PBXNativeTarget "SimpleChatbotTests" */;
buildPhases = (
90031FAF2C616EE900408370 /* Sources */,
90031FB02C616EE900408370 /* Frameworks */,
90031FB12C616EE900408370 /* Resources */,
);
buildRules = (
);
dependencies = (
90031FB52C616EE900408370 /* PBXTargetDependency */,
);
name = SimpleChatbotTests;
productName = SimpleChatbotTests;
productReference = 90031FB32C616EE900408370 /* SimpleChatbotTests.xctest */;
productType = "com.apple.product-type.bundle.unit-test";
};
90031FBC2C616EE900408370 /* SimpleChatbotUITests */ = {
isa = PBXNativeTarget;
buildConfigurationList = 90031FCD2C616EE900408370 /* Build configuration list for PBXNativeTarget "SimpleChatbotUITests" */;
buildPhases = (
90031FB92C616EE900408370 /* Sources */,
90031FBA2C616EE900408370 /* Frameworks */,
90031FBB2C616EE900408370 /* Resources */,
);
buildRules = (
);
dependencies = (
90031FBF2C616EE900408370 /* PBXTargetDependency */,
);
name = SimpleChatbotUITests;
productName = SimpleChatbotUITests;
productReference = 90031FBD2C616EE900408370 /* SimpleChatbotUITests.xctest */;
productType = "com.apple.product-type.bundle.ui-testing";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
90031F9B2C616EE700408370 /* Project object */ = {
isa = PBXProject;
attributes = {
BuildIndependentTargetsInParallel = 1;
LastSwiftUpdateCheck = 1540;
LastUpgradeCheck = 1540;
TargetAttributes = {
90031FA22C616EE700408370 = {
CreatedOnToolsVersion = 15.4;
};
90031FB22C616EE900408370 = {
CreatedOnToolsVersion = 15.4;
TestTargetID = 90031FA22C616EE700408370;
};
90031FBC2C616EE900408370 = {
CreatedOnToolsVersion = 15.4;
TestTargetID = 90031FA22C616EE700408370;
};
};
};
buildConfigurationList = 90031F9E2C616EE700408370 /* Build configuration list for PBXProject "SimpleChatbot" */;
compatibilityVersion = "Xcode 14.0";
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
en,
Base,
);
mainGroup = 90031F9A2C616EE700408370;
packageReferences = (
90383A992DA4620800D0DDA3 /* XCRemoteSwiftPackageReference "pipecat-client-ios-small-webrtc" */,
);
productRefGroup = 90031FA42C616EE700408370 /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
90031FA22C616EE700408370 /* SimpleChatbot */,
90031FB22C616EE900408370 /* SimpleChatbotTests */,
90031FBC2C616EE900408370 /* SimpleChatbotUITests */,
);
};
/* End PBXProject section */
/* Begin PBXResourcesBuildPhase section */
90031FA12C616EE700408370 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
90031FAE2C616EE800408370 /* Preview Assets.xcassets in Resources */,
90031FAB2C616EE800408370 /* Assets.xcassets in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
90031FB12C616EE900408370 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
90031FBB2C616EE900408370 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXResourcesBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
90031F9F2C616EE700408370 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
90ABB99D2C73C2D1000D9CC7 /* CallContainerModel.swift in Sources */,
90ABB9A62C74EA8A000D9CC7 /* SettingsPreference.swift in Sources */,
90383A962D9C35BD00D0DDA3 /* LiveMessage.swift in Sources */,
90ABB99A2C73A6A9000D9CC7 /* MockCallContainerModel.swift in Sources */,
90031FA92C616EE700408370 /* PreJoinView.swift in Sources */,
90383A982D9D85E700D0DDA3 /* CameraButtonView.swift in Sources */,
90383A932D9C35B300D0DDA3 /* ChatView.swift in Sources */,
90ABB9982C738356000D9CC7 /* CustomColors.swift in Sources */,
90ABB98E2C735ED6000D9CC7 /* MeetingView.swift in Sources */,
90ABB9A32C74E1CE000D9CC7 /* SettingsView.swift in Sources */,
90ABB9932C73820D000D9CC7 /* MicrophoneView.swift in Sources */,
90ABB9A82C74EAB1000D9CC7 /* SettingsManager.swift in Sources */,
90031FDC2C6D5DD700408370 /* ToastModifier.swift in Sources */,
90031FA72C616EE700408370 /* SimpleChatbotApp.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
90031FAF2C616EE900408370 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
90031FB82C616EE900408370 /* SimpleChatbotTests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
90031FB92C616EE900408370 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
90031FC42C616EE900408370 /* SimpleChatbotUITestsLaunchTests.swift in Sources */,
90031FC22C616EE900408370 /* SimpleChatbotUITests.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXTargetDependency section */
90031FB52C616EE900408370 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = 90031FA22C616EE700408370 /* SimpleChatbot */;
targetProxy = 90031FB42C616EE900408370 /* PBXContainerItemProxy */;
};
90031FBF2C616EE900408370 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = 90031FA22C616EE700408370 /* SimpleChatbot */;
targetProxy = 90031FBE2C616EE900408370 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
/* Begin XCBuildConfiguration section */
90031FC52C616EE900408370 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_ENABLE_OBJC_WEAK = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
ENABLE_USER_SCRIPT_SANDBOXING = YES;
GCC_C_LANGUAGE_STANDARD = gnu17;
GCC_DYNAMIC_NO_PIC = NO;
GCC_NO_COMMON_BLOCKS = YES;
GCC_OPTIMIZATION_LEVEL = 0;
GCC_PREPROCESSOR_DEFINITIONS = (
"DEBUG=1",
"$(inherited)",
);
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 17.5;
LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
ONLY_ACTIVE_ARCH = YES;
SDKROOT = iphoneos;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)";
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
};
name = Debug;
};
90031FC62C616EE900408370 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES;
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = YES;
CLANG_ENABLE_OBJC_WEAK = YES;
CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
CLANG_WARN_BOOL_CONVERSION = YES;
CLANG_WARN_COMMA = YES;
CLANG_WARN_CONSTANT_CONVERSION = YES;
CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
CLANG_WARN_EMPTY_BODY = YES;
CLANG_WARN_ENUM_CONVERSION = YES;
CLANG_WARN_INFINITE_RECURSION = YES;
CLANG_WARN_INT_CONVERSION = YES;
CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
CLANG_WARN_STRICT_PROTOTYPES = YES;
CLANG_WARN_SUSPICIOUS_MOVE = YES;
CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_USER_SCRIPT_SANDBOXING = YES;
GCC_C_LANGUAGE_STANDARD = gnu17;
GCC_NO_COMMON_BLOCKS = YES;
GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
GCC_WARN_UNDECLARED_SELECTOR = YES;
GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 17.5;
LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
MTL_ENABLE_DEBUG_INFO = NO;
MTL_FAST_MATH = YES;
SDKROOT = iphoneos;
SWIFT_COMPILATION_MODE = wholemodule;
VALIDATE_PRODUCT = YES;
};
name = Release;
};
90031FC82C616EE900408370 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_ASSET_PATHS = "\"SimpleChatbot/Preview Content\"";
DEVELOPMENT_TEAM = EEBGKV9N3N;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = SimpleChatbot/Info.plist;
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
INFOPLIST_KEY_UILaunchScreen_Generation = YES;
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = co.daily.SimpleChatbot;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Debug;
};
90031FC92C616EE900408370 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_ASSET_PATHS = "\"SimpleChatbot/Preview Content\"";
DEVELOPMENT_TEAM = EEBGKV9N3N;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = SimpleChatbot/Info.plist;
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
INFOPLIST_KEY_UILaunchScreen_Generation = YES;
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = co.daily.SimpleChatbot;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = YES;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
};
name = Release;
};
90031FCB2C616EE900408370 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES;
BUNDLE_LOADER = "$(TEST_HOST)";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = EEBGKV9N3N;
GENERATE_INFOPLIST_FILE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 17.5;
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = co.daily.SimpleChatbotTests;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/SimpleChatbot.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/SimpleChatbot";
};
name = Debug;
};
90031FCC2C616EE900408370 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES;
BUNDLE_LOADER = "$(TEST_HOST)";
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = EEBGKV9N3N;
GENERATE_INFOPLIST_FILE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 17.5;
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = co.daily.SimpleChatbotTests;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/SimpleChatbot.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/SimpleChatbot";
};
name = Release;
};
90031FCE2C616EE900408370 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = EEBGKV9N3N;
GENERATE_INFOPLIST_FILE = YES;
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = co.daily.SimpleChatbotUITests;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
TEST_TARGET_NAME = SimpleChatbot;
};
name = Debug;
};
90031FCF2C616EE900408370 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES;
CODE_SIGN_STYLE = Automatic;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = EEBGKV9N3N;
GENERATE_INFOPLIST_FILE = YES;
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = co.daily.SimpleChatbotUITests;
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
SWIFT_VERSION = 5.0;
TARGETED_DEVICE_FAMILY = "1,2";
TEST_TARGET_NAME = SimpleChatbot;
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
90031F9E2C616EE700408370 /* Build configuration list for PBXProject "SimpleChatbot" */ = {
isa = XCConfigurationList;
buildConfigurations = (
90031FC52C616EE900408370 /* Debug */,
90031FC62C616EE900408370 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
90031FC72C616EE900408370 /* Build configuration list for PBXNativeTarget "SimpleChatbot" */ = {
isa = XCConfigurationList;
buildConfigurations = (
90031FC82C616EE900408370 /* Debug */,
90031FC92C616EE900408370 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
90031FCA2C616EE900408370 /* Build configuration list for PBXNativeTarget "SimpleChatbotTests" */ = {
isa = XCConfigurationList;
buildConfigurations = (
90031FCB2C616EE900408370 /* Debug */,
90031FCC2C616EE900408370 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
90031FCD2C616EE900408370 /* Build configuration list for PBXNativeTarget "SimpleChatbotUITests" */ = {
isa = XCConfigurationList;
buildConfigurations = (
90031FCE2C616EE900408370 /* Debug */,
90031FCF2C616EE900408370 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
/* Begin XCRemoteSwiftPackageReference section */
90383A992DA4620800D0DDA3 /* XCRemoteSwiftPackageReference "pipecat-client-ios-small-webrtc" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/pipecat-ai/pipecat-client-ios-small-webrtc";
requirement = {
kind = upToNextMajorVersion;
minimumVersion = 0.0.1;
};
};
/* End XCRemoteSwiftPackageReference section */
/* Begin XCSwiftPackageProductDependency section */
90383A902D9C357F00D0DDA3 /* PipecatClientIOSSmallWebrtc */ = {
isa = XCSwiftPackageProductDependency;
productName = PipecatClientIOSSmallWebrtc;
};
90383A9A2DA4620800D0DDA3 /* PipecatClientIOSSmallWebrtc */ = {
isa = XCSwiftPackageProductDependency;
package = 90383A992DA4620800D0DDA3 /* XCRemoteSwiftPackageReference "pipecat-client-ios-small-webrtc" */;
productName = PipecatClientIOSSmallWebrtc;
};
/* End XCSwiftPackageProductDependency section */
};
rootObject = 90031F9B2C616EE700408370 /* Project object */;
}

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEDidComputeMac32BitWarning</key>
<true/>
</dict>
</plist>

View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEWorkspaceSharedSettings_AutocreateContextsIfNeeded</key>
<false/>
</dict>
</plist>

View File

@@ -0,0 +1,33 @@
{
"originHash" : "77cb3fee4071811f880e69dbcd5a8ba01711a73372960391d6366c4c3a0d36eb",
"pins" : [
{
"identity" : "pipecat-client-ios",
"kind" : "remoteSourceControl",
"location" : "https://github.com/pipecat-ai/pipecat-client-ios.git",
"state" : {
"revision" : "992641fb5f7d1a794ecfc33babb5fe36e2a8ffdd",
"version" : "0.3.5"
}
},
{
"identity" : "pipecat-client-ios-small-webrtc",
"kind" : "remoteSourceControl",
"location" : "https://github.com/pipecat-ai/pipecat-client-ios-small-webrtc",
"state" : {
"revision" : "a6e4516b1fcbed772ca97a9616dddc9329097958",
"version" : "0.0.1"
}
},
{
"identity" : "webrtc",
"kind" : "remoteSourceControl",
"location" : "https://github.com/stasel/WebRTC",
"state" : {
"revision" : "5b2eb61cace7d62726b29a38b768b07d6bc55c45",
"version" : "134.0.0"
}
}
],
"version" : 3
}

View File

@@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "1540"
version = "1.7">
<BuildAction
parallelizeBuildables = "YES"
buildImplicitDependencies = "YES"
buildArchitectures = "Automatic">
<BuildActionEntries>
<BuildActionEntry
buildForTesting = "YES"
buildForRunning = "YES"
buildForProfiling = "YES"
buildForArchiving = "YES"
buildForAnalyzing = "YES">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "90031FA22C616EE700408370"
BuildableName = "SimpleChatbot.app"
BlueprintName = "SimpleChatbot"
ReferencedContainer = "container:SimpleChatbot.xcodeproj">
</BuildableReference>
</BuildActionEntry>
</BuildActionEntries>
</BuildAction>
<TestAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
shouldUseLaunchSchemeArgsEnv = "YES"
shouldAutocreateTestPlan = "YES">
</TestAction>
<LaunchAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
launchStyle = "0"
useCustomWorkingDirectory = "NO"
ignoresPersistentStateOnLaunch = "NO"
debugDocumentVersioning = "YES"
debugServiceExtension = "internal"
allowLocationSimulation = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "90031FA22C616EE700408370"
BuildableName = "SimpleChatbot.app"
BlueprintName = "SimpleChatbot"
ReferencedContainer = "container:SimpleChatbot.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
</LaunchAction>
<ProfileAction
buildConfiguration = "Release"
shouldUseLaunchSchemeArgsEnv = "YES"
savedToolIdentifier = ""
useCustomWorkingDirectory = "NO"
debugDocumentVersioning = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "90031FA22C616EE700408370"
BuildableName = "SimpleChatbot.app"
BlueprintName = "SimpleChatbot"
ReferencedContainer = "container:SimpleChatbot.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
</ProfileAction>
<AnalyzeAction
buildConfiguration = "Debug">
</AnalyzeAction>
<ArchiveAction
buildConfiguration = "Release"
revealArchiveInOrganizer = "YES">
</ArchiveAction>
</Scheme>

View File

@@ -0,0 +1,14 @@
{
"images" : [
{
"filename" : "appstore.png",
"idiom" : "universal",
"platform" : "ios",
"size" : "1024x1024"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

View File

@@ -0,0 +1,6 @@
{
"info" : {
"author" : "xcode",
"version" : 1
}
}

View File

@@ -0,0 +1,21 @@
{
"images" : [
{
"filename" : "Square Black.svg",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}

View File

@@ -0,0 +1,8 @@
<svg width="450" height="450" viewBox="0 0 450 450" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="450" height="450" fill="white"/>
<path d="M104.772 129.77C109.448 128.01 114.725 129.331 118.02 133.086L160.936 182H289.064L331.98 133.086C335.275 129.331 340.552 128.01 345.228 129.77C349.904 131.531 353 136.004 353 141V249H391V273H329V172.873L303.52 201.915C301.242 204.511 297.955 206 294.5 206H155.5C152.045 206 148.758 204.511 146.48 201.915L121 172.873V273H59V249H97V141C97 136.004 100.096 131.531 104.772 129.77Z" fill="black"/>
<path d="M329 297H391V321H329V297Z" fill="black"/>
<path d="M59 297H121V321H59V297Z" fill="black"/>
<path d="M187 257C187 265.837 179.837 273 171 273C162.163 273 155 265.837 155 257C155 248.164 162.163 241 171 241C179.837 241 187 248.164 187 257Z" fill="black"/>
<path d="M295 257C295 265.837 287.837 273 279 273C270.163 273 263 265.837 263 257C263 248.164 270.163 241 279 241C287.837 241 295 248.164 295 257Z" fill="black"/>
</svg>

After

Width:  |  Height:  |  Size: 982 B

View File

@@ -0,0 +1,21 @@
{
"images" : [
{
"filename" : "vision.svg",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}

View File

@@ -0,0 +1,9 @@
<svg width="32" height="32" viewBox="0 0 32 32" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M4 9.33333V6.66667C4 5.95942 4.28095 5.28115 4.78105 4.78105C5.28115 4.28095 5.95942 4 6.66667 4H9.33333" stroke="#15803D" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M22.6667 4H25.3334C26.0407 4 26.7189 4.28095 27.219 4.78105C27.7191 5.28115 28.0001 5.95942 28.0001 6.66667V9.33333" stroke="#15803D" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M28.0001 22.6667V25.3333C28.0001 26.0406 27.7191 26.7188 27.219 27.2189C26.7189 27.719 26.0407 28 25.3334 28H22.6667" stroke="#15803D" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M9.33333 28H6.66667C5.95942 28 5.28115 27.719 4.78105 27.2189C4.28095 26.7188 4 26.0406 4 25.3333V22.6667" stroke="#15803D" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M16.0001 18.2857C17.2624 18.2857 18.2858 17.2624 18.2858 16C18.2858 14.7376 17.2624 13.7143 16.0001 13.7143C14.7377 13.7143 13.7144 14.7376 13.7144 16C13.7144 17.2624 14.7377 18.2857 16.0001 18.2857Z" stroke="#15803D" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M25.2588 16.44C25.3584 16.1551 25.3584 15.8449 25.2588 15.56C24.5081 13.7206 23.2265 12.1464 21.5775 11.0383C19.9285 9.93028 17.9868 9.3385 16.0001 9.3385C14.0134 9.3385 12.0717 9.93028 10.4227 11.0383C8.7737 12.1464 7.49212 13.7206 6.74144 15.56C6.64185 15.8449 6.64185 16.1551 6.74144 16.44C7.49212 18.2794 8.7737 19.8536 10.4227 20.9616C12.0717 22.0697 14.0134 22.6615 16.0001 22.6615C17.9868 22.6615 19.9285 22.0697 21.5775 20.9616C23.2265 19.8536 24.5081 18.2794 25.2588 16.44Z" stroke="#15803D" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
</svg>

After

Width:  |  Height:  |  Size: 1.7 KiB

View File

@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>UIBackgroundModes</key>
<array>
<string>voip</string>
</array>
<key>NSCameraUsageDescription</key>
<string>Camera is necessary for transmitting video in a call</string>
<key>NSMicrophoneUsageDescription</key>
<string>Microphone is necessary for transmitting audio in a call</string>
<key>NSAppTransportSecurity</key>
<dict>
<key>NSAllowsLocalNetworking</key>
<true/>
</dict>
</dict>
</plist>

View File

@@ -0,0 +1,6 @@
{
"info" : {
"author" : "xcode",
"version" : 1
}
}

View File

@@ -0,0 +1,18 @@
import SwiftUI
@main
struct SimpleChatbotApp: App {
@StateObject var callContainerModel = CallContainerModel()
var body: some Scene {
WindowGroup {
if (!callContainerModel.isInCall) {
PreJoinView().environmentObject(callContainerModel)
} else {
MeetingView().environmentObject(callContainerModel)
}
}
}
}

View File

@@ -0,0 +1,284 @@
import SwiftUI
import PipecatClientIOSSmallWebrtc
import PipecatClientIOS
class CallContainerModel: ObservableObject {
@Published var voiceClientStatus: String = TransportState.disconnected.description
@Published var isInCall: Bool = false
@Published var isBotReady: Bool = false
@Published var isMicEnabled: Bool = false
@Published var isCamEnabled: Bool = false
@Published var localCamId: MediaTrackId? = nil
@Published var botCamId: MediaTrackId? = nil
@Published var toastMessage: String? = nil
@Published var showToast: Bool = false
@Published var messages: [LiveMessage] = []
@Published var liveBotMessage: LiveMessage?
@Published var liveUserMessage: LiveMessage?
var rtviClientIOS: RTVIClient?
@Published var selectedMic: MediaDeviceId? = nil {
didSet {
guard let selectedMic else { return } // don't store nil
var settings = SettingsManager.getSettings()
settings.selectedMic = selectedMic.id
SettingsManager.updateSettings(settings: settings)
}
}
@Published var availableMics: [MediaDeviceInfo] = []
init() {
// Changing the log level
PipecatClientIOS.setLogLevel(.warn)
PipecatClientIOSSmallWebrtc.setLogLevel(.info)
}
@MainActor
func connect(backendURL: String) {
self.resetLiveMessages()
let baseUrl = backendURL.trimmingCharacters(in: .whitespacesAndNewlines)
if(baseUrl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty){
self.showError(message: "Need to fill the backendURL")
return
}
let currentSettings = SettingsManager.getSettings()
let rtviClientOptions = RTVIClientOptions.init(
enableMic: currentSettings.enableMic,
enableCam: currentSettings.enableCam,
params: RTVIClientParams(
config: [
.init(
service: SmallWebRTCTransport.SERVICE_NAME,
options: [
.init(name: "server_url", value: .string(baseUrl))
]
)
]
)
)
self.rtviClientIOS = RTVIClient.init(
transport: SmallWebRTCTransport.init(options: rtviClientOptions),
options: rtviClientOptions
)
self.rtviClientIOS?.delegate = self
// Registering the llm helper, we will need this to handle the function calling
let llmHelper = try? self.rtviClientIOS?.registerHelper(service: "llm", helper: LLMHelper.self)
llmHelper?.delegate = self
self.rtviClientIOS?.start() { result in
switch result {
case .failure(let error):
self.showError(message: error.localizedDescription)
self.rtviClientIOS = nil
case .success():
// Apply initial mic preference
if let selectedMic = SettingsManager.getSettings().selectedMic {
self.selectMic(MediaDeviceId(id: selectedMic))
}
// Populate available devices list
self.availableMics = self.rtviClientIOS?.getAllMics() ?? []
}
}
self.saveCredentials(backendURL: backendURL)
}
@MainActor
func disconnect() {
self.rtviClientIOS?.disconnect(completion: nil)
self.rtviClientIOS?.release()
self.rtviClientIOS = nil
}
func showError(message: String) {
self.toastMessage = message
self.showToast = true
// Hide the toast after 5 seconds
DispatchQueue.main.asyncAfter(deadline: .now() + 5) {
self.showToast = false
self.toastMessage = nil
}
}
@MainActor
func toggleMicInput() {
self.rtviClientIOS?.enableMic(enable: !self.isMicEnabled) { result in
switch result {
case .success():
self.isMicEnabled = self.rtviClientIOS?.isMicEnabled ?? false
case .failure(let error):
self.showError(message: error.localizedDescription)
}
}
}
@MainActor
func toggleCamInput() {
self.rtviClientIOS?.enableCam(enable: !self.isCamEnabled) { result in
switch result {
case .success():
self.isCamEnabled = self.rtviClientIOS?.isCamEnabled ?? false
case .failure(let error):
self.showError(message: error.localizedDescription)
}
}
}
func saveCredentials(backendURL: String) {
var currentSettings = SettingsManager.getSettings()
currentSettings.backendURL = backendURL
// Saving the settings
SettingsManager.updateSettings(settings: currentSettings)
}
@MainActor
func selectMic(_ mic: MediaDeviceId) {
self.selectedMic = mic
self.rtviClientIOS?.updateMic(micId: mic, completion: nil)
}
private func createLiveMessage(content:String = "", type:MessageType) {
// Creating a new one
DispatchQueue.main.async {
let liveMessage = LiveMessage(content: content, type: type, updatedAt: Date())
self.messages.append(liveMessage)
if type == .bot {
self.liveBotMessage = liveMessage
} else if type == .user {
self.liveUserMessage = liveMessage
}
}
}
private func appendTextToLiveMessage(fromBot: Bool, content:String) {
DispatchQueue.main.async {
// Updating the last message with the new content
if fromBot {
self.liveBotMessage?.content += content
} else {
self.liveUserMessage?.content += content
}
}
}
private func resetLiveMessages() {
DispatchQueue.main.async {
self.messages = []
}
}
}
extension CallContainerModel:RTVIClientDelegate, LLMHelperDelegate {
private func handleEvent(eventName: String, eventValue: Any? = nil) {
if let value = eventValue {
print("Pipecat Demo, received event: \(eventName), value:\(value)")
} else {
print("Pipecat Demo, received event: \(eventName)")
}
}
func onTransportStateChanged(state: TransportState) {
Task { @MainActor in
self.handleEvent(eventName: "onTransportStateChanged", eventValue: state)
self.voiceClientStatus = state.description
self.isInCall = ( state == .connecting || state == .connected || state == .ready || state == .authenticating )
self.createLiveMessage(content: state.description, type: .system)
}
}
func onBotReady(botReadyData: BotReadyData) {
Task { @MainActor in
self.handleEvent(eventName: "onBotReady")
self.isBotReady = true
}
}
func onConnected() {
Task { @MainActor in
self.handleEvent(eventName: "onConnected")
self.isMicEnabled = self.rtviClientIOS?.isMicEnabled ?? false
self.isCamEnabled = self.rtviClientIOS?.isCamEnabled ?? false
}
}
func onDisconnected() {
Task { @MainActor in
self.handleEvent(eventName: "onDisconnected")
self.isBotReady = false
}
}
func onError(message: String) {
Task { @MainActor in
self.handleEvent(eventName: "onError", eventValue: message)
self.showError(message: message)
}
}
func onAvailableMicsUpdated(mics: [MediaDeviceInfo]) {
Task { @MainActor in
self.availableMics = mics
}
}
func onMicUpdated(mic: MediaDeviceInfo?) {
Task { @MainActor in
self.selectedMic = mic?.id
}
}
func onBotTranscript(data: String) {
self.handleEvent(eventName: "onBotTranscript", eventValue: data)
}
func onTracksUpdated(tracks: Tracks) {
self.handleEvent(eventName: "onTracksUpdated", eventValue: tracks)
Task { @MainActor in
self.localCamId = tracks.local.video
self.botCamId = tracks.bot?.video ?? nil
}
}
func onUserStartedSpeaking() {
self.createLiveMessage(content: "User started speaking", type: .system)
self.handleEvent(eventName: "onUserStartedSpeaking")
self.createLiveMessage(type: .user)
}
func onUserStoppedSpeaking() {
self.createLiveMessage(content: "User stopped speaking", type: .system)
self.handleEvent(eventName: "onUserStoppedSpeaking")
}
func onBotStartedSpeaking() {
self.createLiveMessage(content: "Bot started speaking", type: .system)
self.handleEvent(eventName: "onBotStartedSpeaking")
self.createLiveMessage(type: .bot)
}
func onBotStoppedSpeaking() {
self.createLiveMessage(content: "Bot stopped speaking", type: .system)
self.handleEvent(eventName: "onBotStoppedSpeaking")
}
func onUserTranscript(data: Transcript) {
if data.final ?? false {
self.handleEvent(eventName: "onUserTranscript", eventValue: data.text)
self.appendTextToLiveMessage(fromBot: false, content: data.text)
}
}
func onBotTTSText(data: BotTTSText) {
self.appendTextToLiveMessage(fromBot: true, content: data.text)
}
}

View File

@@ -0,0 +1,43 @@
import SwiftUI
import PipecatClientIOS
class MockCallContainerModel: CallContainerModel {
override init() {
super.init()
let liveMessageFromSystem = LiveMessage(
content: "System message",
type: .system,
updatedAt: Date()
)
let liveMessageFromUser = LiveMessage(
content: "Message from User",
type: .user,
updatedAt: Date()
)
let liveMessageFromBot = LiveMessage(
content: "Message from bot",
type: .bot,
updatedAt: Date()
)
self.messages = [ liveMessageFromSystem, liveMessageFromUser, liveMessageFromBot ]
}
override func connect(backendURL: String) {
print("connect")
}
override func disconnect() {
print("disconnect")
}
override func showError(message: String) {
self.toastMessage = message
self.showToast = true
// Hide the toast after 5 seconds
DispatchQueue.main.asyncAfter(deadline: .now() + 5) {
self.showToast = false
self.toastMessage = nil
}
}
}

View File

@@ -0,0 +1,21 @@
import Foundation
enum MessageType {
case bot, user, system
}
class LiveMessage: ObservableObject, Identifiable, Equatable {
@Published var content: String
let type: MessageType
let updatedAt: Date
init(content: String, type: MessageType, updatedAt: Date) {
self.content = content
self.type = type
self.updatedAt = updatedAt
}
static func == (lhs: LiveMessage, rhs: LiveMessage) -> Bool {
lhs.updatedAt == rhs.updatedAt
}
}

View File

@@ -0,0 +1,81 @@
import SwiftUI
import PipecatClientIOSSmallWebrtc
struct MeetingView: View {
@State private var showingSettings = false
@EnvironmentObject private var model: CallContainerModel
var body: some View {
VStack {
ZStack {
SmallWebRTCVideoViewSwiftUI(videoTrack: self.model.botCamId, videoScaleMode: .fill)
.edgesIgnoringSafeArea(.all)
VStack {
ChatView()
.frame(maxHeight: .infinity)
HStack {
MicrophoneView(audioLevel: 0, isMuted: !self.model.isMicEnabled)
.frame(width: 100, height: 100)
.onTapGesture {
self.model.toggleMicInput()
}
CameraButtonView(trackId: self.model.localCamId, isMuted: !self.model.isCamEnabled)
.frame(width: 120, height: 120)
.onTapGesture {
self.model.toggleCamInput()
}
}
.padding()
}
}
Button(action: {
self.showingSettings = true
}) {
HStack {
Image(systemName: "gearshape")
.resizable()
.frame(width: 24, height: 24)
Text("Settings")
}
.frame(maxWidth: .infinity)
.padding()
.sheet(isPresented: $showingSettings) {
SettingsView(showingSettings: $showingSettings).environmentObject(self.model)
}
}
.foregroundColor(.black)
.background(Color.white)
.border(Color.buttonsBorder, width: 1)
.cornerRadius(12)
.padding([.horizontal])
Button(action: {
self.model.disconnect()
}) {
HStack {
Image(systemName: "rectangle.portrait.and.arrow.right")
.resizable()
.frame(width: 24, height: 24)
Text("End")
}
.frame(maxWidth: .infinity)
.padding()
}
.foregroundColor(.white)
.background(Color.black)
.cornerRadius(12)
.padding([.bottom, .horizontal])
}
.background(Color.backgroundApp)
.toast(message: model.toastMessage, isShowing: model.showToast)
}
}
#Preview {
let mockModel = MockCallContainerModel()
let result = MeetingView().environmentObject(mockModel as CallContainerModel)
return result
}

View File

@@ -0,0 +1,44 @@
import SwiftUI
struct PreJoinView: View {
@State var backendURL: String
@EnvironmentObject private var model: CallContainerModel
init() {
let currentSettings = SettingsManager.getSettings()
self.backendURL = currentSettings.backendURL
}
var body: some View {
VStack(spacing: 20) {
Image("pipecat")
.resizable()
.frame(width: 80, height: 80)
Text("Pipecat Client iOS.")
.font(.headline)
TextField("Server URL", text: $backendURL)
.textFieldStyle(RoundedBorderTextFieldStyle())
.frame(maxWidth: .infinity)
.padding([.bottom, .horizontal])
Button("Connect") {
Task {
await self.model.connect(backendURL: self.backendURL)
}
}
.padding()
.background(Color.black)
.foregroundColor(.white)
.cornerRadius(8)
}
.padding()
.frame(maxHeight: .infinity)
.background(Color.backgroundApp)
.toast(message: model.toastMessage, isShowing: model.showToast)
}
}
#Preview {
PreJoinView().environmentObject(MockCallContainerModel() as CallContainerModel)
}

View File

@@ -0,0 +1,42 @@
import SwiftUI
import PipecatClientIOS
import PipecatClientIOSSmallWebrtc
struct CameraButtonView: View {
var trackId: MediaTrackId?
var isMuted: Bool
var body: some View {
GeometryReader { geometry in
let width = geometry.size.width
let circleSize = width * 0.9
let innerCircleSize = width * 0.82
ZStack {
Circle()
.stroke(Color.gray, lineWidth: 1)
.frame(width: circleSize)
if (!isMuted){
SmallWebRTCVideoViewSwiftUI(videoTrack: trackId, videoScaleMode: .fill)
.aspectRatio(1, contentMode: .fit)
.clipShape(Circle())
} else {
Circle()
.fill(Color.disabledVision)
.frame(width: innerCircleSize)
Image("vision")
.resizable()
.scaledToFit()
.frame(width: width * 0.3)
.foregroundColor(.green)
}
}
.frame(maxWidth: .infinity, maxHeight: .infinity) // Ensures the ZStack is centered
}
}
}
#Preview {
CameraButtonView(trackId: nil, isMuted: true)
}

View File

@@ -0,0 +1,96 @@
import SwiftUI
struct ChatView: View {
@EnvironmentObject private var model: CallContainerModel
@State private var timer = Timer.publish(every: 0.5, on: .main, in: .common).autoconnect()
var body: some View {
VStack {
ScrollViewReader { scrollViewProxy in
ScrollView {
VStack(spacing: 10) {
ForEach(self.model.messages) { message in
MessageView(message: message)
.frame(maxWidth: .infinity, alignment: messageAlignment(for: message.type))
.padding(.horizontal)
.id(message.id)
}
}
.onChange(of: self.model.messages) { _, _ in
scrollToLastMessage(scrollViewProxy)
}
}
.onReceive(timer) { _ in
scrollToLastMessage(scrollViewProxy)
}
.onAppear {
scrollToLastMessage(scrollViewProxy)
}
}
}
.edgesIgnoringSafeArea(.bottom)
}
private func messageAlignment(for type: MessageType) -> Alignment {
switch type {
case .bot: return .leading
case .user: return .trailing
case .system: return .center
}
}
private func scrollToLastMessage(_ scrollViewProxy: ScrollViewProxy) {
if let lastMessageId = self.model.messages.last?.id {
withAnimation {
scrollViewProxy.scrollTo(lastMessageId, anchor: .bottom)
}
}
}
}
struct MessageView: View {
@ObservedObject var message: LiveMessage
var body: some View {
HStack {
if message.type == .bot {
Image(systemName: "gearshape")
.resizable()
.frame(width: 24, height: 24)
}
Text(message.content)
.padding(message.type == .system ? 5 : 10)
.foregroundColor(.white)
.background(messageBackgroundColor(for: message.type))
.cornerRadius(15)
.overlay(
RoundedRectangle(cornerRadius: 15)
.stroke(Color.gray.opacity(0.5), lineWidth: 1)
)
}
.padding(messagePadding(for: message.type))
}
private func messageBackgroundColor(for type: MessageType) -> Color {
switch type {
case .bot: return .black
case .user: return .gray
case .system: return .blue.opacity(0.6)
}
}
private func messagePadding(for type: MessageType) -> EdgeInsets {
switch type {
case .bot: return EdgeInsets(top: 0, leading: 0, bottom: 0, trailing: 40)
case .user: return EdgeInsets(top: 0, leading: 40, bottom: 0, trailing: 0)
case .system: return EdgeInsets(top: 0, leading: 0, bottom: 0, trailing: 0)
}
}
}
#Preview {
let mockModel = MockCallContainerModel()
let result = ChatView().environmentObject(mockModel as CallContainerModel)
return result
}

View File

@@ -0,0 +1,44 @@
import SwiftUI
struct MicrophoneView: View {
var audioLevel: Float // Current audio level
var isMuted: Bool // Muted state
var body: some View {
GeometryReader { geometry in
let width = geometry.size.width
let circleSize = width * 0.9
let innerCircleSize = width * 0.82
let audioCircleSize = CGFloat(audioLevel) * (width * 0.95)
ZStack {
Circle()
.stroke(Color.gray, lineWidth: 1)
.frame(width: circleSize)
Circle()
.fill(isMuted ? Color.disabledMic : Color.backgroundCircle)
.frame(width: innerCircleSize)
if !isMuted {
Circle()
.fill(Color.micVolume)
.opacity(0.5)
.frame(width: audioCircleSize)
.animation(.easeInOut(duration: 0.2), value: audioLevel)
}
Image(systemName: isMuted ? "mic.slash.fill" : "mic.fill")
.resizable()
.scaledToFit()
.frame(width: width * 0.2)
.foregroundColor(.white)
}
.frame(maxWidth: .infinity, maxHeight: .infinity) // Ensures the ZStack is centered
}
}
}
#Preview {
MicrophoneView(audioLevel: 1, isMuted: false)
}

View File

@@ -0,0 +1,31 @@
import SwiftUI
struct ToastModifier: ViewModifier {
var message: String?
var isShowing: Bool
func body(content: Content) -> some View {
ZStack {
content
if isShowing, let message = message {
VStack {
Text(message)
.padding()
.background(Color.black.opacity(0.7))
.foregroundColor(.white)
.cornerRadius(8)
.transition(.slide)
.padding(.top, 50)
Spacer()
}
.animation(.easeInOut(duration: 0.5), value: isShowing)
}
}
}
}
extension View {
func toast(message: String?, isShowing: Bool) -> some View {
self.modifier(ToastModifier(message: message, isShowing: isShowing))
}
}

View File

@@ -0,0 +1,26 @@
import SwiftUI
public extension Color {
static let backgroundCircle = Color(hex: "#374151")
static let backgroundCircleNotConnected = Color(hex: "#D1D5DB")
static let backgroundApp = Color(hex: "#F9FAFB")
static let buttonsBorder = Color(hex: "#E5E7EB")
static let micVolume = Color(hex: "#86EFAC")
static let disabledMic = Color(hex: "#ee6b6e")
static let disabledVision = Color(hex: "#BBF7D0")
init(hex: String) {
let scanner = Scanner(string: hex)
_ = scanner.scanString("#")
var rgb: UInt64 = 0
scanner.scanHexInt64(&rgb)
let red = Double((rgb >> 16) & 0xFF) / 255.0
let green = Double((rgb >> 8) & 0xFF) / 255.0
let blue = Double(rgb & 0xFF) / 255.0
self.init(red: red, green: green, blue: blue)
}
}

View File

@@ -0,0 +1,21 @@
import Foundation
class SettingsManager {
private static let preferencesKey = "settingsPreference"
static func getSettings() -> SettingsPreference {
if let data = UserDefaults.standard.data(forKey: preferencesKey),
let settings = try? JSONDecoder().decode(SettingsPreference.self, from: data) {
return settings
} else {
// default values in case we don't have any settings
return SettingsPreference(enableMic: true, enableCam: true, backendURL: "http://YOUR_IP:7860")
}
}
static func updateSettings(settings: SettingsPreference) {
if let data = try? JSONEncoder().encode(settings) {
UserDefaults.standard.set(data, forKey: preferencesKey)
}
}
}

View File

@@ -0,0 +1,9 @@
import Foundation
struct SettingsPreference: Codable {
var selectedMic: String?
var enableMic: Bool
var enableCam: Bool
var backendURL: String
}

View File

@@ -0,0 +1,82 @@
import SwiftUI
struct SettingsView: View {
@EnvironmentObject private var model: CallContainerModel
@Binding var showingSettings: Bool
@State private var isMicEnabled: Bool = true
@State private var isCamEnabled: Bool = true
@State private var backendURL: String = ""
var body: some View {
NavigationView {
Form {
Section {
List(model.availableMics, id: \.self.id.id) { mic in
Button(action: {
model.selectMic(mic.id)
}) {
HStack {
Text(mic.name)
Spacer()
if mic.id == model.selectedMic {
Image(systemName: "checkmark")
}
}
}
}
} header: {
VStack(alignment: .leading) {
Text("Audio Settings")
Text("(No selection = system default)")
}
}
Section(header: Text("Start options")) {
Toggle("Enable Microphone", isOn: $isMicEnabled)
Toggle("Enable Cam", isOn: $isCamEnabled)
}
Section(header: Text("Server")) {
TextField("Backend URL", text: $backendURL)
.keyboardType(.URL)
}
}
.navigationTitle("Settings")
.toolbar {
ToolbarItem(placement: .cancellationAction) {
Button("Close") {
self.saveSettings()
self.showingSettings = false
}
}
}
.onAppear {
self.loadSettings()
}
}
}
private func saveSettings() {
let newSettings = SettingsPreference(
selectedMic: model.selectedMic?.id,
enableMic: isMicEnabled,
enableCam: isCamEnabled,
backendURL: backendURL
)
SettingsManager.updateSettings(settings: newSettings)
}
private func loadSettings() {
let savedSettings = SettingsManager.getSettings()
self.isMicEnabled = savedSettings.enableMic
self.isCamEnabled = savedSettings.enableCam
self.backendURL = savedSettings.backendURL
}
}
#Preview {
let mockModel = MockCallContainerModel()
let result = SettingsView(showingSettings: .constant(true)).environmentObject(mockModel as CallContainerModel)
return result
}

View File

@@ -0,0 +1,29 @@
import XCTest
@testable import SimpleChatbot
final class SimpleChatbotTests: XCTestCase {
override func setUpWithError() throws {
// Put setup code here. This method is called before the invocation of each test method in the class.
}
override func tearDownWithError() throws {
// Put teardown code here. This method is called after the invocation of each test method in the class.
}
func testExample() throws {
// This is an example of a functional test case.
// Use XCTAssert and related functions to verify your tests produce the correct results.
// Any test you write for XCTest can be annotated as throws and async.
// Mark your test throws to produce an unexpected failure when your test encounters an uncaught error.
// Mark your test async to allow awaiting for asynchronous code to complete. Check the results with assertions afterwards.
}
func testPerformanceExample() throws {
// This is an example of a performance test case.
self.measure {
// Put the code you want to measure the time of here.
}
}
}

View File

@@ -0,0 +1,34 @@
import XCTest
final class SimpleChatbotUITests: XCTestCase {
override func setUpWithError() throws {
// Put setup code here. This method is called before the invocation of each test method in the class.
// In UI tests it is usually best to stop immediately when a failure occurs.
continueAfterFailure = false
// In UI tests its important to set the initial state - such as interface orientation - required for your tests before they run. The setUp method is a good place to do this.
}
override func tearDownWithError() throws {
// Put teardown code here. This method is called after the invocation of each test method in the class.
}
func testExample() throws {
// UI tests must launch the application that they test.
let app = XCUIApplication()
app.launch()
// Use XCTAssert and related functions to verify your tests produce the correct results.
}
func testLaunchPerformance() throws {
if #available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 7.0, *) {
// This measures how long it takes to launch your application.
measure(metrics: [XCTApplicationLaunchMetric()]) {
XCUIApplication().launch()
}
}
}
}

View File

@@ -0,0 +1,25 @@
import XCTest
final class SimpleChatbotUITestsLaunchTests: XCTestCase {
override class var runsForEachTargetApplicationUIConfiguration: Bool {
true
}
override func setUpWithError() throws {
continueAfterFailure = false
}
func testLaunch() throws {
let app = XCUIApplication()
app.launch()
// Insert steps here to perform after app launch but before taking a screenshot,
// such as logging into a test account or navigating somewhere in the app
let attachment = XCTAttachment(screenshot: app.screenshot())
attachment.name = "Launch Screen"
attachment.lifetime = .keepAlways
add(attachment)
}
}

View File

@@ -51,6 +51,7 @@
<div class="bot-container">
<div id="bot-video-container">
<video id="bot-video" autoplay="true" playsinline="true"></video>
<button id="mute-btn">📷</button>
</div>
<audio id="bot-audio" autoplay></audio>
</div>

View File

@@ -10,7 +10,7 @@
"license": "ISC",
"dependencies": {
"@pipecat-ai/client-js": "^0.3.2",
"@pipecat-ai/small-webrtc-transport": "^0.0.1"
"@pipecat-ai/small-webrtc-transport": "^0.0.2"
},
"devDependencies": {
"@types/node": "^22.13.1",
@@ -32,9 +32,9 @@
}
},
"node_modules/@daily-co/daily-js": {
"version": "0.73.0",
"resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.73.0.tgz",
"integrity": "sha512-Wz8c60hgmkx8fcEeDAi4L4J0rbafiihWKyXFyhYoFYPsw2OdChHpA4RYwIB+1enRws5IK+/HdmzFDYLQsB4A6w==",
"version": "0.77.0",
"resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.77.0.tgz",
"integrity": "sha512-icNXKieKAkRR/C5dcPjrCkL1jQGFp5C5WtLHy5uHAdTztm+mo9wlPJuehbWaGOM3TV24mgWHZ/+8jOys1G0I4w==",
"license": "BSD-2-Clause",
"dependencies": {
"@babel/runtime": "^7.12.5",
@@ -78,12 +78,12 @@
}
},
"node_modules/@pipecat-ai/small-webrtc-transport": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-0.0.1.tgz",
"integrity": "sha512-WAOI7lT0V7cYOn0+qwUAryGxcOGe+wPVPEPzkR3qsM5GWIZ73spykZnuOndQGycq4UkcXVawCzERfNhpi+Uv7A==",
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/@pipecat-ai/small-webrtc-transport/-/small-webrtc-transport-0.0.2.tgz",
"integrity": "sha512-9QQBjfAY0yh+ehDt6jX+bX7Ar5GFl+iI6QFS+JPRXeDYCj70bqmUgCYkScbgWzb5uRWZ8ORM+ueVkaLibe+Y4Q==",
"license": "BSD-2-Clause",
"dependencies": {
"@daily-co/daily-js": "^0.73.0",
"@daily-co/daily-js": "^0.77.0",
"dequal": "^2.0.3"
},
"peerDependencies": {

View File

@@ -19,6 +19,6 @@
},
"dependencies": {
"@pipecat-ai/client-js": "^0.3.2",
"@pipecat-ai/small-webrtc-transport": "^0.0.1"
"@pipecat-ai/small-webrtc-transport": "^0.0.2"
}
}

View File

@@ -1,12 +1,13 @@
import {
SmallWebRTCTransport
} from "@pipecat-ai/small-webrtc-transport";
import {Participant, RTVIClient, RTVIClientOptions} from "@pipecat-ai/client-js";
import {Participant, RTVIClient, RTVIClientOptions, Transport} from "@pipecat-ai/client-js";
class WebRTCApp {
private declare connectBtn: HTMLButtonElement;
private declare disconnectBtn: HTMLButtonElement;
private declare muteBtn: HTMLButtonElement;
private declare audioInput: HTMLSelectElement;
private declare videoInput: HTMLSelectElement;
@@ -32,12 +33,10 @@ class WebRTCApp {
private initializeRTVIClient(): void {
const transport = new SmallWebRTCTransport();
const RTVIConfig: RTVIClientOptions = {
// need to understand why it is complaining
// @ts-ignore
transport,
params: {
baseUrl: "/api/offer"
},
transport: transport as Transport,
enableMic: true,
enableCam: true,
callbacks: {
@@ -92,6 +91,7 @@ class WebRTCApp {
private setupDOMElements(): void {
this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement;
this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement;
this.muteBtn = document.getElementById('mute-btn') as HTMLButtonElement;
this.audioInput = document.getElementById('audio-input') as HTMLSelectElement;
this.videoInput = document.getElementById('video-input') as HTMLSelectElement;
@@ -118,6 +118,12 @@ class WebRTCApp {
let videoDevice = e.target?.value
this.rtviClient.updateCam(videoDevice)
})
this.muteBtn.addEventListener('click', () => {
let isCamEnabled = this.rtviClient.isCamEnabled
this.rtviClient.enableCam(!isCamEnabled)
this.muteBtn.textContent = isCamEnabled ? '📵' : '📷';
});
}
private log(message: string): void {

View File

@@ -89,6 +89,7 @@ button:disabled {
display: flex;
align-items: center;
justify-content: center;
position: relative;
}
#bot-video-container video {
@@ -97,6 +98,20 @@ button:disabled {
object-fit: cover;
}
#mute-btn {
position: absolute;
bottom: 10px;
right: 10px;
background-color: rgba(0, 0, 0, 0.6);
color: white;
border: none;
border-radius: 20px;
padding: 8px 12px;
cursor: pointer;
font-size: 16px;
z-index: 1;
}
.debug-panel {
background-color: #fff;
border-radius: 8px;

View File

@@ -3,4 +3,5 @@ fastapi[all]
uvicorn
aiortc
opencv-python
pipecat-ai[google,silero]
pipecat-ai[google,silero,webrtc]
pipecat-ai-small-webrtc-prebuilt

View File

@@ -8,6 +8,8 @@ import uvicorn
from bot import run_bot
from dotenv import load_dotenv
from fastapi import BackgroundTasks, FastAPI
from fastapi.responses import RedirectResponse
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
@@ -23,6 +25,14 @@ pcs_map: Dict[str, SmallWebRTCConnection] = {}
ice_servers = ["stun:stun.l.google.com:19302"]
# Mount the frontend at /
app.mount("/prebuilt", SmallWebRTCPrebuiltUI)
@app.get("/", include_in_schema=False)
async def root_redirect():
return RedirectResponse(url="/prebuilt/")
@app.post("/api/offer")
async def offer(request: dict, background_tasks: BackgroundTasks):

View File

@@ -47,7 +47,7 @@ canonical = [ "aiofiles~=24.1.0" ]
cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ]
cerebras = []
deepseek = []
daily = [ "daily-python~=0.16.1" ]
daily = [ "daily-python~=0.17.0" ]
deepgram = [ "deepgram-sdk~=3.8.0" ]
elevenlabs = [ "websockets~=13.1" ]
fal = [ "fal-client~=0.5.9" ]

View File

@@ -158,7 +158,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
msg = {
"transcript": text or " ", # Text must contain at least one character
"transcript": text,
"continue": continue_transcript,
"context_id": self._context_id,
"model_id": self.model_name,
@@ -287,7 +287,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
self._context_id = str(uuid.uuid4())
await self.create_audio_context(self._context_id)
msg = self._build_msg(text=text or " ") # Text must contain at least one character
msg = self._build_msg(text=text)
try:
await self._get_websocket().send(msg)

View File

@@ -164,8 +164,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
self,
*,
api_key: str,
base_url="generativelanguage.googleapis.com",
model="models/gemini-2.0-flash-exp",
base_url: str = "generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent",
model="models/gemini-2.0-flash-live-001",
voice_id: str = "Charon",
start_audio_paused: bool = False,
start_video_paused: bool = False,
@@ -179,8 +179,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
):
super().__init__(base_url=base_url, **kwargs)
self._last_sent_time = 0
self.api_key = api_key
self.base_url = base_url
self._api_key = api_key
self._base_url = base_url
self.set_model_name(model)
self._voice_id = voice_id
@@ -407,8 +407,8 @@ class GeminiMultimodalLiveLLMService(LLMService):
logger.info("Connecting to Gemini service")
try:
uri = f"wss://{self.base_url}/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key={self.api_key}"
logger.info(f"Connecting to {uri}")
logger.info(f"Connecting to wss://{self._base_url}")
uri = f"wss://{self._base_url}?key={self._api_key}"
self._websocket = await websockets.connect(uri=uri)
self._receive_task = self.create_task(self._receive_task_handler())
self._transcribe_audio_task = self.create_task(self._transcribe_audio_handler())

View File

@@ -0,0 +1,141 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import os
import json
from typing import Any, Dict, List, Mapping, Optional, Union
from loguru import logger
from pipecat.services.llm_service import LLMService
from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema
from mcp_run import Client
try:
from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
logger.error(
"In order to use mcp.run, you need to `pip install pipecat-ai[mcp_run]`. "
+ "Also, set `MCP_RUN_SESSION_ID` environment variable."
)
raise Exception(f"Missing module: {e}")
class MCPRun(Client):
def __init__(
self,
llm: LLMService,
mcp_run_session_id: Optional[str] = None,
**kwargs,
):
super().__init__(**kwargs)
self._client = Client()
self._mcp_run_session_id = mcp_run_session_id or os.getenv("MCP_RUN_SESSION_ID")
def convert_mcp_schema_to_pipecat(self, tool_name: str, tool_schema: dict[str, any]) -> FunctionSchema:
"""Convert an mcp.run tool schema to Pipecat's FunctionSchema format.
Args:
tool_name: The name of the tool
tool_schema: The mcp.run tool schema
Returns:
A FunctionSchema instance
"""
logger.debug(f"Converting schema for tool '{tool_name}'")
logger.debug(f"Original schema: {json.dumps(tool_schema, indent=2)}")
# Extract properties and required fields from the mcp.run schema
properties = tool_schema["input_schema"].get("properties", {})
required = tool_schema["input_schema"].get("required", [])
schema = FunctionSchema(
name=tool_name,
description=tool_schema["description"],
properties=properties,
required=required
)
logger.debug(f"Converted schema: {json.dumps(schema.to_default_dict(), indent=2)}")
return schema
def register_mcp_tools(self, llm) -> ToolsSchema:
"""Register all available mcp.run tools with the LLM service.
Args:
llm: The Pipecat LLM service to register tools with
Returns:
A ToolsSchema containing all registered tools
"""
async def mcp_tool_wrapper(function_name: str, tool_call_id: str, arguments: dict[str, any],
llm: any, context: any, result_callback: any) -> None:
"""Wrapper for mcp.run tool calls to match Pipecat's function call interface.
"""
logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}")
logger.debug(f"Tool arguments: {json.dumps(arguments, indent=2)}")
try:
# Call the mcp.run tool
logger.debug(f"Calling mcp.run tool '{function_name}'")
results = self._client.call_tool(function_name, params=arguments)
# Combine all content into a single response
response = ""
for i, content in enumerate(results.content):
logger.debug(f"Tool response chunk {i}: {content.text}")
response += content.text
logger.info(f"Tool '{function_name}' completed successfully")
logger.info(f"Final response: {response}")
# Send result back through callback
await result_callback(response)
except Exception as e:
error_msg = f"Error calling mcp.run tool {function_name}: {str(e)}"
logger.error(error_msg)
logger.exception("Full exception details:")
await result_callback(error_msg)
logger.debug("Starting registration of mcp.run tools")
tool_schemas: List[FunctionSchema] = []
# Get all available tools from mcp.run
available_tools = self._client.tools
logger.debug(f"Found {len(available_tools)} available tools")
for tool_name, tool in available_tools.items():
logger.debug(f"Processing tool: {tool_name}")
logger.debug(f"Tool description: {tool.description}")
try:
# Convert the schema
function_schema = self.convert_mcp_schema_to_pipecat(tool_name, {
"description": tool.description,
"input_schema": tool.input_schema
})
# Register the wrapped function
logger.debug(f"Registering function handler for '{tool_name}'")
llm.register_function(tool_name, mcp_tool_wrapper)
# Add to our list of schemas
tool_schemas.append(function_schema)
logger.debug(f"Successfully registered tool '{tool_name}'")
except Exception as e:
logger.error(f"Failed to register tool '{tool_name}': {str(e)}")
logger.exception("Full exception details:")
continue
logger.info(f"Completed registration of {len(tool_schemas)} tools")
tools_schema = ToolsSchema(standard_tools=tool_schemas)
return tools_schema

View File

@@ -105,7 +105,7 @@ class OpenAITTSService(TTSService):
extra_body["instructions"] = self._instructions
async with self._client.audio.speech.with_streaming_response.create(
input=text or " ", # Text must contain at least one character
input=text,
model=self.model_name,
voice=VALID_VOICES[self._voice_id],
response_format="pcm",

View File

@@ -269,7 +269,8 @@ class TTSService(AIService):
filter.reset_interruption()
text = filter.filter(text)
await self.process_generator(self.run_tts(text))
if text:
await self.process_generator(self.run_tts(text))
await self.stop_processing_metrics()

View File

@@ -37,7 +37,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.transports.base_transport import TransportParams
from pipecat.utils.time import nanoseconds_to_seconds
BOT_VAD_STOP_SECS = 0.3
BOT_VAD_STOP_SECS = 0.35
class BaseOutputTransport(FrameProcessor):

View File

@@ -17,13 +17,17 @@ from pydantic import BaseModel
from pipecat.frames.frames import (
CancelFrame,
EndFrame,
Frame,
InputAudioRawFrame,
InputImageRawFrame,
OutputImageRawFrame,
StartFrame,
TransportMessageFrame,
TransportMessageUrgentFrame,
UserImageRawFrame,
UserImageRequestFrame,
)
from pipecat.processors.frame_processor import FrameDirection
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import BaseTransport, TransportParams
@@ -59,9 +63,7 @@ class RawAudioTrack(AudioStreamTrack):
self._chunk_queue = deque()
def add_audio_bytes(self, audio_bytes: bytes):
"""
Adds bytes to the audio buffer and returns a Future that completes when the data is processed.
"""
"""Adds bytes to the audio buffer and returns a Future that completes when the data is processed."""
if len(audio_bytes) % self._bytes_per_10ms != 0:
raise ValueError("Audio bytes must be a multiple of 10ms size.")
future = asyncio.get_running_loop().create_future()
@@ -76,9 +78,7 @@ class RawAudioTrack(AudioStreamTrack):
return future
async def recv(self):
"""
Returns the next audio frame, generating silence if needed.
"""
"""Returns the next audio frame, generating silence if needed."""
# Compute required wait time for synchronization
if self._timestamp > 0:
wait = self._start + (self._timestamp / self._sample_rate) - time.time()
@@ -179,8 +179,7 @@ class SmallWebRTCClient:
await self._handle_app_message(message)
def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
"""
Convert a given frame to RGB format based on the input format.
"""Convert a given frame to RGB format based on the input format.
Args:
frame_array (np.ndarray): The input frame.
@@ -203,8 +202,7 @@ class SmallWebRTCClient:
return cv2.cvtColor(frame_array, conversion_code)
async def read_video_frame(self):
"""
Reads a video frame from the given MediaStreamTrack, converts it to RGB,
"""Reads a video frame from the given MediaStreamTrack, converts it to RGB,
and creates an InputImageRawFrame.
"""
while True:
@@ -242,9 +240,7 @@ class SmallWebRTCClient:
yield image_frame
async def read_audio_frame(self):
"""
Reads 20ms of audio from the given MediaStreamTrack and creates an InputAudioRawFrame.
"""
"""Reads 20ms of audio from the given MediaStreamTrack and creates an InputAudioRawFrame."""
while True:
if self._audio_input_track is None:
await asyncio.sleep(0.01)
@@ -379,6 +375,13 @@ class SmallWebRTCInputTransport(BaseInputTransport):
self._params = params
self._receive_audio_task = None
self._receive_video_task = None
self._image_requests = {}
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, UserImageRequestFrame):
await self.request_participant_image(frame)
async def start(self, frame: StartFrame):
await super().start(frame)
@@ -424,6 +427,22 @@ class SmallWebRTCInputTransport(BaseInputTransport):
if video_frame:
await self.push_frame(video_frame)
# Check if there are any pending image requests and create UserImageRawFrame
if self._image_requests:
for req_id, request_frame in list(self._image_requests.items()):
# Create UserImageRawFrame using the current video frame
image_frame = UserImageRawFrame(
user_id=request_frame.user_id,
request=request_frame,
image=video_frame.image,
size=video_frame.size,
format=video_frame.format,
)
# Push the frame to the pipeline
await self.push_frame(image_frame)
# Remove from pending requests
del self._image_requests[req_id]
except Exception as e:
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
@@ -432,6 +451,24 @@ class SmallWebRTCInputTransport(BaseInputTransport):
frame = TransportMessageUrgentFrame(message=message)
await self.push_frame(frame)
# Add this method similar to DailyInputTransport.request_participant_image
async def request_participant_image(self, frame: UserImageRequestFrame):
"""Requests an image frame from the participant's video stream.
When a UserImageRequestFrame is received, this method will store the request
and the next video frame received will be converted to a UserImageRawFrame.
"""
logger.debug(f"Requesting image from participant: {frame.user_id}")
# Store the request
request_id = f"{frame.function_name}:{frame.tool_call_id}"
self._image_requests[request_id] = frame
# If we're not already receiving video, try to get a frame now
if not self._receive_video_task and self._params.camera_in_enabled:
# Start video reception if it's not already running
self._receive_video_task = self.create_task(self._receive_video())
class SmallWebRTCOutputTransport(BaseOutputTransport):
def __init__(

View File

@@ -7,15 +7,22 @@
import asyncio
import json
import time
from enum import Enum
from typing import Any, Optional
from typing import Any, Literal, Optional, Union
from av.frame import Frame
from loguru import logger
from pydantic import BaseModel, TypeAdapter
from pipecat.utils.base_object import BaseObject
try:
from aiortc import RTCConfiguration, RTCIceServer, RTCPeerConnection, RTCSessionDescription
from aiortc import (
MediaStreamTrack,
RTCConfiguration,
RTCIceServer,
RTCPeerConnection,
RTCSessionDescription,
)
from aiortc.rtcrtpreceiver import RemoteStreamTrack
except ModuleNotFoundError as e:
logger.error(f"Exception: {e}")
@@ -23,10 +30,57 @@ except ModuleNotFoundError as e:
raise Exception(f"Missing module: {e}")
SIGNALLING_TYPE = "signalling"
AUDIO_TRANSCEIVER_INDEX = 0
VIDEO_TRANSCEIVER_INDEX = 1
class SignallingMessage(Enum):
RENEGOTIATE = "renegotiate"
class TrackStatusMessage(BaseModel):
type: Literal["trackStatus"]
receiver_index: int
enabled: bool
class RenegotiateMessage(BaseModel):
type: Literal["renegotiate"] = "renegotiate"
class SignallingMessage:
Inbound = Union[TrackStatusMessage] # in case we need to add new messages in the future
outbound = Union[RenegotiateMessage]
class SmallWebRTCTrack:
def __init__(self, track: MediaStreamTrack):
self._track = track
self._enabled = True
def set_enabled(self, enabled: bool) -> None:
self._enabled = enabled
def is_enabled(self) -> bool:
return self._enabled
async def discard_old_frames(self):
remote_track = self._track
if isinstance(remote_track, RemoteStreamTrack):
if not hasattr(remote_track, "_queue") or not isinstance(
remote_track._queue, asyncio.Queue
):
print("Warning: _queue does not exist or has changed in aiortc.")
return
logger.debug("Discarding old frames")
while not remote_track._queue.empty():
remote_track._queue.get_nowait() # Remove the oldest frame
remote_track._queue.task_done()
async def recv(self) -> Optional[Frame]:
if not self._enabled:
return None
return await self._track.recv()
def __getattr__(self, name):
# Forward other attribute/method calls to the underlying track
return getattr(self._track, name)
class SmallWebRTCConnection(BaseObject):
@@ -37,6 +91,12 @@ class SmallWebRTCConnection(BaseObject):
else:
self.ice_servers = []
self._connect_invoked = False
self._track_map = {}
self._track_getters = {
AUDIO_TRANSCEIVER_INDEX: self.audio_input_track,
VIDEO_TRANSCEIVER_INDEX: self.video_input_track,
}
self._initialize()
# Register supported handlers. The user will only be able to register
@@ -68,7 +128,6 @@ class SmallWebRTCConnection(BaseObject):
self._pc = RTCPeerConnection(rtc_config)
self._pc_id = self.name
self._setup_listeners()
self._tracks = set()
self._data_channel = None
self._renegotiation_in_progress = False
self._last_received_time = None
@@ -96,7 +155,10 @@ class SmallWebRTCConnection(BaseObject):
self._last_received_time = time.time()
else:
json_message = json.loads(message)
await self._call_event_handler("app-message", json_message)
if json_message["type"] == SIGNALLING_TYPE and json_message.get("message"):
self._handle_signalling_message(json_message["message"])
else:
await self._call_event_handler("app-message", json_message)
except Exception as e:
logger.exception(f"Error parsing JSON message {message}, {e}")
@@ -121,13 +183,11 @@ class SmallWebRTCConnection(BaseObject):
@self._pc.on("track")
async def on_track(track):
logger.debug(f"Track {track.kind} received")
self._tracks.add(track)
await self._call_event_handler("track-started", track)
@track.on("ended")
async def on_ended():
logger.debug(f"Track {track.kind} ended")
self._tracks.discard(track)
await self._call_event_handler("track-ended", track)
async def _create_answer(self, sdp: str, type: str):
@@ -148,17 +208,6 @@ class SmallWebRTCConnection(BaseObject):
async def initialize(self, sdp: str, type: str):
await self._create_answer(sdp, type)
async def discard_old_frames(self, remote_track: RemoteStreamTrack):
if not hasattr(remote_track, "_queue") or not isinstance(
remote_track._queue, asyncio.Queue
):
print("Warning: _queue does not exist or has changed in aiortc.")
return
logger.debug("Discarding old frames")
while not remote_track._queue.empty():
remote_track._queue.get_nowait() # Remove the oldest frame
remote_track._queue.task_done()
async def connect(self):
self._connect_invoked = True
# If we already connected, trigger again the connected event
@@ -166,9 +215,7 @@ class SmallWebRTCConnection(BaseObject):
await self._call_event_handler("connected")
# We are renegotiating here, because likely we have loose the first video frames
# and aiortc does not handle that pretty well.
remove_video_track = self.video_input_track()
if isinstance(remove_video_track, RemoteStreamTrack):
await self.discard_old_frames(remove_video_track)
await self.video_input_track().discard_old_frames()
self.ask_to_renegotiate()
async def renegotiate(self, sdp: str, type: str, restart_pc: bool = False):
@@ -228,6 +275,7 @@ class SmallWebRTCConnection(BaseObject):
if self._pc:
await self._pc.close()
self._message_queue.clear()
self._track_map = {}
def get_answer(self):
if not self._answer:
@@ -267,29 +315,38 @@ class SmallWebRTCConnection(BaseObject):
return (time.time() - self._last_received_time) < 3
def audio_input_track(self):
if self._track_map.get(AUDIO_TRANSCEIVER_INDEX):
return self._track_map[AUDIO_TRANSCEIVER_INDEX]
# Transceivers always appear in creation-order for both peers
# For now we are only considering that we are going to have 02 transceivers,
# one for audio and one for video
transceivers = self._pc.getTransceivers()
if len(transceivers) == 0 or not transceivers[0].receiver:
if len(transceivers) == 0 or not transceivers[AUDIO_TRANSCEIVER_INDEX].receiver:
logger.warning("No audio transceiver is available")
return None
return transceivers[0].receiver.track
track = transceivers[AUDIO_TRANSCEIVER_INDEX].receiver.track
audio_track = SmallWebRTCTrack(track) if track else None
self._track_map[AUDIO_TRANSCEIVER_INDEX] = audio_track
return audio_track
def video_input_track(self):
if self._track_map.get(VIDEO_TRANSCEIVER_INDEX):
return self._track_map[VIDEO_TRANSCEIVER_INDEX]
# Transceivers always appear in creation-order for both peers
# For now we are only considering that we are going to have 02 transceivers,
# one for audio and one for video
transceivers = self._pc.getTransceivers()
if len(transceivers) <= 1 or not transceivers[1].receiver:
if len(transceivers) <= 1 or not transceivers[VIDEO_TRANSCEIVER_INDEX].receiver:
logger.warning("No video transceiver is available")
return None
return transceivers[1].receiver.track
def tracks(self):
return self._tracks
track = transceivers[VIDEO_TRANSCEIVER_INDEX].receiver.track
video_track = SmallWebRTCTrack(track) if track else None
self._track_map[VIDEO_TRANSCEIVER_INDEX] = video_track
return video_track
def send_app_message(self, message: Any):
json_message = json.dumps(message)
@@ -305,5 +362,17 @@ class SmallWebRTCConnection(BaseObject):
self._renegotiation_in_progress = True
self.send_app_message(
{"type": SIGNALLING_TYPE, "message": SignallingMessage.RENEGOTIATE.value}
{"type": SIGNALLING_TYPE, "message": RenegotiateMessage().model_dump()}
)
def _handle_signalling_message(self, message):
logger.debug(f"Signalling message received: {message}")
inbound_adapter = TypeAdapter(SignallingMessage.Inbound)
signalling_message = inbound_adapter.validate_python(message)
match signalling_message:
case TrackStatusMessage():
track = (
self._track_getters.get(signalling_message.receiver_index) or (lambda: None)
)()
if track:
track.set_enabled(signalling_message.enabled)