diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index db0bcb0e3..59a02f767 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -6,11 +6,13 @@ on: - main paths: - "examples/simple-chatbot/client/android/**" + - "examples/p2p-webrtc/video-transform/client/android/**" pull_request: branches: - "**" paths: - "examples/simple-chatbot/client/android/**" + - "examples/p2p-webrtc/video-transform/client/android/**" workflow_dispatch: inputs: sdk_git_ref: @@ -23,7 +25,7 @@ concurrency: jobs: sdk: - name: "Simple chatbot demo" + name: "Demo apps" runs-on: ubuntu-latest steps: - name: Checkout repo @@ -37,12 +39,22 @@ jobs: distribution: 'temurin' java-version: '17' - - name: Build demo app + - name: "Example app: Simple Chatbot" working-directory: examples/simple-chatbot/client/android run: ./gradlew :simple-chatbot-client:assembleDebug - - name: Upload demo APK + - name: Upload Simple Chatbot APK uses: actions/upload-artifact@v4 with: name: Simple Chatbot Android Client path: examples/simple-chatbot/client/android/simple-chatbot-client/build/outputs/apk/debug/simple-chatbot-client-debug.apk + + - name: "Example app: Small WebRTC Client" + working-directory: examples/p2p-webrtc/video-transform/client/android + run: ./gradlew :small-webrtc-client:assembleDebug + + - name: Upload Small WebRTC APK + uses: actions/upload-artifact@v4 + with: + name: Small WebRTC Android Client + path: examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/build/outputs/apk/debug/small-webrtc-client-debug.apk diff --git a/CHANGELOG.md b/CHANGELOG.md index 538b01f1e..41d1b6236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,177 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.71] - 2025-06-10 + +### Added + +- Adds a parameter called `additional_span_attributes` to PipelineTask that + lets you add any additional attributes you'd like to the conversation span. + +### Fixed + +- Fixed an issue with `CartesiaSTTService` initialization. + +## [0.0.70] - 2025-06-10 + +### Added + +- Added `ExotelFrameSerializer` to handle telephony calls via Exotel. + +- Added the option `informal` to `TranslationConfig` on Gladia config. + Allowing to force informal language forms when available. + +- Added `CartesiaSTTService` which is a websocket based implementation to + transcribe audio. Added a foundational example in + `13f-cartesia-transcription.py` + +- Added an `websocket` example, showing how to use the new Pipecat client + `WebsocketTransport` to connect with Pipecat `FastAPIWebsocketTransport` or + `WebsocketServerTransport`. + +- Added language support to `RimeHttpTTSService`. Extended languages to include + German and French for both `RimeTTSService` and `RimeHttpTTSService`. + +### Changed + +- Upgraded `daily-python` to 0.19.2. + +- Make `PipelineTask.add_observer()` synchronous. This allows callers to call it + before doing the work of running the `PipelineTask` (i.e. without invoking + `PipelineTask.set_event_loop()` first). + +- Pipecat 0.0.69 forced `uvloop` event loop on Linux on macOS. Unfortunately, + this is causing issue in some systems. So, `uvloop` is not enabled by default + anymore. If you want to use `uvloop` you can just set the `asyncio` event + policy before starting your agent with: + +```python +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) +``` + +### Fixed + +- Fixed an issue with various TTS services that would cause audio glitches at + the start of every bot turn. + +- Fixed an `ElevenLabsTTSService` issue where a context warning was printed + when pushing a `TTSSpeakFrame`. + +- Fixed an `AssemblyAISTTService` issue that could cause unexpected behavior + when yielding empty `Frame()`s. + +- Fixed an issue where `OutputAudioRawFrame.transport_destination` was being + reset to `None` instead of retaining its intended value before sending the + audio frame to `write_audio_frame`. + +- Fixed a typo in Livekit transport that prevented initialization. + +## [0.0.69] - 2025-06-02 "AI Engineer World's Fair release" ✨ + +### Added + +- Added a new frame `FunctionCallsStartedFrame`. This frame is pushed both + upstream and downstream from the LLM service to indicate that one or more + function calls are going to be executed. + +- Added LLM services `on_function_calls_started` event. This event will be + triggered when the LLM service receives function calls from the model and is + going to start executing them. + +- Function calls can now be executed sequentially (in the order received in the + completion) by passing `run_in_parallel=False` when creating your LLM + service. By default, if the LLM completion returns 2 or more function calls + they run concurrently. In both cases, concurrently and sequentially, a new LLM + completion will run when the last function call finishes. + +- Added OpenTelemetry tracing for `GeminiMultimodalLiveLLMService` and + `OpenAIRealtimeBetaLLMService`. + +- Added initial support for interruption strategies, which determine if the user + should interrupt the bot while the bot is speaking. Interruption strategies + can be based on factors such as audio volume or the number of words spoken by + the user. These can be specified via the new `interruption_strategies` field + in `PipelineParams`. A new `MinWordsInterruptionStrategy` strategy has been + introduced which triggers an interruption if the user has spoken a minimum + number of words. If no interruption strategies are specified, the normal + interruption behavior applies. If multiple strategies are provided, the first + one that evaluates to true will trigger the interruption. + +- `BaseInputTransport` now handles `StopFrame`. When a `StopFrame` is received + the transport will pause sending frames downstream until a new `StartFrame` is + received. This allows the transport to be reused (keeping the same connection) + in a different pipeline. + +- Updated AssemblyAI STT service to support their latest streaming + speech-to-text model with improved transcription latency and endpointing. + +- You can now access STT service results through the new + `TranscriptionFrame.result` and `InterimTranscriptionFrame.result` field. This + is useful in case you use some specific settings for the STT and you want to + access the STT results. + +- The examples runner is now public from the `pipecat.examples` package. This + allows everyone to build their own examples and run them easily. + +- It is now possible to push `OutputDTMFFrame` or `OutputDTMFUrgentFrame` with + `DailyTransport`. This will be sent properly if a Daily dial-out connection + has been established. + +- Added `OutputDTMFUrgentFrame` to send a DTMF keypress quickly. The previous + `OutputDTMFFrame` queues the keypress with the rest of data frames. + +- Added `DTMFAggregator`, which aggregates keypad presses into + `TranscriptionFrame`s. Aggregation occurs after a timeout, termination key + press, or user interruption. You can specify the prefix of the + `TranscriptionFrame`. + +- Added new functions `DailyTransport.start_transcription()` and + `DailyTransport.stop_transcription()` to be able to start and stop Daily + transcription dynamically (maybe with different settings). + +### Changed + +- Reverted the default model for `GeminiMultimodalLiveLLMService` back to + `models/gemini-2.0-flash-live-001`. + `gemini-2.5-flash-preview-native-audio-dialog` has inconsistent performance. + You can opt in to using this model by setting the `model` arg. + +- Function calls are now cancelled by default if there's an interruption. To + disable this behavior you can set `cancel_on_interruption=False` when + registering the function call. Since function calls are executed as tasks you + can tell if a function call has been cancelled by catching the + `asyncio.CancelledError` exception (and don't forget to raise it again!). + +- Updated OpenTelemetry tracing attribute `metrics.ttfb_ms` to `metrics.ttfb`. + The attribute reports TTFB in seconds. + +### Deprecated + +- `DailyTransport.send_dtmf()` is deprecated, push an `OutputDTMFFrame` or an + `OutputDTMFUrgentFrame` instead. + +### Fixed + +- Fixed an issue with `ElevenLabsTTSService` where long responses would + continue generating output even after an interruption. + +- Fixed an issue with the `OpenAILLMContext` where non-Roman characters were + being incorrectly encoded as Unicode escape sequences. This was a logging + issue and did not impact the actual conversation. + +- In `AWSBedrockLLMService`, worked around a possible bug in AWS Bedrock where + a `toolConfig` is required if there has been previous tool use in the + messages array. This workaround includes a no_op factory function call is + used to satisfy the requirement. + +- Fixed `WebsocketClientTransport` to use `FrameProcessorSetup.task_manager` + instead of `StartFrame.task_manager`. + +### Performance + +- Use `uvloop` as the new event loop on Linux and macOS systems. + +## [0.0.68] - 2025-05-28 ### Added @@ -89,6 +259,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Upgraded `daily-python` to 0.19.1. + +- ⚠️ Updated `SmallWebRTCTransport` to align with how other transports handle + `on_client_disconnected`. Now, when the connection is closed and no reconnection + is attempted, `on_client_disconnected` is called instead of `on_client_close`. The + `on_client_close` callback is no longer used, use `on_client_disconnected` instead. + +- Check if `PipelineTask` has already been cancelled. + +- Don't raise an exception if event handler is not registered. + - Upgraded `deepgram-sdk` to 4.1.0. - Updated `GoogleTTSService` to use Google's streaming TTS API. The default @@ -147,6 +328,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `DailyTransport` issue that was not allow capturing video frames if + framerate was greater than zero. + - Fixed a `DeegramSTTService` connection issue when the user provided their own `LiveOptions`. @@ -173,6 +357,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- It is now possible to run all (or most) foundational example with multiple + transports. By default, they run with P2P (Peer-To-Peer) WebRTC so you can try + everything locally. You can also run them with Daily or even with a Twilio + phone number. + - Added foundation examples `07y-interruptible-minimax.py` and `07z-interruptible-sarvam.py`to show how to use the `MiniMaxHttpTTSService` and `SarvamTTSService`, respectively. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..9e1a65b34 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +prune docs +prune examples +prune scripts +prune tests diff --git a/README.md b/README.md index 6833cf7c5..14a37033b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ You can connect to Pipecat from any platform using our official SDKs: | Category | Services | | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [AWS](https://docs.pipecat.ai/server/services/stt/aws), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Cartesia](https://docs.pipecat.ai/server/services/stt/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Soniox](https://docs.pipecat.ai/server/services/stt/soniox), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [AWS](https://docs.pipecat.ai/server/services/llm/aws), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Qwen](https://docs.pipecat.ai/server/services/llm/qwen), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | | Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [MiniMax](https://docs.pipecat.ai/server/services/tts/minimax), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [Piper](https://docs.pipecat.ai/server/services/tts/piper), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [Sarvam](https://docs.pipecat.ai/server/services/tts/sarvam), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | | Speech-to-Speech | [AWS Nova Sonic](https://docs.pipecat.ai/server/services/s2s/aws), [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | diff --git a/examples/foundational/01-say-one-thing-piper.py b/examples/foundational/01-say-one-thing-piper.py index 2c6d6eebb..5093a9612 100644 --- a/examples/foundational/01-say-one-thing-piper.py +++ b/examples/foundational/01-say-one-thing-piper.py @@ -16,23 +16,25 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.piper.tts import PiperTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_out_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_out_enabled=True), + "webrtc": lambda: TransportParams(audio_out_enabled=True), +} - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create an HTTP session async with aiohttp.ClientSession() as session: @@ -47,12 +49,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/01-say-one-thing-rime.py b/examples/foundational/01-say-one-thing-rime.py index 46efbb3cd..67542e596 100644 --- a/examples/foundational/01-say-one-thing-rime.py +++ b/examples/foundational/01-say-one-thing-rime.py @@ -16,24 +16,25 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.rime.tts import RimeHttpTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_out_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_out_enabled=True), + "webrtc": lambda: TransportParams(audio_out_enabled=True), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - ), - ) - # Create an HTTP session async with aiohttp.ClientSession() as session: tts = RimeHttpTTSService( @@ -49,12 +50,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index fbbf23b6c..4a34a3c3d 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -15,23 +15,25 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_out_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_out_enabled=True), + "webrtc": lambda: TransportParams(audio_out_enabled=True), +} - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), @@ -45,12 +47,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index 732783b05..1b5c1b45a 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -77,37 +77,36 @@ async def configure_livekit(): async def main(): - async with aiohttp.ClientSession() as session: - (url, token, room_name) = await configure_livekit() + (url, token, room_name) = await configure_livekit() - transport = LiveKitTransport( - url=url, - token=token, - room_name=room_name, - params=LiveKitParams(audio_out_enabled=True), - ) + transport = LiveKitTransport( + url=url, + token=token, + room_name=room_name, + params=LiveKitParams(audio_out_enabled=True), + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - runner = PipelineRunner() + runner = PipelineRunner() - task = PipelineTask(Pipeline([tts, transport.output()])) + task = PipelineTask(Pipeline([tts, transport.output()])) - # Register an event handler so we can play the audio when the - # participant joins. - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant_id): - await asyncio.sleep(1) - await task.queue_frame( - TextFrame( - "Hello there! How are you doing today? Would you like to talk about the weather?" - ) + # Register an event handler so we can play the audio when the + # participant joins. + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant_id): + await asyncio.sleep(1) + await task.queue_frame( + TextFrame( + "Hello there! How are you doing today? Would you like to talk about the weather?" ) + ) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-fastpitch.py index effed6f01..0795097fb 100644 --- a/examples/foundational/01c-fastpitch.py +++ b/examples/foundational/01c-fastpitch.py @@ -15,23 +15,25 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.riva.tts import FastPitchTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_out_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_out_enabled=True), + "webrtc": lambda: TransportParams(audio_out_enabled=True), +} - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY")) @@ -42,12 +44,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): await task.queue_frames([TTSSpeakFrame(f"Hello there!"), EndFrame()]) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index 3162ffef4..b76385d7d 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -16,23 +16,25 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_out_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_out_enabled=True), + "webrtc": lambda: TransportParams(audio_out_enabled=True), +} - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), @@ -55,12 +57,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): await task.queue_frames([LLMMessagesFrame(messages), EndFrame()]) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/03-still-frame.py b/examples/foundational/03-still-frame.py index 3b7ef84e3..106250e11 100644 --- a/examples/foundational/03-still-frame.py +++ b/examples/foundational/03-still-frame.py @@ -16,25 +16,31 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.fal.image import FalImageGenService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + ), + "webrtc": lambda: TransportParams( + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + ), +} - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - video_out_enabled=True, - video_out_width=1024, - video_out_height=1024, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create an HTTP session async with aiohttp.ClientSession() as session: @@ -54,18 +60,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/03b-still-frame-imagen.py b/examples/foundational/03b-still-frame-imagen.py index 783fd9be8..83c68ef19 100644 --- a/examples/foundational/03b-still-frame-imagen.py +++ b/examples/foundational/03b-still-frame-imagen.py @@ -15,25 +15,31 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.google.image import GoogleImageGenService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + ), + "webrtc": lambda: TransportParams( + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + ), +} - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - video_out_enabled=True, - video_out_width=1024, - video_out_height=1024, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") imagegen = GoogleImageGenService( api_key=os.getenv("GOOGLE_API_KEY"), @@ -54,18 +60,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/04-transports-small-webrtc.py b/examples/foundational/04-transports-small-webrtc.py index 29ac495ea..627c13ad1 100644 --- a/examples/foundational/04-transports-small-webrtc.py +++ b/examples/foundational/04-transports-small-webrtc.py @@ -5,10 +5,17 @@ # import argparse +import asyncio import os +from contextlib import asynccontextmanager +from typing import Dict +import uvicorn from dotenv import load_dotenv +from fastapi import BackgroundTasks, FastAPI +from fastapi.responses import RedirectResponse from loguru import logger +from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline @@ -20,14 +27,29 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_transport import TransportParams from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.network.webrtc_connection import IceServer, SmallWebRTCConnection load_dotenv(override=True) +app = FastAPI() -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# Store connections by pc_id +pcs_map: Dict[str, SmallWebRTCConnection] = {} + +ice_servers = [ + IceServer( + urls="stun:stun.l.google.com:19302", + ) +] + +# Mount the frontend at / +app.mount("/client", SmallWebRTCPrebuiltUI) + + +async def run_example(webrtc_connection: SmallWebRTCConnection): logger.info(f"Starting bot") + # Create a transport using the WebRTC connection transport = SmallWebRTCTransport( webrtc_connection=webrtc_connection, params=TransportParams( @@ -88,10 +110,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() runner = PipelineRunner(handle_sigint=False) @@ -99,7 +117,58 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac await runner.run(task) -if __name__ == "__main__": - from run import main +@app.get("/", include_in_schema=False) +async def root_redirect(): + return RedirectResponse(url="/client/") - main() + +@app.post("/api/offer") +async def offer(request: dict, background_tasks: BackgroundTasks): + pc_id = request.get("pc_id") + + if pc_id and pc_id in pcs_map: + pipecat_connection = pcs_map[pc_id] + logger.info(f"Reusing existing connection for pc_id: {pc_id}") + await pipecat_connection.renegotiate( + sdp=request["sdp"], + type=request["type"], + restart_pc=request.get("restart_pc", False), + ) + else: + pipecat_connection = SmallWebRTCConnection(ice_servers) + await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"]) + + @pipecat_connection.event_handler("closed") + async def handle_disconnected(webrtc_connection: SmallWebRTCConnection): + logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}") + pcs_map.pop(webrtc_connection.pc_id, None) + + # Run example function with SmallWebRTC transport arguments. + background_tasks.add_task(run_example, pipecat_connection) + + answer = pipecat_connection.get_answer() + # Updating the peer connection inside the map + pcs_map[answer["pc_id"]] = pipecat_connection + + return answer + + +@asynccontextmanager +async def lifespan(app: FastAPI): + yield # Run app + coros = [pc.disconnect() for pc in pcs_map.values()] + await asyncio.gather(*coros) + pcs_map.clear() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Pipecat Bot Runner") + parser.add_argument( + "--host", default="localhost", help="Host for HTTP server (default: localhost)" + ) + parser.add_argument( + "--port", type=int, default=7860, help="Port for HTTP server (default: 7860)" + ) + args = parser.parse_args() + + uvicorn.run(app, host=args.host, port=args.port) diff --git a/examples/foundational/04a-transports-daily.py b/examples/foundational/04a-transports-daily.py index 98060dbab..a968c3abb 100644 --- a/examples/foundational/04a-transports-daily.py +++ b/examples/foundational/04a-transports-daily.py @@ -9,11 +9,11 @@ import os import sys import aiohttp -from daily_runner import configure from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.daily_runner import configure from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/examples/foundational/04b-transports-livekit.py b/examples/foundational/04b-transports-livekit.py index ef2679ab8..3cd5c7eea 100644 --- a/examples/foundational/04b-transports-livekit.py +++ b/examples/foundational/04b-transports-livekit.py @@ -10,7 +10,6 @@ import json import os import sys -import aiohttp from deepgram import LiveOptions from dotenv import load_dotenv from livekit import api @@ -104,101 +103,100 @@ async def configure_livekit(): async def main(): - async with aiohttp.ClientSession() as session: - (url, token, room_name) = await configure_livekit() + (url, token, room_name) = await configure_livekit() - transport = LiveKitTransport( - url=url, - token=token, - room_name=room_name, - params=LiveKitParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + transport = LiveKitTransport( + url=url, + token=token, + room_name=room_name, + params=LiveKitParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - live_options=LiveOptions( - vad_events=True, - ), - ) + stt = DeepgramSTTService( + api_key=os.getenv("DEEPGRAM_API_KEY"), + live_options=LiveOptions( + vad_events=True, + ), + ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. " - "Your goal is to demonstrate your capabilities in a succinct way. " - "Your output will be converted to audio so don't include special characters in your answers. " - "Respond to what the user said in a creative and helpful way.", - }, - ] + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. " + "Your goal is to demonstrate your capabilities in a succinct way. " + "Your output will be converted to audio so don't include special characters in your answers. " + "Respond to what the user said in a creative and helpful way.", + }, + ] - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) - runner = PipelineRunner() + runner = PipelineRunner() - task = PipelineTask( - Pipeline( - [ - transport.input(), - stt, - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ], - ), - params=PipelineParams( - allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True - ), - ) + task = PipelineTask( + Pipeline( + [ + transport.input(), + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ], + ), + params=PipelineParams( + allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True + ), + ) - # Register an event handler so we can play the audio when the - # participant joins. - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant_id): - await asyncio.sleep(1) - await task.queue_frame( - TextFrame( - "Hello there! How are you doing today? Would you like to talk about the weather?" - ) + # Register an event handler so we can play the audio when the + # participant joins. + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant_id): + await asyncio.sleep(1) + await task.queue_frame( + TextFrame( + "Hello there! How are you doing today? Would you like to talk about the weather?" ) + ) - # Register an event handler to receive data from the participant via text chat - # in the LiveKit room. This will be used to as transcription frames and - # interrupt the bot and pass it to llm for processing and - # then pass back to the participant as audio output. - @transport.event_handler("on_data_received") - async def on_data_received(transport, data, participant_id): - logger.info(f"Received data from participant {participant_id}: {data}") - # convert data from bytes to string - json_data = json.loads(data) + # Register an event handler to receive data from the participant via text chat + # in the LiveKit room. This will be used to as transcription frames and + # interrupt the bot and pass it to llm for processing and + # then pass back to the participant as audio output. + @transport.event_handler("on_data_received") + async def on_data_received(transport, data, participant_id): + logger.info(f"Received data from participant {participant_id}: {data}") + # convert data from bytes to string + json_data = json.loads(data) - await task.queue_frames( - [ - BotInterruptionFrame(), - UserStartedSpeakingFrame(), - TranscriptionFrame( - user_id=participant_id, - timestamp=json_data["timestamp"], - text=json_data["message"], - ), - UserStoppedSpeakingFrame(), - ], - ) + await task.queue_frames( + [ + BotInterruptionFrame(), + UserStartedSpeakingFrame(), + TranscriptionFrame( + user_id=participant_id, + timestamp=json_data["timestamp"], + text=json_data["message"], + ), + UserStoppedSpeakingFrame(), + ], + ) - await runner.run(task) + await runner.run(task) if __name__ == "__main__": diff --git a/examples/foundational/04c-transports-daily-audio-source.py b/examples/foundational/04c-transports-daily-audio-source.py deleted file mode 100644 index 00cb8a603..000000000 --- a/examples/foundational/04c-transports-daily-audio-source.py +++ /dev/null @@ -1,111 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys - -import aiohttp -from daily_runner import configure -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_in_passthrough=False, - audio_out_enabled=True, - audio_out_sample_rate=16000, - transcription_enabled=False, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - stt = DeepgramSTTService( - api_key=os.getenv("DEEPGRAM_API_KEY"), - live_options=LiveOptions(language=Language.EN), - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_audio(participant["id"]) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index e91c49eb5..d4c816714 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -28,9 +28,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaHttpTTSService from pipecat.services.fal.image import FalImageGenService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -64,7 +63,26 @@ class MonthPrepender(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + ), + "webrtc": lambda: TransportParams( + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): """Run the Calendar Month Narration bot using WebRTC transport. Args: @@ -73,17 +91,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac """ logger.info(f"Starting bot") - # Create a transport using the WebRTC connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - video_out_enabled=True, - video_out_width=1024, - video_out_height=1024, - ), - ) - # Create an HTTP session for API calls async with aiohttp.ClientSession() as session: llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) @@ -159,18 +166,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() # Run the pipeline - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 921144e85..b660459cf 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -26,9 +26,9 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -53,17 +53,30 @@ class MetricsLogger(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -117,17 +130,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index a9d4e16b5..61ba1c6f4 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -26,9 +26,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -68,20 +67,31 @@ class ImageSyncAggregator(FrameProcessor): await self.push_frame(frame) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_out_enabled=True, - video_out_width=1024, - video_out_height=1024, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -139,17 +149,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07-interruptible-cartesia-http.py b/examples/foundational/07-interruptible-cartesia-http.py index 957938df2..75f8a0ddb 100644 --- a/examples/foundational/07-interruptible-cartesia-http.py +++ b/examples/foundational/07-interruptible-cartesia-http.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaHttpTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -88,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 29ac495ea..9429f5a33 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -88,18 +100,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 0b352719f..2a5e72c1a 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -27,9 +27,9 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -43,17 +43,30 @@ def get_session_history(session_id: str) -> BaseChatMessageHistory: return message_store[session_id] -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -120,18 +133,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index 945cdc447..f228dcecd 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -24,23 +24,34 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService( api_key=os.getenv("DEEPGRAM_API_KEY"), @@ -101,18 +112,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index 2a707da4a..f523f2aa5 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -85,18 +98,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py index 9fadd2cf5..eb7e03bf5 100644 --- a/examples/foundational/07d-interruptible-elevenlabs-http.py +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -19,24 +19,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsHttpTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create an HTTP session async with aiohttp.ClientSession() as session: @@ -92,18 +105,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 885a034c0..945c0e47b 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -88,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index 5ac99640e..534cd6927 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.playht.tts import PlayHTHttpTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = PlayHTHttpTTSService( @@ -89,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 321a876c0..1bbb04fe2 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -19,25 +19,37 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.playht.tts import PlayHTTTSService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = PlayHTTTSService( @@ -91,18 +103,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 5c2f42247..0da653f4d 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.azure.llm import AzureLLMService from pipecat.services.azure.stt import AzureSTTService from pipecat.services.azure.tts import AzureTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = AzureSTTService( api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"), @@ -95,18 +107,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index c89baa068..5d028603e 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.openai.stt import OpenAISTTService from pipecat.services.openai.tts import OpenAITTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = OpenAISTTService( api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe", @@ -90,18 +102,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 90daf9311..e488ad6db 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -19,25 +19,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openpipe.llm import OpenPipeLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -94,18 +106,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 1ca56b865..32c7a6348 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -19,25 +19,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.xtts.tts import XTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - # Create an HTTP session async with aiohttp.ClientSession() as session: stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -92,18 +104,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 757ae2697..6975cfec7 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -20,25 +20,37 @@ from pipecat.services.gladia.config import GladiaInputParams, LanguageConfig from pipecat.services.gladia.stt import GladiaSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = GladiaSTTService( api_key=os.getenv("GLADIA_API_KEY", ""), params=GladiaInputParams( @@ -97,17 +109,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index 7447f8257..c3c46ab15 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.lmnt.tts import LmntTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = LmntTTSService(api_key=os.getenv("LMNT_API_KEY"), voice_id="morgan") @@ -85,18 +97,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py index 869548274..3e73194ab 100644 --- a/examples/foundational/07l-interruptible-groq.py +++ b/examples/foundational/07l-interruptible-groq.py @@ -19,25 +19,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.groq.llm import GroqLLMService from pipecat.services.groq.stt import GroqSTTService from pipecat.services.groq.tts import GroqTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) llm = GroqLLMService( @@ -89,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07m-interruptible-aws.py b/examples/foundational/07m-interruptible-aws.py index bbcfe7313..78ef705a1 100644 --- a/examples/foundational/07m-interruptible-aws.py +++ b/examples/foundational/07m-interruptible-aws.py @@ -17,25 +17,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.aws.llm import AWSBedrockLLMService from pipecat.services.aws.stt import AWSTranscribeSTTService from pipecat.services.aws.tts import AWSPollyTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = AWSTranscribeSTTService() tts = AWSPollyTTSService( @@ -92,18 +104,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index 36cb27193..7e032513c 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -19,25 +19,37 @@ from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.stt import GoogleSTTService from pipecat.services.google.tts import GoogleTTSService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = GoogleSTTService( params=GoogleSTTService.InputParams(languages=Language.EN_US), credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), @@ -93,18 +105,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 2b371be50..72963c0dc 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.assemblyai.stt import AssemblyAISTTService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = AssemblyAISTTService( api_key=os.getenv("ASSEMBLYAI_API_KEY"), @@ -90,18 +103,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index baaef1852..21046a3b9 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -19,26 +19,40 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + audio_in_filter=KrispFilter(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + audio_in_filter=KrispFilter(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + audio_in_filter=KrispFilter(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - audio_in_filter=KrispFilter(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") @@ -87,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py index 40fc6be5f..132b743ab 100644 --- a/examples/foundational/07q-interruptible-rime-http.py +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -19,24 +19,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.rime.tts import RimeHttpTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create an HTTP session async with aiohttp.ClientSession() as session: @@ -93,18 +106,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 27f678930..d859b83aa 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.rime.tts import RimeTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = RimeTTSService( @@ -88,18 +100,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index ddb80181c..1d202bcb5 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -16,31 +16,39 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.nim.llm import NimLLMService -from pipecat.services.riva.stt import ( - ParakeetSTTService, - RivaSegmentedSTTService, - RivaSTTService, -) -from pipecat.services.riva.tts import FastPitchTTSService, RivaTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.services.riva.stt import RivaSTTService +from pipecat.services.riva.tts import RivaTTSService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct") @@ -89,18 +97,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 360a5d350..9f8c9fbbf 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -32,9 +32,9 @@ from pipecat.processors.frame_processor import FrameProcessor from pipecat.services.google.llm import GoogleLLMService from pipecat.services.google.tts import GoogleTTSService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -191,17 +191,30 @@ class TanscriptionContextFixup(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") @@ -261,18 +274,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index 58fb4cd61..50eee52ec 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.fish.tts import FishAudioTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -88,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index b1e2e3756..23bc409f3 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -11,15 +11,14 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.ultravox.stt import UltravoxSTTService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -36,17 +35,30 @@ ultravox_processor = UltravoxSTTService( ) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tts = CartesiaTTSService( api_key=os.environ.get("CARTESIA_API_KEY"), @@ -77,18 +89,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07v-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py index 24eafa2e5..453caa00d 100644 --- a/examples/foundational/07v-interruptible-neuphonic-http.py +++ b/examples/foundational/07v-interruptible-neuphonic-http.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.neuphonic.tts import NeuphonicHttpTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -88,18 +101,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07v-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py index 660925544..934875db4 100644 --- a/examples/foundational/07v-interruptible-neuphonic.py +++ b/examples/foundational/07v-interruptible-neuphonic.py @@ -18,25 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.neuphonic.tts import NeuphonicTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = NeuphonicTTSService( @@ -88,18 +100,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py index 754aac7e9..2b00c4b27 100644 --- a/examples/foundational/07w-interruptible-fal.py +++ b/examples/foundational/07w-interruptible-fal.py @@ -18,24 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.fal.stt import FalSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = FalSTTService( api_key=os.getenv("FAL_KEY"), @@ -90,18 +103,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07y-interruptible-minimax.py b/examples/foundational/07y-interruptible-minimax.py index 5add114bc..b01a51b57 100644 --- a/examples/foundational/07y-interruptible-minimax.py +++ b/examples/foundational/07y-interruptible-minimax.py @@ -20,27 +20,40 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.minimax.tts import MiniMaxHttpTTSService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") # Create an HTTP session async with aiohttp.ClientSession() as session: - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = MiniMaxHttpTTSService( @@ -94,18 +107,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/07z-interruptible-sarvam.py b/examples/foundational/07z-interruptible-sarvam.py index fafee5e93..02225b209 100644 --- a/examples/foundational/07z-interruptible-sarvam.py +++ b/examples/foundational/07z-interruptible-sarvam.py @@ -20,24 +20,38 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.sarvam.tts import SarvamTTSService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) # Create an HTTP session async with aiohttp.ClientSession() as session: stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -92,18 +106,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index 2a62b5a88..67c21945c 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -20,9 +20,8 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -47,21 +46,33 @@ class MirrorProcessor(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - video_out_width=1280, - video_out_height=720, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") pipeline = Pipeline([transport.input(), MirrorProcessor(), transport.output()]) @@ -77,18 +88,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 489f68888..5cbde5700 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -11,6 +11,7 @@ import tkinter as tk from dotenv import load_dotenv from loguru import logger +from pipecat.examples.run import maybe_capture_participant_camera from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -22,10 +23,9 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transports.base_transport import TransportParams +from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -50,21 +50,33 @@ class MirrorProcessor(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), +} - p2p_transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - video_out_width=1280, - video_out_height=720, - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tk_root = tk.Tk() tk_root.title("Local Mirror") @@ -80,11 +92,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ), ) - @p2p_transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - - pipeline = Pipeline([p2p_transport.input(), MirrorProcessor(), tk_transport.output()]) + pipeline = Pipeline([transport.input(), MirrorProcessor(), tk_transport.output()]) task = PipelineTask( pipeline, @@ -97,12 +105,22 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac tk_root.update_idletasks() await asyncio.sleep(0.1) - runner = PipelineRunner(handle_sigint=False) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + await maybe_capture_participant_camera(transport, client, framerate=30) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=handle_sigint) await asyncio.gather(runner.run(task), run_tk()) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 66b4045d6..186ff97aa 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -20,25 +20,37 @@ from pipecat.processors.filters.wake_check_filter import WakeCheckFilter from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -84,18 +96,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index d896ddfb9..3ca4f8855 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -30,9 +30,9 @@ from pipecat.processors.logger import FrameLogger from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -78,17 +78,30 @@ class InboundSoundEffectWrapper(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -141,18 +154,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index 3afd6dd0c..c7edd67c4 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -12,6 +12,7 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -22,9 +23,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.moondream.vision import MoondreamService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -47,21 +47,27 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - # Get WebRTC peer connection ID - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") user_response = UserResponseAggregator() @@ -99,27 +105,26 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - # Welcome message - await tts.say("Hi there! Feel free to ask me what I see.") + await maybe_capture_participant_camera(transport, client) # Set the participant ID in the image requester - image_requester.set_participant_id(webrtc_peer_id) + client_id = get_transport_client_id(transport, client) + image_requester.set_participant_id(client_id) + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index f81655298..fa2d9aec6 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -12,6 +12,7 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -22,9 +23,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -47,21 +47,27 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - # Get WebRTC peer connection ID - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") user_response = UserResponseAggregator() @@ -102,27 +108,26 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - # Welcome message - await tts.say("Hi there! Feel free to ask me what I see.") + await maybe_capture_participant_camera(transport, client) # Set the participant ID in the image requester - image_requester.set_participant_id(webrtc_peer_id) + client_id = get_transport_client_id(transport, client) + image_requester.set_participant_id(client_id) + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index 00d64863c..3c7937f54 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -12,6 +12,7 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -22,9 +23,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -47,21 +47,27 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - # Get WebRTC peer connection ID - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") user_response = UserResponseAggregator() @@ -102,27 +108,26 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - # Welcome message - await tts.say("Hi there! Feel free to ask me what I see.") + await maybe_capture_participant_camera(transport, client) # Set the participant ID in the image requester - image_requester.set_participant_id(webrtc_peer_id) + client_id = get_transport_client_id(transport, client) + image_requester.set_participant_id(client_id) + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index cc454d2e1..0e82f1077 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -12,6 +12,7 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -22,9 +23,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -47,21 +47,27 @@ class UserImageRequester(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - # Get WebRTC peer connection ID - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") user_response = UserResponseAggregator() @@ -102,27 +108,26 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - # Welcome message - await tts.say("Hi there! Feel free to ask me what I see.") + await maybe_capture_participant_camera(transport, client) # Set the participant ID in the image requester - image_requester.set_participant_id(webrtc_peer_id) + client_id = get_transport_client_id(transport, client) + image_requester.set_participant_id(client_id) + + # Welcome message + await tts.say("Hi there! Feel free to ask me what I see.") @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index 1cee7a00b..8a5b951a8 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -16,9 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper.stt import WhisperSTTService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -31,16 +31,27 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = WhisperSTTService() @@ -53,18 +64,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index 4c7e75dc1..ae8953f0a 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -16,9 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.deepgram.stt import DeepgramSTTService, Language, LiveOptions -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -31,13 +31,18 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_in_enabled=True), + "webrtc": lambda: TransportParams(audio_in_enabled=True), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams(audio_in_enabled=True), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService( api_key=os.getenv("DEEPGRAM_API_KEY"), @@ -53,18 +58,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13c-gladia-transcription.py b/examples/foundational/13c-gladia-transcription.py index a0a6264bf..2e4f5b375 100644 --- a/examples/foundational/13c-gladia-transcription.py +++ b/examples/foundational/13c-gladia-transcription.py @@ -16,9 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.gladia import GladiaSTTService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -31,13 +31,18 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_in_enabled=True), + "webrtc": lambda: TransportParams(audio_in_enabled=True), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams(audio_in_enabled=True), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = GladiaSTTService( api_key=os.getenv("GLADIA_API_KEY"), @@ -53,18 +58,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13c-gladia-translation.py b/examples/foundational/13c-gladia-translation.py index 0c821df5d..74e9437c0 100644 --- a/examples/foundational/13c-gladia-translation.py +++ b/examples/foundational/13c-gladia-translation.py @@ -23,9 +23,9 @@ from pipecat.services.gladia.config import ( ) from pipecat.services.gladia.stt import GladiaSTTService from pipecat.transcriptions.language import Language -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -40,13 +40,18 @@ class TranscriptionLogger(FrameProcessor): print(f"Translation ({frame.language}): {frame.text}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_in_enabled=True), + "webrtc": lambda: TransportParams(audio_in_enabled=True), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams(audio_in_enabled=True), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = GladiaSTTService( api_key=os.getenv("GLADIA_API_KEY"), @@ -74,18 +79,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index 8fa99d8de..de22b9bb6 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -16,9 +16,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.assemblyai.stt import AssemblyAISTTService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -31,13 +31,18 @@ class TranscriptionLogger(FrameProcessor): print(f"Transcription: {frame.text}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_in_enabled=True), + "webrtc": lambda: TransportParams(audio_in_enabled=True), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams(audio_in_enabled=True), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = AssemblyAISTTService( api_key=os.getenv("ASSEMBLYAI_API_KEY"), @@ -52,18 +57,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13e-whisper-mlx.py b/examples/foundational/13e-whisper-mlx.py index 9ab7bc82b..5d7dc53e3 100644 --- a/examples/foundational/13e-whisper-mlx.py +++ b/examples/foundational/13e-whisper-mlx.py @@ -18,9 +18,9 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper.stt import MLXModel, WhisperSTTServiceMLX -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -52,16 +52,27 @@ class TranscriptionLogger(FrameProcessor): self._last_transcription_time = time.time() -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=STOP_SECS)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = WhisperSTTServiceMLX(model=MLXModel.LARGE_V3_TURBO) @@ -80,18 +91,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/13f-cartesia-transcription.py b/examples/foundational/13f-cartesia-transcription.py new file mode 100644 index 000000000..147d5fb3e --- /dev/null +++ b/examples/foundational/13f-cartesia-transcription.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.frames.frames import Frame, TranscriptionFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.cartesia.stt import CartesiaSTTService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + + +class TranscriptionLogger(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + print(f"Transcription: {frame.text}") + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams(audio_in_enabled=True), + "twilio": lambda: FastAPIWebsocketParams(audio_in_enabled=True), + "webrtc": lambda: TransportParams(audio_in_enabled=True), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") + + stt = CartesiaSTTService( + api_key=os.getenv("CARTESIA_API_KEY"), + base_url=os.getenv("CARTESIA_BASE_URL"), + ) + + tl = TranscriptionLogger() + + pipeline = Pipeline([transport.input(), stt, tl]) + + task = PipelineTask(pipeline) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=handle_sigint) + + await runner.run(task) + + +if __name__ == "__main__": + from pipecat.examples.run import main + + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index ab51a6050..b4e09c99a 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -22,29 +22,45 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -58,6 +74,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) weather_function = FunctionSchema( name="get_current_weather", @@ -75,7 +96,18 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac }, required=["location", "format"], ) - tools = ToolsSchema(standard_tools=[weather_function]) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) messages = [ { @@ -118,18 +150,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 38d4aed2f..bf34e211c 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -21,9 +21,9 @@ from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -33,17 +33,34 @@ async def get_weather(params: FunctionCallParams): await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -53,9 +70,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" + api_key=os.getenv("ANTHROPIC_API_KEY"), + model="claude-3-7-sonnet-latest", ) llm.register_function("get_weather", get_weather) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) weather_function = FunctionSchema( name="get_weather", @@ -68,7 +87,18 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac }, required=["location"], ) - tools = ToolsSchema(standard_tools=[weather_function]) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) # todo: test with very short initial user message @@ -111,18 +141,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index c18f7bf48..04049984f 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -14,6 +14,7 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -22,15 +23,14 @@ from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -# Global variable to store the peer connection ID -webrtc_peer_id = "" +# Global variable to store the client ID +client_id = "" async def get_weather(params: FunctionCallParams): @@ -40,11 +40,11 @@ async def get_weather(params: FunctionCallParams): async def get_image(params: FunctionCallParams): question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + logger.debug(f"Requesting image with user_id={client_id}, question={question}") # Request the image frame await params.llm.request_image_frame( - user_id=webrtc_peer_id, + user_id=client_id, function_name=params.function_name, tool_call_id=params.tool_call_id, text_content=question, @@ -59,21 +59,27 @@ async def get_image(params: FunctionCallParams): ) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - global webrtc_peer_id - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -174,24 +180,26 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") + + await maybe_capture_participant_camera(transport, client) + + global client_id + client_id = get_transport_client_id(transport, client) + # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index da79b68b0..1188f32e2 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -22,29 +22,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.together.llm import TogetherLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -61,6 +73,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -111,18 +127,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 18772b609..7761e1be2 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -14,6 +14,7 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask @@ -22,15 +23,14 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -# Global variable to store the peer connection ID -webrtc_peer_id = "" +# Global variable to store the client ID +client_id = "" async def get_weather(params: FunctionCallParams): @@ -40,11 +40,11 @@ async def get_weather(params: FunctionCallParams): async def get_image(params: FunctionCallParams): question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + logger.debug(f"Requesting image with user_id={client_id}, question={question}") # Request the image frame await params.llm.request_image_frame( - user_id=webrtc_peer_id, + user_id=client_id, function_name=params.function_name, tool_call_id=params.tool_call_id, text_content=question, @@ -59,21 +59,27 @@ async def get_image(params: FunctionCallParams): ) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - global webrtc_peer_id - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -157,24 +163,26 @@ indicate you should use the get_image tool are: @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + + await maybe_capture_participant_camera(transport, client) + + global client_id + client_id = get_transport_client_id(transport, client) + # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 9725e7968..ccbe952fb 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -14,6 +14,7 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -23,30 +24,32 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -# Global variable to store the peer connection ID -webrtc_peer_id = "" +# Global variable to store the client ID +client_id = "" async def get_weather(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) location = params.arguments["location"] await params.result_callback(f"The weather in {location} is currently 72 degrees and sunny.") +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + async def get_image(params: FunctionCallParams): question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + logger.debug(f"Requesting image with user_id={client_id}, question={question}") # Request the image frame await params.llm.request_image_frame( - user_id=webrtc_peer_id, + user_id=client_id, function_name=params.function_name, tool_call_id=params.tool_call_id, text_content=question, @@ -61,21 +64,27 @@ async def get_image(params: FunctionCallParams): ) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - global webrtc_peer_id - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -87,6 +96,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") llm.register_function("get_weather", get_weather) llm.register_function("get_image", get_image) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) weather_function = FunctionSchema( name="get_weather", @@ -104,6 +118,17 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac }, required=["location", "format"], ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) get_image_function = FunctionSchema( name="get_image", description="Get an image from the video stream.", @@ -115,14 +140,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac }, required=["question"], ) - tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function, restaurant_function]) system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. Your response will be turned into speech so use only simple words and punctuation. -You have access to two tools: get_weather and get_image. +You have access to three tools: get_weather, get_restaurant_recommendation, and get_image. You can respond to questions about the weather using the get_weather tool. @@ -167,24 +192,26 @@ indicate you should use the get_image tool are: @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") + + await maybe_capture_participant_camera(transport, client) + + global client_id + client_id = get_transport_client_id(transport, client) + # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index ee6a9a855..3ff56a915 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -23,29 +23,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.groq.llm import GroqLLMService from pipecat.services.groq.stt import GroqSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"), model="distil-whisper-large-v3-en") @@ -61,6 +73,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -120,18 +136,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 63d625bec..ce3349ecb 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -21,9 +21,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.grok.llm import GrokLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -32,17 +32,30 @@ async def fetch_weather_from_api(params: FunctionCallParams): await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -113,18 +126,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 6c7bda50f..25cf7defa 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -22,29 +22,41 @@ from pipecat.services.azure.llm import AzureLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -62,6 +74,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -119,18 +135,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index e970e8213..028d9fa64 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -22,29 +22,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.fireworks.llm import FireworksLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -61,6 +73,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -118,18 +134,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index a07109753..d254e0d4f 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -22,29 +22,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.nim.llm import NimLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -59,6 +71,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -116,18 +132,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 1016b8146..70ffceffd 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -22,29 +22,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.cerebras.llm import CerebrasLLMService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -58,6 +70,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -126,18 +142,14 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index a5e3f814c..7e992942d 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -22,29 +22,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepseek.llm import DeepSeekLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -58,6 +70,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -126,18 +142,14 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 68b3973c5..023f725f6 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -22,29 +22,41 @@ from pipecat.services.azure.tts import AzureTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openrouter.llm import OpenRouterLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -62,6 +74,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -120,18 +136,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 168441611..c4a70d627 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -25,25 +25,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.perplexity.llm import PerplexityLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -94,18 +106,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py index 15077ba0f..8f0036e07 100644 --- a/examples/foundational/14o-function-calling-gemini-openai-format.py +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -22,29 +22,41 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.google.llm_openai import GoogleLLMOpenAIBetaService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -58,6 +70,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -115,18 +131,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index cc152f837..4348b663a 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -22,29 +22,41 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.google.llm_vertex import GoogleVertexLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -54,15 +66,19 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ) llm = GoogleVertexLLMService( - # credentials="", + credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), params=GoogleVertexLLMService.InputParams( project_id="", - ) + ), ) # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -121,18 +137,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14q-function-calling-qwen.py b/examples/foundational/14q-function-calling-qwen.py index 45e0b6463..6e07a97e1 100644 --- a/examples/foundational/14q-function-calling-qwen.py +++ b/examples/foundational/14q-function-calling-qwen.py @@ -22,29 +22,41 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.qwen.llm import QwenLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -59,6 +71,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -118,18 +134,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/14r-function-calling-aws.py b/examples/foundational/14r-function-calling-aws.py index cf4859576..4443bec27 100644 --- a/examples/foundational/14r-function-calling-aws.py +++ b/examples/foundational/14r-function-calling-aws.py @@ -21,9 +21,9 @@ from pipecat.services.aws.llm import AWSBedrockLLMService from pipecat.services.aws.stt import AWSTranscribeSTTService from pipecat.services.aws.tts import AWSPollyTTSService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -32,17 +32,34 @@ async def fetch_weather_from_api(params: FunctionCallParams): await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = AWSTranscribeSTTService() @@ -61,6 +78,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) weather_function = FunctionSchema( name="get_current_weather", @@ -78,7 +96,18 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac }, required=["location", "format"], ) - tools = ToolsSchema(standard_tools=[weather_function]) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) messages = [ { @@ -122,18 +151,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 6b195722d..0735b62c0 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -22,9 +22,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -54,17 +54,30 @@ async def barbershop_man_filter(frame) -> bool: return current_voice == "Barbershop Man" -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -151,18 +164,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 93610841d..48bc7e63b 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -23,9 +23,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -49,17 +49,30 @@ async def spanish_filter(frame) -> bool: return current_language == "Spanish" -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService( api_key=os.getenv("DEEPGRAM_API_KEY"), live_options=LiveOptions(language="multi") @@ -139,18 +152,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index d5e560010..c13c2bbff 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -18,26 +18,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection -from pipecat.transports.services.daily import DailyTransportMessageFrame +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams, DailyTransportMessageFrame load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = DeepgramTTSService( @@ -124,18 +135,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 96189805e..8f5e3ba1d 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -20,25 +20,37 @@ from pipecat.processors.user_idle_processor import UserIdleProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -121,18 +133,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/18-gstreamer-filesrc.py b/examples/foundational/18-gstreamer-filesrc.py index 92dcf973e..94cbaed84 100644 --- a/examples/foundational/18-gstreamer-filesrc.py +++ b/examples/foundational/18-gstreamer-filesrc.py @@ -13,26 +13,35 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, args: argparse.Namespace): - logger.info(f"Starting bot with video input: {args.input}") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_out_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), + "webrtc": lambda: TransportParams( + audio_out_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_out_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - video_out_width=1280, - video_out_height=720, - ), - ) + +async def run_example(transport: BaseTransport, args: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot with video input: {args.input}") gst = GStreamerPipelineSource( pipeline=f"filesrc location={args.input}", @@ -51,15 +60,15 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, args: argparse.Names task = PipelineTask(pipeline) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main parser = argparse.ArgumentParser(description="Pipecat Bot Runner") parser.add_argument("-i", "--input", type=str, required=True, help="Input video file") - main(parser) + main(run_example, parser=parser, transport_params=transport_params) diff --git a/examples/foundational/18a-gstreamer-videotestsrc.py b/examples/foundational/18a-gstreamer-videotestsrc.py index ece124667..91b7535bd 100644 --- a/examples/foundational/18a-gstreamer-videotestsrc.py +++ b/examples/foundational/18a-gstreamer-videotestsrc.py @@ -13,27 +13,33 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), + "webrtc": lambda: TransportParams( + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot with video test source") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - video_out_width=1280, - video_out_height=720, - ), - ) - gst = GStreamerPipelineSource( pipeline='videotestsrc ! capsfilter caps="video/x-raw,width=1280,height=720,framerate=30/1"', out_params=GStreamerPipelineSource.OutputParams( @@ -50,12 +56,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac task = PipelineTask(pipeline) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 17b5462c0..9eb816432 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -14,11 +14,12 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.frames.frames import TranscriptionMessage from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai_realtime_beta import ( InputAudioNoiseReduction, @@ -27,9 +28,9 @@ from pipecat.services.openai_realtime_beta import ( SemanticTurnDetection, SessionProperties, ) -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -46,6 +47,10 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -63,22 +68,47 @@ weather_function = FunctionSchema( required=["location", "format"], ) +restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], +) + # Create tools schema -tools = ToolsSchema(standard_tools=[weather_function]) +tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) - session_properties = SessionProperties( input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it @@ -88,7 +118,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # turn_detection=False, input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + instructions="""You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -97,10 +127,14 @@ playful tone. If interacting in a non-English language, start by using the standard accent or dialect familiar to the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, even if you're asked about them. -- + You are participating in a voice conversation. Keep your responses concise, short, and to the point unless specifically asked to elaborate on a topic. +You have access to the following tools: +- get_current_weather: Get the current weather for a given location. +- get_restaurant_recommendation: Get a restaurant recommendation for a given location. + Remember, your responses should be short. Just one or two sentences, usually.""", ) @@ -113,6 +147,9 @@ Remember, your responses should be short. Just one or two sentences, usually.""" # you can either register a single function for all function calls, or specific functions # llm.register_function(None, fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) + + transcript = TranscriptProcessor() # Create a standard OpenAI LLM context object using the normal messages format. The # OpenAIRealtimeBetaLLMService will convert this internally to messages that the @@ -139,7 +176,9 @@ Remember, your responses should be short. Just one or two sentences, usually.""" transport.input(), # Transport user input context_aggregator.user(), llm, # LLM + transcript.user(), # Placed after the LLM, as LLM pushes TranscriptionFrames downstream transport.output(), # Transport bot output + transcript.assistant(), # After the transcript output, to time with the audio output context_aggregator.assistant(), ] ) @@ -163,18 +202,23 @@ Remember, your responses should be short. Just one or two sentences, usually.""" @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for msg in frame.messages: + if isinstance(msg, TranscriptionMessage): + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + line = f"{timestamp}{msg.role}: {msg.content}" + logger.info(f"Transcript: {line}") + + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index c7778d6b8..54f0302e5 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -14,7 +14,6 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -25,9 +24,9 @@ from pipecat.services.openai_realtime_beta import ( InputAudioTranscription, SessionProperties, ) -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -44,6 +43,10 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + # Define weather function using standardized schema weather_function = FunctionSchema( name="get_current_weather", @@ -62,22 +65,47 @@ weather_function = FunctionSchema( required=["location", "format"], ) +restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], +) + # Create tools schema -tools = ToolsSchema(standard_tools=[weather_function]) +tools = ToolsSchema(standard_tools=[weather_function, restaurant_function]) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) - session_properties = SessionProperties( input_audio_transcription=InputAudioTranscription(model="whisper-1"), # Set openai TurnDetection parameters. Not setting this at all will turn it @@ -86,7 +114,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, # tools=tools, - instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + instructions="""You are a helpful and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and @@ -99,6 +127,10 @@ even if you're asked about them. You are participating in a voice conversation. Keep your responses concise, short, and to the point unless specifically asked to elaborate on a topic. +You have access to the following tools: +- get_current_weather: Get the current weather for a given location. +- get_restaurant_recommendation: Get a restaurant recommendation for a given location. + Remember, your responses should be short. Just one or two sentences, usually.""", ) @@ -112,6 +144,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" # you can either register a single function for all function calls, or specific functions # llm.register_function(None, fetch_weather_from_api) llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) # Create a standard OpenAI LLM context object using the normal messages format. The # OpenAIRealtimeBetaLLMService will convert this internally to messages that the @@ -162,18 +195,14 @@ Remember, your responses should be short. Just one or two sentences, usually.""" @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 32d50326b..360d5f659 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -25,9 +25,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -164,22 +164,33 @@ tools = [ ] -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") global tts - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady @@ -228,18 +239,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 12d82ff37..a1ea1f4c5 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -15,7 +15,6 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -30,9 +29,9 @@ from pipecat.services.openai_realtime_beta import ( SessionProperties, TurnDetection, ) -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -153,17 +152,30 @@ tools = [ ] -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -237,18 +249,14 @@ Remember, your responses should be short. Just one or two sentences, usually.""" @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 1d304ed6e..686bed5a7 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -25,9 +25,9 @@ from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -159,20 +159,33 @@ tools = [ ] -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") global tts - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -225,18 +238,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index a0c9bff7a..5219e3a2e 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -15,6 +15,7 @@ from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -25,19 +26,16 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -video_participant_id = None - - BASE_FILENAME = "/tmp/pipecat_conversation_" -tts = None -webrtc_peer_id = "" + +# Global variable to store the client ID +client_id = "" async def fetch_weather_from_api(params: FunctionCallParams): @@ -54,11 +52,11 @@ async def fetch_weather_from_api(params: FunctionCallParams): async def get_image(params: FunctionCallParams): question = params.arguments["question"] - logger.debug(f"Requesting image with user_id={webrtc_peer_id}, question={question}") + logger.debug(f"Requesting image with user_id={client_id}, question={question}") # Request the image frame await params.llm.request_image_frame( - user_id=webrtc_peer_id, + user_id=client_id, function_name=params.function_name, tool_call_id=params.tool_call_id, text_content=question, @@ -96,7 +94,6 @@ async def save_conversation(params: FunctionCallParams): async def load_conversation(params: FunctionCallParams): - global tts filename = params.arguments["filename"] logger.debug(f"loading conversation from {filename}") try: @@ -221,21 +218,27 @@ tools = [ ] -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - global tts, webrtc_peer_id - webrtc_peer_id = webrtc_connection.pc_id +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + ), +} - logger.info(f"Starting bot with peer_id: {webrtc_peer_id}") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -282,24 +285,26 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected") + + await maybe_capture_participant_camera(transport, client) + + global client_id + client_id = get_transport_client_id(transport, client) + # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/20e-persistent-context-aws-nova-sonic.py b/examples/foundational/20e-persistent-context-aws-nova-sonic.py index 1519f1c53..86630ce6d 100644 --- a/examples/foundational/20e-persistent-context-aws-nova-sonic.py +++ b/examples/foundational/20e-persistent-context-aws-nova-sonic.py @@ -17,16 +17,15 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.aws_nova_sonic.aws import AWSNovaSonicLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -170,17 +169,30 @@ tools = ToolsSchema( ) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Specify initial system instruction. # HACK: note that, for now, we need to inject a special bit of text into this instruction to @@ -250,18 +262,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/21-tavus-layer-tavus-transport.py b/examples/foundational/21-tavus-transport.py similarity index 98% rename from examples/foundational/21-tavus-layer-tavus-transport.py rename to examples/foundational/21-tavus-transport.py index c9bcd2501..68614a854 100644 --- a/examples/foundational/21-tavus-layer-tavus-transport.py +++ b/examples/foundational/21-tavus-transport.py @@ -103,7 +103,7 @@ async def main(): logger.info(f"Client disconnected") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner() await runner.run(task) diff --git a/examples/foundational/21a-tavus-layer-small-webrtc.py b/examples/foundational/21a-tavus-video-service.py similarity index 75% rename from examples/foundational/21a-tavus-layer-small-webrtc.py rename to examples/foundational/21a-tavus-video-service.py index 2f557b5cd..db79d343d 100644 --- a/examples/foundational/21a-tavus-layer-small-webrtc.py +++ b/examples/foundational/21a-tavus-video-service.py @@ -20,29 +20,39 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService from pipecat.services.tavus.video import TavusVideoService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=1280, + video_out_height=720, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") async with aiohttp.ClientSession() as session: - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - vad_analyzer=SileroVADAnalyzer(), - video_out_width=1280, - video_out_height=720, - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -108,18 +118,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/21b-tavus-layer-daily-transport.py b/examples/foundational/21b-tavus-layer-daily-transport.py deleted file mode 100644 index 564828136..000000000 --- a/examples/foundational/21b-tavus-layer-daily-transport.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys - -import aiohttp -from daily_runner import configure -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.google.llm import GoogleLLMService -from pipecat.services.tavus.video import TavusVideoService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Pipecat bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - vad_analyzer=SileroVADAnalyzer(), - video_out_width=1280, - video_out_height=720, - ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="a167e0f3-df7e-4d52-a9c3-f949145efdab", - ) - - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) - - tavus = TavusVideoService( - api_key=os.getenv("TAVUS_API_KEY"), - replica_id=os.getenv("TAVUS_REPLICA_ID"), - session=session, - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - tavus, # Tavus output layer - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - audio_in_sample_rate=16000, - audio_out_sample_rate=24000, - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Kick off the conversation. - messages.append( - { - "role": "system", - "content": "Start by greeting the user and ask how you can help.", - } - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() - - runner = PipelineRunner(handle_sigint=False) - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 75683e41c..0a49f141c 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -25,24 +25,37 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -151,18 +164,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 45c25776e..b76bfd274 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -48,9 +48,9 @@ from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -198,21 +198,33 @@ class OutputGate(FrameProcessor): async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -232,6 +244,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + tools = [ ChatCompletionToolParam( type="function", @@ -296,9 +312,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # to start the conversation. bot_output_gate = OutputGate(notifier=notifier, start_open=True) - async def block_user_stopped_speaking(frame): - return not isinstance(frame, UserStoppedSpeakingFrame) - async def pass_only_llm_trigger_frames(frame): return ( isinstance(frame, OpenAILLMContextFrame) @@ -315,11 +328,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac stt, context_aggregator.user(), ParallelPipeline( - [ - # Pass everything except UserStoppedSpeaking to the elements after - # this ParallelPipeline - FunctionFilter(filter=block_user_stopped_speaking), - ], [ # Ignore everything except an OpenAILLMContextFrame. Pass a specially constructed # LLMMessagesFrame to the statement classifier LLM. The only frame this @@ -376,18 +384,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 2cbda6b96..0bcbf75bd 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -49,9 +49,9 @@ from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -402,21 +402,33 @@ class OutputGate(FrameProcessor): async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -436,6 +448,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + tools = [ ChatCompletionToolParam( type="function", @@ -583,18 +599,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 3460e079c..b6b66db0c 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -48,9 +48,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService from pipecat.sync.base_notifier import BaseNotifier from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -627,17 +627,30 @@ class OutputGate(FrameProcessor): break -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), @@ -762,18 +775,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/23-bot-background-sound-daily.py b/examples/foundational/23-bot-background-sound-daily.py deleted file mode 100644 index 67c03386f..000000000 --- a/examples/foundational/23-bot-background-sound-daily.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import argparse -import asyncio -import os -import sys - -import aiohttp -from daily_runner import configure_with_args -from dotenv import load_dotenv -from loguru import logger - -from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import MixerEnableFrame, MixerUpdateSettingsFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -async def main(): - async with aiohttp.ClientSession() as session: - parser = argparse.ArgumentParser(description="Bot Background Sound") - parser.add_argument("-i", "--input", type=str, required=True, help="Input audio file") - - (room_url, token, args) = await configure_with_args(session, parser) - - soundfile_mixer = SoundfileMixer( - sound_files={"office": args.input}, - default_sound="office", - volume=2.0, - ) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - audio_out_mixer=soundfile_mixer, - transcription_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Show how to use mixer control frames. - await asyncio.sleep(10.0) - await task.queue_frame(MixerUpdateSettingsFrame({"volume": 0.5})) - await asyncio.sleep(5.0) - await task.queue_frame(MixerEnableFrame(False)) - await asyncio.sleep(5.0) - await task.queue_frame(MixerEnableFrame(True)) - await asyncio.sleep(5.0) - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/23-bot-background-sound-p2p.py b/examples/foundational/23-bot-background-sound.py similarity index 60% rename from examples/foundational/23-bot-background-sound-p2p.py rename to examples/foundational/23-bot-background-sound.py index 1d75c305e..098a62f86 100644 --- a/examples/foundational/23-bot-background-sound-p2p.py +++ b/examples/foundational/23-bot-background-sound.py @@ -4,16 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""Usage ------ -Set the path to your background audio file using the `INPUT_AUDIO_PATH` environment variable, then run the bot using: - - INPUT_AUDIO_PATH=path/to/your_audio.mp3 python 23-bot-background-sound.py - -Example: - INPUT_AUDIO_PATH=my_audio.mp3 python 23-bot-background-sound.py -""" - import argparse import asyncio import os @@ -31,36 +21,54 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -audio_path = os.getenv("INPUT_AUDIO_PATH") -if not audio_path: - raise ValueError("No INPUT_AUDIO_PATH specified in environment variables") +OFFICE_SOUND_FILE = os.path.join( + os.path.dirname(__file__), "assets", "office-ambience-24000-mono.mp3" +) - -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") - - soundfile_mixer = SoundfileMixer( - sound_files={"office": audio_path}, - default_sound="office", - volume=2.0, - ) - - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - audio_out_mixer=soundfile_mixer, - vad_analyzer=SileroVADAnalyzer(), +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_mixer=SoundfileMixer( + sound_files={"office": OFFICE_SOUND_FILE}, + default_sound="office", + volume=2.0, ), - ) + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_mixer=SoundfileMixer( + sound_files={"office": OFFICE_SOUND_FILE}, + default_sound="office", + volume=2.0, + ), + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_out_mixer=SoundfileMixer( + sound_files={"office": OFFICE_SOUND_FILE}, + default_sound="office", + volume=2.0, + ), + vad_analyzer=SileroVADAnalyzer(), + ), +} + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -83,7 +91,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac pipeline = Pipeline( [ transport.input(), # Transport user input - stt, # STT service + stt, # STT context_aggregator.user(), # User responses llm, # LLM tts, # TTS @@ -103,16 +111,18 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ) @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") + async def on_client_connected(transport, participant): # Show how to use mixer control frames. - await asyncio.sleep(10.0) + logger.info(f"Listening for background sound for a bit...") + await asyncio.sleep(5.0) + logger.info(f"Reducing volume...") await task.queue_frame(MixerUpdateSettingsFrame({"volume": 0.5})) await asyncio.sleep(5.0) + logger.info(f"Disabling background sound for a bit...") await task.queue_frame(MixerEnableFrame(False)) await asyncio.sleep(5.0) + logger.info(f"Re-enabling background sound and starting bot...") await task.queue_frame(MixerEnableFrame(True)) - await asyncio.sleep(5.0) # Kick off the conversation. messages.append({"role": "system", "content": "Please introduce yourself to the user."}) await task.queue_frames([context_aggregator.user().get_context_frame()]) @@ -120,18 +130,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 83f7b85a9..c1c4580fd 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -23,9 +23,9 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.deepgram.tts import DeepgramTTSService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -38,17 +38,30 @@ async def fetch_weather_from_api(params: FunctionCallParams): await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -125,18 +138,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index eea8c5a2a..83005a3ed 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -34,9 +34,9 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.google.llm import GoogleLLMContext, GoogleLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -269,17 +269,30 @@ class TranscriptionContextFixup(FrameProcessor): await self.push_frame(frame, direction) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), @@ -359,18 +372,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 3e0bccb74..03c063982 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -17,29 +17,44 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams # Load environment variables load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} - # Initialize the SmallWebRTCTransport with the connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_in_enabled=False, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create the Gemini Multimodal Live LLM service system_instruction = f""" @@ -96,18 +111,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() # Run the pipeline - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py index 94fdfac2a..9da336459 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py @@ -19,29 +19,49 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} - # Initialize the SmallWebRTCTransport with the connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") llm = GeminiMultimodalLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), @@ -102,10 +122,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() # Register event handler for transcript updates @@ -117,12 +133,12 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac line = f"{timestamp}{msg.role}: {msg.content}" logger.info(f"Transcript: {line}") - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 29017ba00..15db2faa0 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -21,9 +21,9 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -40,30 +40,56 @@ async def fetch_weather_from_api(params: FunctionCallParams): ) +async def fetch_restaurant_recommendation(params: FunctionCallParams): + await params.result_callback({"name": "The Golden Dragon"}) + + system_instruction = """ You are a helpful assistant who can answer questions and use tools. -You have a tool called "get_current_weather" that can be used to get the current weather. If the user asks -for the weather, call this function. +You have three tools available to you: +1. get_current_weather: Use this tool to get the current weather in a specific location. +2. get_restaurant_recommendation: Use this tool to get a restaurant recommendation in a specific location. +3. google_search: Use this tool to search the web for information. """ -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} - # Initialize the SmallWebRTCTransport with the connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") weather_function = FunctionSchema( name="get_current_weather", @@ -81,9 +107,21 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac }, required=["location", "format"], ) + restaurant_function = FunctionSchema( + name="get_restaurant_recommendation", + description="Get a restaurant recommendation", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + }, + required=["location"], + ) search_tool = {"google_search": {}} tools = ToolsSchema( - standard_tools=[weather_function], custom_tools={AdapterType.GEMINI: [search_tool]} + standard_tools=[weather_function, restaurant_function], + custom_tools={AdapterType.GEMINI: [search_tool]}, ) llm = GeminiMultimodalLiveLLMService( @@ -93,6 +131,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ) llm.register_function("get_current_weather", fetch_weather_from_api) + llm.register_function("get_restaurant_recommendation", fetch_restaurant_recommendation) context = OpenAILLMContext( [{"role": "user", "content": "Say hello."}], @@ -127,18 +166,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py index cabbd8353..c39aeb63b 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-multimodal-live-video.py @@ -4,105 +4,114 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import argparse import asyncio import os -import sys -import aiohttp -from daily_runner import configure from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.examples.run import maybe_capture_participant_camera, maybe_capture_participant_screen from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede + # system_instruction="Talk like a pirate." + # inference_on_context_initialization=False, + ) - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + context = OpenAILLMContext( + [ + { + "role": "user", + "content": "Say hello.", + }, + ], + ) + context_aggregator = llm.create_context_aggregator(context) - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - voice_id="Aoede", # Puck, Charon, Kore, Fenrir, Aoede - # system_instruction="Talk like a pirate." - # inference_on_context_initialization=False, - ) + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) - context = OpenAILLMContext( - [ - { - "role": "user", - "content": "Say hello.", - }, - ], - ) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - transport.output(), - context_aggregator.assistant(), - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) + await maybe_capture_participant_camera(transport, client, framerate=1) + await maybe_capture_participant_screen(transport, client, framerate=1) - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - # Enable both camera and screenshare. From the client side - # send just one. - await transport.capture_participant_video( - participant["id"], framerate=1, video_source="camera" - ) - await transport.capture_participant_video( - participant["id"], framerate=1, video_source="screenVideo" - ) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - await asyncio.sleep(3) - logger.debug("Unpausing audio and video") - llm.set_audio_input_paused(False) - llm.set_video_input_paused(False) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + await asyncio.sleep(3) + logger.debug("Unpausing audio and video") + llm.set_audio_input_paused(False) + llm.set_video_input_paused(False) - runner = PipelineRunner() + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() - await runner.run(task) + runner = PipelineRunner(handle_sigint=handle_sigint) + + await runner.run(task) if __name__ == "__main__": - asyncio.run(main()) + from pipecat.examples.run import main + + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index c42acd34b..036e383b9 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -22,9 +22,9 @@ from pipecat.services.gemini_multimodal_live.gemini import ( GeminiMultimodalModalities, InputParams, ) -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -40,22 +40,42 @@ Respond to what the user said in a creative and helpful way. Keep your responses """ -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} - # Initialize the SmallWebRTCTransport with the connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - # set stop_secs to something roughly similar to the internal setting - # of the Multimodal Live api, just to align events. This doesn't really - # matter because we can only use the Multimodal Live API's phrase - # endpointing, for now. - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") llm = GeminiMultimodalLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), @@ -114,18 +134,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-multimodal-google-search.py index 97483028c..8638dfc11 100644 --- a/examples/foundational/26e-gemini-multimodal-google-search.py +++ b/examples/foundational/26e-gemini-multimodal-google-search.py @@ -11,14 +11,15 @@ from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -41,18 +42,42 @@ Start each interaction by asking the user about which place they would like to k """ -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "twilio": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + # set stop_secs to something roughly similar to the internal setting + # of the Multimodal Live api, just to align events. This doesn't really + # matter because we can only use the Multimodal Live API's phrase + # endpointing, for now. + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), + ), +} - # Initialize the SmallWebRTCTransport with the connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Initialize the Gemini Multimodal Live model llm = GeminiMultimodalLiveLLMService( @@ -93,18 +118,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 38721e50e..9f7f6aa38 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -20,29 +20,39 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.simli.video import SimliVideoService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=512, + video_out_height=512, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_is_live=True, + video_out_width=512, + video_out_height=512, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_out_enabled=True, - video_out_is_live=True, - video_out_width=512, - video_out_height=512, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -96,17 +106,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/28-transcription-processor.py b/examples/foundational/28-transcription-processor.py index e538c20e9..d8211071c 100644 --- a/examples/foundational/28-transcription-processor.py +++ b/examples/foundational/28-transcription-processor.py @@ -21,9 +21,9 @@ from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -89,17 +89,30 @@ class TranscriptHandler: await self.save_message(msg) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -155,17 +168,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/29-turn-tracking-observer.py b/examples/foundational/29-turn-tracking-observer.py index 08c39fe55..987ef372f 100644 --- a/examples/foundational/29-turn-tracking-observer.py +++ b/examples/foundational/29-turn-tracking-observer.py @@ -19,25 +19,37 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -104,18 +116,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index c9cd08aee..c8172d2fa 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -34,9 +34,9 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -74,17 +74,30 @@ class CustomObserver(BaseObserver): logger.info(f"🤖 BOT STOP SPEAKING: {src} {arrow} {dst} at {time_sec:.2f}s") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -148,18 +161,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 8c53f7367..47b94b979 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -22,9 +22,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService, LLMSearchResponseFrame from pipecat.services.llm_service import LLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams sys.path.append(str(Path(__file__).parent.parent)) @@ -67,17 +67,30 @@ class LLMSearchLoggerObserver(BaseObserver): logger.debug(f"🧠 {arrow} {dst} LLM SEARCH RESPONSE FRAME: {frame} at {time_sec:.2f}s") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -130,17 +143,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/33-gemini-rag.py b/examples/foundational/33-gemini-rag.py index 43da2ccf1..b1c88f289 100644 --- a/examples/foundational/33-gemini-rag.py +++ b/examples/foundational/33-gemini-rag.py @@ -65,9 +65,9 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -154,17 +154,30 @@ async def query_knowledge_base(params: FunctionCallParams): await params.result_callback(response.text) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -241,17 +254,13 @@ Your response will be turned into speech so use only simple words and punctuatio @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index a7d07e7b2..5c6b60d72 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -65,9 +65,9 @@ from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -86,17 +86,30 @@ async def save_audio_file(audio: bytes, filename: str, sample_rate: int, num_cha logger.info(f"Audio saved to {filename}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) @@ -146,10 +159,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() # Handler for merged audio @@ -174,11 +183,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac bot_filename = f"recordings/bot_{timestamp}.wav" await save_audio_file(bot_audio, bot_filename, sample_rate, 1) - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 7b1218015..2663380be 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -59,9 +59,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator load_dotenv(override=True) @@ -74,19 +74,31 @@ VOICE_IDS = { "male": "7cf0e2b1-8daf-4fe4-89ad-f6039398f359", # Male character voice } +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - # Create pattern pair aggregator for voice switching pattern_aggregator = PatternPairAggregator() @@ -215,17 +227,13 @@ Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue."" @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py index a917a626a..47996a492 100644 --- a/examples/foundational/36-user-email-gathering.py +++ b/examples/foundational/36-user-email-gathering.py @@ -21,9 +21,9 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService from pipecat.services.rime.tts import RimeHttpTTSService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -32,17 +32,30 @@ async def store_user_emails(params: FunctionCallParams): print(f"User emails: {params.arguments}") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -132,17 +145,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/37-mem0.py b/examples/foundational/37-mem0.py index dfb3e7db4..654a53a80 100644 --- a/examples/foundational/37-mem0.py +++ b/examples/foundational/37-mem0.py @@ -58,9 +58,9 @@ from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.elevenlabs.tts import ElevenLabsTTSService from pipecat.services.mem0.memory import Mem0MemoryService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -123,7 +123,29 @@ async def get_initial_greeting( return "Hello! How can I help you today?" -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): """Main bot execution function. Sets up and runs the bot pipeline including: @@ -138,15 +160,6 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) # Initialize text-to-speech service @@ -272,17 +285,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/38-smart-turn-fal.py b/examples/foundational/38-smart-turn-fal.py index b4a0708cb..1fa8a2891 100644 --- a/examples/foundational/38-smart-turn-fal.py +++ b/examples/foundational/38-smart-turn-fal.py @@ -21,92 +21,109 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +aiohttp_session = aiohttp.ClientSession() -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=FalSmartTurnAnalyzer( + api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp_session + ), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=FalSmartTurnAnalyzer( + api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp_session + ), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=FalSmartTurnAnalyzer( + api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=aiohttp_session + ), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - async with aiohttp.ClientSession() as session: - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=FalSmartTurnAnalyzer( - api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=session - ), - ), - ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses ] + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] - ) + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected") - # Kick off the conversation. - messages.append({"role": "system", "content": "Please introduce yourself to the user."}) - await task.queue_frames([context_aggregator.user().get_context_frame()]) + runner = PipelineRunner(handle_sigint=handle_sigint) - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") + await runner.run(task) - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") - await task.cancel() - - runner = PipelineRunner(handle_sigint=False) - - await runner.run(task) + await aiohttp_session.close() if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/38a-smart-turn-local-coreml.py b/examples/foundational/38a-smart-turn-local-coreml.py index 4168e8bbc..ceedfcfbe 100644 --- a/examples/foundational/38a-smart-turn-local-coreml.py +++ b/examples/foundational/38a-smart-turn-local-coreml.py @@ -21,44 +21,62 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH +# to the path where the smart-turn repo is cloned. +# +# Example setup: +# +# # Git LFS (Large File Storage) +# brew install git-lfs +# # Hugging Face uses LFS to store large model files, including .mlpackage +# git lfs install +# # Clone the repo with the smart_turn_classifier.mlpackage +# git clone https://huggingface.co/pipecat-ai/smart-turn +# +# Then set the env variable: +# export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn +# or add it to your .env file +smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") - - # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH - # to the path where the smart-turn repo is cloned. - # - # Example setup: - # - # # Git LFS (Large File Storage) - # brew install git-lfs - # # Hugging Face uses LFS to store large model files, including .mlpackage - # git lfs install - # # Clone the repo with the smart_turn_classifier.mlpackage - # git clone https://huggingface.co/pipecat-ai/smart-turn - # - # Then set the env variable: - # export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn - # or add it to your .env file - smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") - - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalCoreMLSmartTurnAnalyzer( - smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() - ), +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalCoreMLSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() ), - ) + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalCoreMLSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() + ), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalCoreMLSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() + ), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -111,18 +129,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/38b-smart-turn-local.py b/examples/foundational/38b-smart-turn-local.py index e95d06eac..42fec18ad 100644 --- a/examples/foundational/38b-smart-turn-local.py +++ b/examples/foundational/38b-smart-turn-local.py @@ -21,44 +21,62 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH +# to the path where the smart-turn repo is cloned. +# +# Example setup: +# +# # Git LFS (Large File Storage) +# brew install git-lfs +# # Hugging Face uses LFS to store large model files, including .mlpackage +# git lfs install +# # Clone the repo with the smart_turn_classifier.mlpackage +# git clone https://huggingface.co/pipecat-ai/smart-turn +# +# Then set the env variable: +# export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn +# or add it to your .env file +smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") - - # To use this locally, set the environment variable LOCAL_SMART_TURN_MODEL_PATH - # to the path where the smart-turn repo is cloned. - # - # Example setup: - # - # # Git LFS (Large File Storage) - # brew install git-lfs - # # Hugging Face uses LFS to store large model files, including .mlpackage - # git lfs install - # # Clone the repo with the smart_turn_classifier.mlpackage - # git clone https://huggingface.co/pipecat-ai/smart-turn - # - # Then set the env variable: - # export LOCAL_SMART_TURN_MODEL_PATH=./smart-turn - # or add it to your .env file - smart_turn_model_path = os.getenv("LOCAL_SMART_TURN_MODEL_PATH") - - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzer( - smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() - ), +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() ), - ) + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() + ), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + turn_analyzer=LocalSmartTurnAnalyzer( + smart_turn_model_path=smart_turn_model_path, params=SmartTurnParams() + ), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -111,18 +129,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index 1c23931d2..235a38523 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -33,9 +33,8 @@ from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.mcp_service import MCPClient -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -81,20 +80,31 @@ class UrlToImageProcessor(FrameProcessor): logger.error(error_msg) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_out_enabled=True, - video_out_width=1024, - video_out_height=1024, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create an HTTP session for API calls async with aiohttp.ClientSession() as session: @@ -127,15 +137,15 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac tools = await mcp.register_tools(llm) system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. + You are a helpful LLM in a WebRTC call. + Your goal is to demonstrate your capabilities in a succinct way. You have access to a number of tools provided by NASA MCP. Use any and all tools to help users. When asked for the astronomy picture of the day, PASS in NO date to the API. This ensures we get the latest picture available. If as specific date is asked for, you can pass in that date to the API. - Your output will be converted to audio so don't include special characters in your answers. - Respond to what the user said in a creative and helpful way. - Don't overexplain what you are doing. + Your output will be converted to audio so don't include special characters in your answers. + Respond to what the user said in a creative and helpful way. + Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ @@ -174,18 +184,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py index a567f4d10..c02245174 100644 --- a/examples/foundational/39a-mcp-run-sse.py +++ b/examples/foundational/39a-mcp-run-sse.py @@ -6,9 +6,7 @@ import argparse import os -import sys -import aiohttp from dotenv import load_dotenv from loguru import logger @@ -17,30 +15,41 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.mcp_service import MCPClient -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) tts = CartesiaTTSService( @@ -62,13 +71,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac tools = await mcp.register_tools(llm) system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. + You are a helpful LLM in a WebRTC call. + Your goal is to demonstrate your capabilities in a succinct way. You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be converted to audio so don't include special characters in your answers. - Respond to what the user said in a creative and helpful way. + Your output will be converted to audio so don't include special characters in your answers. + Respond to what the user said in a creative and helpful way. When asked for today's date, use 'https://www.datetoday.net/'. - Don't overexplain what you are doing. + Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ @@ -106,18 +115,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index 2ce0dc201..fe2869a52 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -10,7 +10,6 @@ import io import os import re import shutil -import sys import aiohttp from dotenv import load_dotenv @@ -34,9 +33,8 @@ from pipecat.services.anthropic.llm import AnthropicLLMService from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.mcp_service import MCPClient -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) @@ -82,20 +80,31 @@ class UrlToImageProcessor(FrameProcessor): logger.error(error_msg) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_out_enabled=True, + video_out_width=1024, + video_out_height=1024, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - video_out_enabled=True, - video_out_width=1024, - video_out_height=1024, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Create an HTTP session for API calls async with aiohttp.ClientSession() as session: @@ -111,13 +120,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac ) system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. + You are a helpful LLM in a WebRTC call. + Your goal is to demonstrate your capabilities in a succinct way. You have access to a number of tools provided by NASA MCP. Use any and all tools to help users. When asked for today's date, use 'https://www.datetoday.net/'. When asked for the astronomy picture of the day, use 'https://www.datetoday.net/', to get today's date. - Your output will be converted to audio so don't include special characters in your answers. - Respond to what the user said in a creative and helpful way. + Your output will be converted to audio so don't include special characters in your answers. + Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. Just respond with short sentences when you are carrying out tool calls. """ @@ -185,18 +194,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/40-aws-nova-sonic.py b/examples/foundational/40-aws-nova-sonic.py index 4ed533e18..38fc4c189 100644 --- a/examples/foundational/40-aws-nova-sonic.py +++ b/examples/foundational/40-aws-nova-sonic.py @@ -14,16 +14,15 @@ from loguru import logger from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.aws_nova_sonic import AWSNovaSonicLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams # Load environment variables load_dotenv(override=True) @@ -62,20 +61,30 @@ weather_function = FunctionSchema( tools = ToolsSchema(standard_tools=[weather_function]) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - # Initialize the SmallWebRTCTransport with the connection - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_in_sample_rate=16000, - audio_out_enabled=True, - camera_in_enabled=False, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") # Specify initial system instruction. # HACK: note that, for now, we need to inject a special bit of text into this instruction to @@ -156,18 +165,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() # Run the pipeline - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/41a-text-only-webrtc.py b/examples/foundational/41a-text-only-webrtc.py new file mode 100644 index 000000000..bfd7a6051 --- /dev/null +++ b/examples/foundational/41a-text-only-webrtc.py @@ -0,0 +1,161 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.frames.frames import ( + LLMMessagesAppendFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frameworks.rtvi import ( + ActionResult, + RTVIAction, + RTVIActionArgument, + RTVIConfig, + RTVIObserver, + RTVIProcessor, + RTVIServerMessageFrame, +) +from pipecat.services.openai import OpenAIContextAggregatorPair +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + + +# This is an example of a text-only chatbot using small webrtc tranport. +# It uses the small webrtc transport prebuilt web UI. +# https://github.com/pipecat-ai/small-webrtc-prebuilt + + +def create_action_llm_append_to_messages(context_aggregator: OpenAIContextAggregatorPair): + async def action_llm_append_to_messages_handler( + rtvi: RTVIProcessor, service: str, arguments: dict[str, any] + ) -> ActionResult: + run_immediately = arguments["run_immediately"] if "run_immediately" in arguments else True + logger.info(f"run_immediately: {run_immediately}") + if run_immediately: + await rtvi.interrupt_bot() + # We just interrupted the bot so it should be fine to use the + # context directly instead of through frame. + if "messages" in arguments and arguments["messages"]: + frame = LLMMessagesAppendFrame(messages=arguments["messages"]) + await rtvi.push_frame(frame) + + frame = context_aggregator.user().get_context_frame() + await rtvi.push_frame(frame) + return True + + action_llm_append_to_messages = RTVIAction( + service="llm", + action="append_to_messages", + result="bool", + arguments=[ + RTVIActionArgument(name="messages", type="array"), + RTVIActionArgument(name="run_immediately", type="bool"), + ], + handler=action_llm_append_to_messages_handler, + ) + return action_llm_append_to_messages + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "webrtc": lambda: TransportParams(), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + action_llm_append_to_messages = create_action_llm_append_to_messages(context_aggregator) + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + rtvi.register_action(action_llm_append_to_messages) + + pipeline = Pipeline( + [ + transport.input(), + rtvi, + context_aggregator.user(), + llm, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + observers=[RTVIObserver(rtvi)], + ) + + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + logger.info("Pipecat client ready.") + await rtvi.set_bot_ready() + + # This block is frontend UI specific + # These messages are intended for small webrtc UI to only handle text + # https://github.com/pipecat-ai/small-webrtc-prebuilt + messages = { + "show_text_container": True, + "show_video_container": False, + "show_debug_container": False, + } + + rtvi_frame = RTVIServerMessageFrame(data=messages) + await task.queue_frames([rtvi_frame]) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from pipecat.examples.run import main + + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/41b-text-and-audio-webrtc.py b/examples/foundational/41b-text-and-audio-webrtc.py new file mode 100644 index 000000000..5ac250286 --- /dev/null +++ b/examples/foundational/41b-text-and-audio-webrtc.py @@ -0,0 +1,177 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import ( + LLMMessagesAppendFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frameworks.rtvi import ( + ActionResult, + RTVIAction, + RTVIActionArgument, + RTVIConfig, + RTVIObserver, + RTVIProcessor, + RTVIServerMessageFrame, +) +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai import OpenAIContextAggregatorPair +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + +# This is an example of a chatbot in which a user can speak and/or type text to communicate with the bot. +# It uses the small webrtc transport prebuilt web UI. +# https://github.com/pipecat-ai/small-webrtc-prebuilt + + +def create_action_llm_append_to_messages(context_aggregator: OpenAIContextAggregatorPair): + async def action_llm_append_to_messages_handler( + rtvi: RTVIProcessor, service: str, arguments: dict[str, any] + ) -> ActionResult: + run_immediately = arguments["run_immediately"] if "run_immediately" in arguments else True + + if run_immediately: + await rtvi.interrupt_bot() + + # We just interrupted the bot so it should be fine to use the + # context directly instead of through frame. + if "messages" in arguments and arguments["messages"]: + mess = arguments["messages"] + frame = LLMMessagesAppendFrame(messages=arguments["messages"]) + await rtvi.push_frame(frame) + + if run_immediately: + frame = context_aggregator.user().get_context_frame() + await rtvi.push_frame(frame) + + return True + + action_llm_append_to_messages = RTVIAction( + service="llm", + action="append_to_messages", + result="bool", + arguments=[ + RTVIActionArgument(name="messages", type="array"), + RTVIActionArgument(name="run_immediately", type="bool"), + ], + handler=action_llm_append_to_messages_handler, + ) + return action_llm_append_to_messages + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Respond to what the user says in a creative and helpful way. Explain to the User they can speak or type text to communicate with you.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + action_llm_append_to_messages = create_action_llm_append_to_messages(context_aggregator) + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + rtvi.register_action(action_llm_append_to_messages) + + pipeline = Pipeline( + [ + transport.input(), + rtvi, + stt, + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + observers=[RTVIObserver(rtvi)], + ) + + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + logger.info("Pipecat client ready.") + await rtvi.set_bot_ready() + + # This block is frontend UI specific + # These messages are intended for small webrtc UI to only handle text + # https://github.com/pipecat-ai/small-webrtc-prebuilt + messages = { + "show_text_container": True, + "show_debug_container": False, + } + rtvi_frame = RTVIServerMessageFrame(data=messages) + await task.queue_frames([rtvi_frame]) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + + @transport.event_handler("on_client_closed") + async def on_client_closed(transport, client): + logger.info(f"Client closed connection") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + from pipecat.examples.run import main + + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/42-interruption-config.py b/examples/foundational/42-interruption-config.py new file mode 100644 index 000000000..3cb64c204 --- /dev/null +++ b/examples/foundational/42-interruption-config.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import MinWordsInterruptionStrategy +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + + transcript = TranscriptProcessor() + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + interruption_strategies=[MinWordsInterruptionStrategy(min_words=3)], + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected") + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for message in frame.messages: + logger.info(f"Transcription [{message.role}]: {message.content}") + + runner = PipelineRunner(handle_sigint=handle_sigint) + + await runner.run(task) + + +if __name__ == "__main__": + from pipecat.examples.run import main + + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/README.md b/examples/foundational/README.md index 3c77a7bef..16e3ce42f 100644 --- a/examples/foundational/README.md +++ b/examples/foundational/README.md @@ -32,11 +32,42 @@ Depending on what you're trying to build, these learning paths will guide you th 4. Run any example: ```bash - python run.py 01-say-one-thing.py + python 01-say-one-thing.py ``` 5. Open the web interface at http://localhost:7860 and click "Connect" +## Running examples with other transports + +It is possible to run most of the examples with other transports such as Twilio or Daily. + +### Daily + +You need to create a Daily account at https://dashboard.daily.co/u/signup. Once signed up, you can create your own room from the dashboard and set the environment variables `DAILY_SAMPLE_ROOM_URL` and `DAILY_API_KEY`. Alternatively, you can let the example create a room for you (still needs `DAILY_API_KEY` environment variable). Then, start any example with `-t daily`: + +```bash +python 07-interruptible.py -t daily +``` + +### Twilio + +It is also possible to run the example through a Twilio phone number. You will +need to setup a few things: + +1. Install and run [ngrok](https://ngrok.com/download). + + ```bash + ngrok http 7860 + ``` + +2. Configure your Twilio phone number. One way is to setup a TwiML app and set the request URL to the ngrok URL from step (1). Then, set your phone number to use the new TwiML app. + +Then, run the example with: + +```bash +python 07-interruptible.py -t twilio -x NGROK_HOST_NAME (no protocol) +``` + ## Examples by Feature ### Basics @@ -109,7 +140,7 @@ Depending on what you're trying to build, these learning paths will guide you th ### Customizing Network Settings ```bash -python run.py --host 0.0.0.0 --port 8080 +python --host 0.0.0.0 --port 8080 ``` ### Troubleshooting diff --git a/examples/foundational/assets/office-ambience-24000-mono.mp3 b/examples/foundational/assets/office-ambience-24000-mono.mp3 new file mode 100644 index 000000000..60a452694 Binary files /dev/null and b/examples/foundational/assets/office-ambience-24000-mono.mp3 differ diff --git a/examples/foundational/requirements.txt b/examples/foundational/requirements.txt index 18e2f367c..8e4255cf8 100644 --- a/examples/foundational/requirements.txt +++ b/examples/foundational/requirements.txt @@ -1,5 +1,5 @@ fastapi uvicorn python-dotenv -pipecat-ai[webrtc,deepgram,cartesia] +pipecat-ai[webrtc,daily,deepgram,cartesia] pipecat-ai-small-webrtc-prebuilt \ No newline at end of file diff --git a/examples/foundational/run.py b/examples/foundational/run.py deleted file mode 100644 index e7012c9e9..000000000 --- a/examples/foundational/run.py +++ /dev/null @@ -1,205 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import argparse -import asyncio -import importlib.util -import os -import sys -from contextlib import asynccontextmanager -from inspect import iscoroutinefunction, signature -from typing import Any, Callable, Dict, Optional, Tuple - -import uvicorn -from dotenv import load_dotenv -from fastapi import BackgroundTasks, FastAPI -from fastapi.responses import RedirectResponse -from loguru import logger -from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI - -from pipecat.transports.network.webrtc_connection import IceServer, SmallWebRTCConnection - -# Load environment variables -load_dotenv(override=True) - -app = FastAPI() - -# Store connections by pc_id -pcs_map: Dict[str, SmallWebRTCConnection] = {} - -ice_servers = [ - IceServer( - urls="stun:stun.l.google.com:19302", - ) -] - -# Mount the frontend at / -app.mount("/client", SmallWebRTCPrebuiltUI) - -# Store program arguments -args: argparse.Namespace = argparse.Namespace() - -# Store the bot module and function info -bot_module: Any = None -run_bot_func: Optional[Callable] = None -is_webrtc_bot: bool = True - - -def import_bot_file(file_path: str) -> Tuple[Any, Callable, bool]: - """Dynamically import the bot file and determine how to run it. - - Returns: - tuple: (module, run_function, is_webrtc_bot) - - module: The imported module - - run_function: Either run_bot or main function - - is_webrtc_bot: True if run_bot function exists and accepts a WebRTC connection - """ - if not os.path.exists(file_path): - raise FileNotFoundError(f"Bot file not found: {file_path}") - - # Extract module name without extension - module_name = os.path.splitext(os.path.basename(file_path))[0] - - # Load the module - spec = importlib.util.spec_from_file_location(module_name, file_path) - if not spec or not spec.loader: - raise ImportError(f"Could not load spec for {file_path}") - - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - - # Check for run_bot function first - if hasattr(module, "run_bot"): - run_func = module.run_bot - # Check if the function accepts a WebRTC connection - sig = signature(run_func) - is_webrtc = len(sig.parameters) > 0 - return module, run_func, is_webrtc - - # Fall back to main function - if hasattr(module, "main") and iscoroutinefunction(module.main): - return module, module.main, False - - raise AttributeError(f"No run_bot or async main function found in {file_path}") - - -@app.get("/", include_in_schema=False) -async def root_redirect(): - return RedirectResponse(url="/client/") - - -@app.post("/api/offer") -async def offer(request: dict, background_tasks: BackgroundTasks): - global run_bot_func, is_webrtc_bot - - if not run_bot_func: - raise RuntimeError("No bot file has been loaded") - - if not is_webrtc_bot: - return { - "error": "This bot doesn't support WebRTC connections, it's running in standalone mode" - } - - pc_id = request.get("pc_id") - - if pc_id and pc_id in pcs_map: - pipecat_connection = pcs_map[pc_id] - logger.info(f"Reusing existing connection for pc_id: {pc_id}") - await pipecat_connection.renegotiate( - sdp=request["sdp"], type=request["type"], restart_pc=request.get("restart_pc", False) - ) - else: - pipecat_connection = SmallWebRTCConnection(ice_servers) - await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"]) - - @pipecat_connection.event_handler("closed") - async def handle_disconnected(webrtc_connection: SmallWebRTCConnection): - logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}") - pcs_map.pop(webrtc_connection.pc_id, None) - - # We've already checked that run_bot_func exists - assert run_bot_func is not None - background_tasks.add_task(run_bot_func, pipecat_connection, args) - - answer = pipecat_connection.get_answer() - # Updating the peer connection inside the map - pcs_map[answer["pc_id"]] = pipecat_connection - - return answer - - -@asynccontextmanager -async def lifespan(app: FastAPI): - yield # Run app - coros = [pc.close() for pc in pcs_map.values()] - await asyncio.gather(*coros) - pcs_map.clear() - - -async def run_standalone_bot() -> None: - """Run a standalone bot that doesn't require WebRTC""" - global run_bot_func - if run_bot_func is not None: - await run_bot_func() - else: - raise RuntimeError("No bot function available to run") - - -def main(parser: Optional[argparse.ArgumentParser] = None): - global args - - if not parser: - parser = argparse.ArgumentParser(description="Pipecat Bot Runner") - parser.add_argument("bot_file", nargs="?", help="Path to the bot file", default=None) - parser.add_argument( - "--host", default="localhost", help="Host for HTTP server (default: localhost)" - ) - parser.add_argument( - "--port", type=int, default=7860, help="Port for HTTP server (default: 7860)" - ) - parser.add_argument("--verbose", "-v", action="count", default=0) - args = parser.parse_args() - - logger.remove(0) - if args.verbose: - logger.add(sys.stderr, level="TRACE") - else: - logger.add(sys.stderr, level="DEBUG") - - # Infer the bot file from the caller if not provided explicitly - bot_file = args.bot_file - if bot_file is None: - # Get the __file__ of the script that called main() - import inspect - - caller_frame = inspect.stack()[1] - caller_globals = caller_frame.frame.f_globals - bot_file = caller_globals.get("__file__") - - if not bot_file: - print("❌ Could not determine the bot file. Pass it explicitly to main().") - sys.exit(1) - - # Import the bot file - try: - global run_bot_func, bot_module, is_webrtc_bot - bot_module, run_bot_func, is_webrtc_bot = import_bot_file(bot_file) - logger.info(f"Successfully loaded bot from {bot_file}") - - if is_webrtc_bot: - logger.info("Detected WebRTC-compatible bot, starting web server...") - uvicorn.run(app, host=args.host, port=args.port) - else: - logger.info("Detected standalone bot, running directly...") - asyncio.run(run_standalone_bot()) - except Exception as e: - logger.error(f"Error loading bot file: {e}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/examples/open-telemetry/jaeger/bot.py b/examples/open-telemetry/jaeger/bot.py index 18fe34ef4..0f3084fd3 100644 --- a/examples/open-telemetry/jaeger/bot.py +++ b/examples/open-telemetry/jaeger/bot.py @@ -6,7 +6,6 @@ import argparse import os -import sys from dotenv import load_dotenv from loguru import logger @@ -24,9 +23,8 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams from pipecat.utils.tracing.setup import setup_tracing load_dotenv(override=True) @@ -51,21 +49,33 @@ if IS_TRACING_ENABLED: async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -82,6 +92,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -143,19 +157,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/open-telemetry/langfuse/bot.py b/examples/open-telemetry/langfuse/bot.py index 9f311970e..9c4f34695 100644 --- a/examples/open-telemetry/langfuse/bot.py +++ b/examples/open-telemetry/langfuse/bot.py @@ -6,7 +6,6 @@ import argparse import os -import sys from dotenv import load_dotenv from loguru import logger @@ -24,9 +23,8 @@ from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams from pipecat.services.openai.llm import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.network.small_webrtc import SmallWebRTCTransport -from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.services.daily import DailyParams from pipecat.utils.tracing.setup import setup_tracing load_dotenv(override=True) @@ -48,21 +46,33 @@ if IS_TRACING_ENABLED: async def fetch_weather_from_api(params: FunctionCallParams): - await params.llm.push_frame(TTSSpeakFrame("Let me check on that.")) await params.result_callback({"conditions": "nice", "temperature": "75"}) -async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace): - logger.info(f"Starting bot") +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} - transport = SmallWebRTCTransport( - webrtc_connection=webrtc_connection, - params=TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -79,6 +89,10 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac # sent to the same callback with an additional function_name parameter. llm.register_function("get_current_weather", fetch_weather_from_api) + @llm.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + await tts.queue_frame(TTSSpeakFrame("Let me check on that.")) + weather_function = FunctionSchema( name="get_current_weather", description="Get the current weather", @@ -140,19 +154,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") - - @transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info(f"Client closed connection") await task.cancel() - runner = PipelineRunner(handle_sigint=False) + runner = PipelineRunner(handle_sigint=handle_sigint) await runner.run(task) if __name__ == "__main__": - sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) - from run import main + from pipecat.examples.run import main - main() + main(run_example, transport_params=transport_params) diff --git a/examples/open-telemetry/run.py b/examples/open-telemetry/run.py deleted file mode 100644 index e7012c9e9..000000000 --- a/examples/open-telemetry/run.py +++ /dev/null @@ -1,205 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import argparse -import asyncio -import importlib.util -import os -import sys -from contextlib import asynccontextmanager -from inspect import iscoroutinefunction, signature -from typing import Any, Callable, Dict, Optional, Tuple - -import uvicorn -from dotenv import load_dotenv -from fastapi import BackgroundTasks, FastAPI -from fastapi.responses import RedirectResponse -from loguru import logger -from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI - -from pipecat.transports.network.webrtc_connection import IceServer, SmallWebRTCConnection - -# Load environment variables -load_dotenv(override=True) - -app = FastAPI() - -# Store connections by pc_id -pcs_map: Dict[str, SmallWebRTCConnection] = {} - -ice_servers = [ - IceServer( - urls="stun:stun.l.google.com:19302", - ) -] - -# Mount the frontend at / -app.mount("/client", SmallWebRTCPrebuiltUI) - -# Store program arguments -args: argparse.Namespace = argparse.Namespace() - -# Store the bot module and function info -bot_module: Any = None -run_bot_func: Optional[Callable] = None -is_webrtc_bot: bool = True - - -def import_bot_file(file_path: str) -> Tuple[Any, Callable, bool]: - """Dynamically import the bot file and determine how to run it. - - Returns: - tuple: (module, run_function, is_webrtc_bot) - - module: The imported module - - run_function: Either run_bot or main function - - is_webrtc_bot: True if run_bot function exists and accepts a WebRTC connection - """ - if not os.path.exists(file_path): - raise FileNotFoundError(f"Bot file not found: {file_path}") - - # Extract module name without extension - module_name = os.path.splitext(os.path.basename(file_path))[0] - - # Load the module - spec = importlib.util.spec_from_file_location(module_name, file_path) - if not spec or not spec.loader: - raise ImportError(f"Could not load spec for {file_path}") - - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - - # Check for run_bot function first - if hasattr(module, "run_bot"): - run_func = module.run_bot - # Check if the function accepts a WebRTC connection - sig = signature(run_func) - is_webrtc = len(sig.parameters) > 0 - return module, run_func, is_webrtc - - # Fall back to main function - if hasattr(module, "main") and iscoroutinefunction(module.main): - return module, module.main, False - - raise AttributeError(f"No run_bot or async main function found in {file_path}") - - -@app.get("/", include_in_schema=False) -async def root_redirect(): - return RedirectResponse(url="/client/") - - -@app.post("/api/offer") -async def offer(request: dict, background_tasks: BackgroundTasks): - global run_bot_func, is_webrtc_bot - - if not run_bot_func: - raise RuntimeError("No bot file has been loaded") - - if not is_webrtc_bot: - return { - "error": "This bot doesn't support WebRTC connections, it's running in standalone mode" - } - - pc_id = request.get("pc_id") - - if pc_id and pc_id in pcs_map: - pipecat_connection = pcs_map[pc_id] - logger.info(f"Reusing existing connection for pc_id: {pc_id}") - await pipecat_connection.renegotiate( - sdp=request["sdp"], type=request["type"], restart_pc=request.get("restart_pc", False) - ) - else: - pipecat_connection = SmallWebRTCConnection(ice_servers) - await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"]) - - @pipecat_connection.event_handler("closed") - async def handle_disconnected(webrtc_connection: SmallWebRTCConnection): - logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}") - pcs_map.pop(webrtc_connection.pc_id, None) - - # We've already checked that run_bot_func exists - assert run_bot_func is not None - background_tasks.add_task(run_bot_func, pipecat_connection, args) - - answer = pipecat_connection.get_answer() - # Updating the peer connection inside the map - pcs_map[answer["pc_id"]] = pipecat_connection - - return answer - - -@asynccontextmanager -async def lifespan(app: FastAPI): - yield # Run app - coros = [pc.close() for pc in pcs_map.values()] - await asyncio.gather(*coros) - pcs_map.clear() - - -async def run_standalone_bot() -> None: - """Run a standalone bot that doesn't require WebRTC""" - global run_bot_func - if run_bot_func is not None: - await run_bot_func() - else: - raise RuntimeError("No bot function available to run") - - -def main(parser: Optional[argparse.ArgumentParser] = None): - global args - - if not parser: - parser = argparse.ArgumentParser(description="Pipecat Bot Runner") - parser.add_argument("bot_file", nargs="?", help="Path to the bot file", default=None) - parser.add_argument( - "--host", default="localhost", help="Host for HTTP server (default: localhost)" - ) - parser.add_argument( - "--port", type=int, default=7860, help="Port for HTTP server (default: 7860)" - ) - parser.add_argument("--verbose", "-v", action="count", default=0) - args = parser.parse_args() - - logger.remove(0) - if args.verbose: - logger.add(sys.stderr, level="TRACE") - else: - logger.add(sys.stderr, level="DEBUG") - - # Infer the bot file from the caller if not provided explicitly - bot_file = args.bot_file - if bot_file is None: - # Get the __file__ of the script that called main() - import inspect - - caller_frame = inspect.stack()[1] - caller_globals = caller_frame.frame.f_globals - bot_file = caller_globals.get("__file__") - - if not bot_file: - print("❌ Could not determine the bot file. Pass it explicitly to main().") - sys.exit(1) - - # Import the bot file - try: - global run_bot_func, bot_module, is_webrtc_bot - bot_module, run_bot_func, is_webrtc_bot = import_bot_file(bot_file) - logger.info(f"Successfully loaded bot from {bot_file}") - - if is_webrtc_bot: - logger.info("Detected WebRTC-compatible bot, starting web server...") - uvicorn.run(app, host=args.host, port=args.port) - else: - logger.info("Detected standalone bot, running directly...") - asyncio.run(run_standalone_bot()) - except Exception as e: - logger.error(f"Error loading bot file: {e}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/examples/p2p-webrtc/daily-interop-bridge/bot.py b/examples/p2p-webrtc/daily-interop-bridge/bot.py index 0e859b5a0..659d3fcef 100644 --- a/examples/p2p-webrtc/daily-interop-bridge/bot.py +++ b/examples/p2p-webrtc/daily-interop-bridge/bot.py @@ -112,10 +112,6 @@ async def run_bot(webrtc_connection): @pipecat_transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info("Pipecat Client disconnected") - - @pipecat_transport.event_handler("on_client_closed") - async def on_client_closed(transport, client): - logger.info("Pipecat Client closed") await task.cancel() runner = PipelineRunner(handle_sigint=False) diff --git a/examples/p2p-webrtc/daily-interop-bridge/server.py b/examples/p2p-webrtc/daily-interop-bridge/server.py index 0025b3490..c8dae7888 100644 --- a/examples/p2p-webrtc/daily-interop-bridge/server.py +++ b/examples/p2p-webrtc/daily-interop-bridge/server.py @@ -74,7 +74,7 @@ async def offer(request: dict, background_tasks: BackgroundTasks): @asynccontextmanager async def lifespan(app: FastAPI): yield # Run app - coros = [pc.close() for pc in pcs_map.values()] + coros = [pc.disconnect() for pc in pcs_map.values()] await asyncio.gather(*coros) pcs_map.clear() diff --git a/examples/p2p-webrtc/video-transform/client/android/.gitignore b/examples/p2p-webrtc/video-transform/client/android/.gitignore new file mode 100644 index 000000000..10cfdbfaf --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/.gitignore @@ -0,0 +1,10 @@ +*.iml +.gradle +/local.properties +/.idea +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/examples/p2p-webrtc/video-transform/client/android/LICENSE b/examples/p2p-webrtc/video-transform/client/android/LICENSE new file mode 100644 index 000000000..976aa72fe --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/LICENSE @@ -0,0 +1,24 @@ +BSD 2-Clause License + +Copyright (c) 2024–2025, Daily + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/p2p-webrtc/video-transform/client/android/README.md b/examples/p2p-webrtc/video-transform/client/android/README.md new file mode 100644 index 000000000..d0718b88b --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/README.md @@ -0,0 +1,12 @@ +# Pipecat Small WebRTC Client for Android + +Demo app which connects to the `video-transform` backend over the small WebRTC transport. + +## How to run + +```bash +./gradlew runDebug +``` + +Ensure that the `video-transform` server is running as described in the parent README. + diff --git a/examples/p2p-webrtc/video-transform/client/android/build.gradle.kts b/examples/p2p-webrtc/video-transform/client/android/build.gradle.kts new file mode 100644 index 000000000..861303b1f --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/build.gradle.kts @@ -0,0 +1,5 @@ +plugins { + alias(libs.plugins.jetbrains.kotlin.android) apply false + alias(libs.plugins.android.application) apply false + alias(libs.plugins.compose.compiler) apply false +} diff --git a/examples/p2p-webrtc/video-transform/client/android/gradle.properties b/examples/p2p-webrtc/video-transform/client/android/gradle.properties new file mode 100644 index 000000000..20e2a0152 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/gradle.properties @@ -0,0 +1,23 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. For more details, visit +# https://developer.android.com/r/tools/gradle-multi-project-decoupled-projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/gradle/libs.versions.toml b/examples/p2p-webrtc/video-transform/client/android/gradle/libs.versions.toml new file mode 100644 index 000000000..be90d31de --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/gradle/libs.versions.toml @@ -0,0 +1,34 @@ +[versions] +accompanistPermissions = "0.34.0" +agp = "8.7.3" +constraintlayoutCompose = "1.1.0" +pipecatClient = "0.3.7" +kotlin = "2.0.20" +coreKtx = "1.15.0" +lifecycleRuntimeKtx = "2.8.7" +activityCompose = "1.9.3" +composeBom = "2024.12.01" +kotlinxSerializationJson = "1.7.1" +kotlinxSerializationPlugin = "2.0.20" + +[libraries] +accompanist-permissions = { module = "com.google.accompanist:accompanist-permissions", version.ref = "accompanistPermissions" } +androidx-constraintlayout-compose = { module = "androidx.constraintlayout:constraintlayout-compose", version.ref = "constraintlayoutCompose" } +androidx-core-ktx = { group = "androidx.core", name = "core-ktx", version.ref = "coreKtx" } +pipecat-client-smallwebrtc = { module = "ai.pipecat:small-webrtc-transport", version.ref = "pipecatClient" } +androidx-lifecycle-runtime-ktx = { group = "androidx.lifecycle", name = "lifecycle-runtime-ktx", version.ref = "lifecycleRuntimeKtx" } +androidx-activity-compose = { group = "androidx.activity", name = "activity-compose", version.ref = "activityCompose" } +androidx-compose-bom = { group = "androidx.compose", name = "compose-bom", version.ref = "composeBom" } +androidx-ui = { group = "androidx.compose.ui", name = "ui" } +androidx-ui-graphics = { group = "androidx.compose.ui", name = "ui-graphics" } +androidx-ui-tooling = { group = "androidx.compose.ui", name = "ui-tooling" } +androidx-ui-tooling-preview = { group = "androidx.compose.ui", name = "ui-tooling-preview" } +androidx-ui-test-manifest = { group = "androidx.compose.ui", name = "ui-test-manifest" } +androidx-material3 = { group = "androidx.compose.material3", name = "material3" } +kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } + +[plugins] +jetbrains-kotlin-android = { id = "org.jetbrains.kotlin.android", version.ref = "kotlin" } +android-application = { id = "com.android.application", version.ref = "agp" } +compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } +jetbrains-kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlinxSerializationPlugin" } diff --git a/examples/p2p-webrtc/video-transform/client/android/gradle/wrapper/gradle-wrapper.jar b/examples/p2p-webrtc/video-transform/client/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..e708b1c02 Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/p2p-webrtc/video-transform/client/android/gradle/wrapper/gradle-wrapper.properties b/examples/p2p-webrtc/video-transform/client/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..2c47b5b4b --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Aug 05 13:01:27 BST 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/examples/p2p-webrtc/video-transform/client/android/gradlew b/examples/p2p-webrtc/video-transform/client/android/gradlew new file mode 100755 index 000000000..4f906e0c8 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/examples/p2p-webrtc/video-transform/client/android/gradlew.bat b/examples/p2p-webrtc/video-transform/client/android/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/examples/p2p-webrtc/video-transform/client/android/settings.gradle.kts b/examples/p2p-webrtc/video-transform/client/android/settings.gradle.kts new file mode 100644 index 000000000..afd7e3285 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/settings.gradle.kts @@ -0,0 +1,24 @@ +pluginManagement { + repositories { + google { + content { + includeGroupByRegex("com\\.android.*") + includeGroupByRegex("com\\.google.*") + includeGroupByRegex("androidx.*") + } + } + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + mavenLocal() + } +} + +rootProject.name = "Pipecat Small WebRTC Client" +include(":small-webrtc-client") diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/.gitignore b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/.gitignore new file mode 100644 index 000000000..42afabfd2 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/build.gradle.kts b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/build.gradle.kts new file mode 100644 index 000000000..c1810cd14 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/build.gradle.kts @@ -0,0 +1,75 @@ +plugins { + alias(libs.plugins.android.application) + alias(libs.plugins.jetbrains.kotlin.android) + alias(libs.plugins.jetbrains.kotlin.serialization) + alias(libs.plugins.compose.compiler) +} + +android { + namespace = "ai.pipecat.small_webrtc_client" + compileSdk = 35 + + defaultConfig { + applicationId = "ai.pipecat.small_webrtc_client" + minSdk = 26 + targetSdk = 35 + versionCode = 1 + versionName = "1.0" + + vectorDrawables { + useSupportLibrary = true + } + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + kotlinOptions { + jvmTarget = "1.8" + } + + buildFeatures { + compose = true + buildConfig = true + } + + composeOptions { + kotlinCompilerExtensionVersion = "1.5.1" + } + + packaging { + resources { + excludes += "/META-INF/{AL2.0,LGPL2.1}" + } + } +} + +dependencies { + implementation(libs.pipecat.client.smallwebrtc) + implementation(libs.androidx.core.ktx) + implementation(libs.androidx.lifecycle.runtime.ktx) + implementation(libs.androidx.activity.compose) + implementation(platform(libs.androidx.compose.bom)) + implementation(libs.androidx.ui) + implementation(libs.androidx.ui.graphics) + implementation(libs.androidx.ui.tooling.preview) + implementation(libs.androidx.material3) + implementation(libs.accompanist.permissions) + implementation(libs.androidx.constraintlayout.compose) + implementation(libs.kotlinx.serialization.json) + androidTestImplementation(platform(libs.androidx.compose.bom)) + debugImplementation(libs.androidx.ui.tooling) + debugImplementation(libs.androidx.ui.test.manifest) +} diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/proguard-rules.pro b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/proguard-rules.pro new file mode 100644 index 000000000..481bb4348 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/AndroidManifest.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/AndroidManifest.xml new file mode 100644 index 000000000..397ea75cd --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/AndroidManifest.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/MainActivity.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/MainActivity.kt new file mode 100644 index 000000000..4e8339707 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/MainActivity.kt @@ -0,0 +1,257 @@ +package ai.pipecat.small_webrtc_client + +import ai.pipecat.small_webrtc_client.ui.InCallLayout +import ai.pipecat.small_webrtc_client.ui.PermissionScreen +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import ai.pipecat.small_webrtc_client.ui.theme.RTVIClientTheme +import ai.pipecat.small_webrtc_client.ui.theme.TextStyles +import ai.pipecat.small_webrtc_client.ui.theme.textFieldColors +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.activity.enableEdgeToEdge +import androidx.annotation.DrawableRes +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.Icon +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.material3.TextField +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.shadow +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.input.ImeAction +import androidx.compose.ui.text.input.KeyboardType +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp + + +class MainActivity : ComponentActivity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + enableEdgeToEdge() + + val voiceClientManager = VoiceClientManager(this) + + setContent { + RTVIClientTheme { + Scaffold(modifier = Modifier.fillMaxSize()) { innerPadding -> + Box( + Modifier + .fillMaxSize() + .padding(innerPadding) + ) { + PermissionScreen() + + val vcState = voiceClientManager.state.value + + if (vcState != null) { + InCallLayout(voiceClientManager) + + } else { + ConnectSettings(voiceClientManager) + } + + voiceClientManager.errors.firstOrNull()?.let { errorText -> + + val dismiss: () -> Unit = { voiceClientManager.errors.removeAt(0) } + + AlertDialog( + onDismissRequest = dismiss, + confirmButton = { + Button(onClick = dismiss) { + Text( + text = "OK", + fontSize = 14.sp, + fontWeight = FontWeight.W700, + color = Color.White, + style = TextStyles.base + ) + } + }, + containerColor = Color.White, + title = { + Text( + text = "Error", + fontSize = 22.sp, + fontWeight = FontWeight.W600, + color = Color.Black, + style = TextStyles.base + ) + }, + text = { + Text( + text = errorText.message, + fontSize = 16.sp, + fontWeight = FontWeight.W400, + color = Color.Black, + style = TextStyles.base + ) + } + ) + } + } + } + } + } + } +} + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun ConnectSettings( + voiceClientManager: VoiceClientManager, +) { + val scrollState = rememberScrollState() + + val start = { + val backendUrl = Preferences.backendUrl.value + + voiceClientManager.start(baseUrl = backendUrl!!) + } + + Box( + modifier = Modifier + .fillMaxSize() + .verticalScroll(scrollState) + .imePadding() + .padding(20.dp), + contentAlignment = Alignment.Center + ) { + Box( + Modifier + .fillMaxWidth() + .shadow(2.dp, RoundedCornerShape(16.dp)) + .clip(RoundedCornerShape(16.dp)) + .background(Colors.mainSurfaceBackground) + ) { + Column( + Modifier + .fillMaxWidth() + .padding( + vertical = 24.dp, + horizontal = 28.dp + ) + ) { + Spacer(modifier = Modifier.height(12.dp)) + + Text( + modifier = Modifier.align(Alignment.CenterHorizontally), + text = "Connect to WebRTC server", + fontSize = 22.sp, + fontWeight = FontWeight.W700, + style = TextStyles.base + ) + + Spacer(modifier = Modifier.height(36.dp)) + + Text( + text = "Backend URL", + fontSize = 16.sp, + fontWeight = FontWeight.W400, + style = TextStyles.base + ) + + Spacer(modifier = Modifier.height(12.dp)) + + TextField( + modifier = Modifier + .fillMaxWidth() + .border(1.dp, Colors.textFieldBorder, RoundedCornerShape(12.dp)), + value = Preferences.backendUrl.value ?: "", + onValueChange = { Preferences.backendUrl.value = it }, + keyboardOptions = KeyboardOptions( + keyboardType = KeyboardType.Uri, + imeAction = ImeAction.Next + ), + colors = textFieldColors(), + shape = RoundedCornerShape(12.dp) + ) + + Spacer(modifier = Modifier.height(36.dp)) + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(16.dp) + ) { + ConnectDialogButton( + modifier = Modifier.weight(1f), + onClick = start, + text = "Connect", + foreground = Color.White, + background = Colors.buttonNormal, + border = Colors.buttonNormal + ) + } + } + } + } +} + +@Composable +private fun ConnectDialogButton( + onClick: () -> Unit, + text: String, + foreground: Color, + background: Color, + border: Color, + modifier: Modifier = Modifier, + @DrawableRes icon: Int? = null, +) { + val shape = RoundedCornerShape(8.dp) + + Row( + modifier + .border(1.dp, border, shape) + .clip(shape) + .background(background) + .clickable(onClick = onClick) + .padding(vertical = 10.dp, horizontal = 24.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.Center + ) { + if (icon != null) { + Icon( + modifier = Modifier.size(24.dp), + painter = painterResource(icon), + tint = foreground, + contentDescription = null + ) + + Spacer(modifier = Modifier.width(8.dp)) + } + + Text( + text = text, + fontSize = 16.sp, + fontWeight = FontWeight.W500, + color = foreground + ) + } +} diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/Preferences.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/Preferences.kt new file mode 100644 index 000000000..667dc261a --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/Preferences.kt @@ -0,0 +1,75 @@ +package ai.pipecat.small_webrtc_client + +import android.content.Context +import android.content.SharedPreferences +import androidx.compose.runtime.mutableStateOf +import kotlinx.serialization.KSerializer +import kotlinx.serialization.json.Json + +private val JSON_INSTANCE = Json { ignoreUnknownKeys = true } + +object Preferences { + + private const val PREF_BACKEND_URL = "backend_url" + + private lateinit var prefs: SharedPreferences + + fun initAppStart(context: Context) { + prefs = context.applicationContext.getSharedPreferences("prefs", Context.MODE_PRIVATE) + + listOf(backendUrl).forEach { it.init() } + } + + private fun getString(key: String): String? = prefs.getString(key, null) + + interface BasePref { + fun init() + } + + class StringPref(private val key: String): BasePref { + private val cachedValue = mutableStateOf(null) + + override fun init() { + cachedValue.value = getString(key) + prefs.registerOnSharedPreferenceChangeListener { _, changedKey -> + if (key == changedKey) { + cachedValue.value = getString(key) + } + } + } + + var value: String? + get() = cachedValue.value + set(newValue) { + cachedValue.value = newValue + prefs.edit().putString(key, newValue).apply() + } + } + + class JsonPref(private val key: String, private var serializer: KSerializer): BasePref { + private val cachedValue = mutableStateOf(null) + + private fun lookupValue(): E? = + getString(key)?.let { JSON_INSTANCE.decodeFromString(serializer, it) } + + override fun init() { + cachedValue.value = lookupValue() + prefs.registerOnSharedPreferenceChangeListener { _, changedKey -> + if (key == changedKey) { + cachedValue.value = lookupValue() + } + } + } + + var value: E? + get() = cachedValue.value + set(newValue) { + cachedValue.value = newValue + prefs.edit() + .putString(key, newValue?.let { JSON_INSTANCE.encodeToString(serializer, it) }) + .apply() + } + } + + val backendUrl = StringPref(PREF_BACKEND_URL) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/RTVIApplication.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/RTVIApplication.kt new file mode 100644 index 000000000..5298deafc --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/RTVIApplication.kt @@ -0,0 +1,10 @@ +package ai.pipecat.small_webrtc_client + +import android.app.Application + +class RTVIApplication : Application() { + override fun onCreate() { + super.onCreate() + Preferences.initAppStart(this) + } +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/VoiceClientManager.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/VoiceClientManager.kt new file mode 100644 index 000000000..2225d8ac4 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/VoiceClientManager.kt @@ -0,0 +1,191 @@ +package ai.pipecat.small_webrtc_client + +import ai.pipecat.client.RTVIClient +import ai.pipecat.client.RTVIClientOptions +import ai.pipecat.client.RTVIClientParams +import ai.pipecat.client.RTVIEventCallbacks +import ai.pipecat.client.result.Future +import ai.pipecat.client.result.RTVIError +import ai.pipecat.client.result.Result +import ai.pipecat.client.small_webrtc_transport.SmallWebRTCTransport +import ai.pipecat.client.types.ActionDescription +import ai.pipecat.client.types.Participant +import ai.pipecat.client.types.PipecatMetrics +import ai.pipecat.client.types.ServiceConfig +import ai.pipecat.client.types.Tracks +import ai.pipecat.client.types.Transcript +import ai.pipecat.client.types.TransportState +import ai.pipecat.small_webrtc_client.utils.Timestamp +import android.content.Context +import android.util.Log +import androidx.compose.runtime.Immutable +import androidx.compose.runtime.Stable +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.mutableStateOf + +@Immutable +data class Error(val message: String) + +@Stable +class VoiceClientManager(private val context: Context) { + + companion object { + private const val TAG = "VoiceClientManager" + } + + private val client = mutableStateOf(null) + + val state = mutableStateOf(null) + + val errors = mutableStateListOf() + + val actionDescriptions = + mutableStateOf, RTVIError>?>(null) + + val expiryTime = mutableStateOf(null) + + val botReady = mutableStateOf(false) + val botIsTalking = mutableStateOf(false) + val userIsTalking = mutableStateOf(false) + val botAudioLevel = mutableFloatStateOf(0f) + val userAudioLevel = mutableFloatStateOf(0f) + + val mic = mutableStateOf(false) + val camera = mutableStateOf(false) + val tracks = mutableStateOf(null) + + private fun Future.displayErrors() = withErrorCallback { + Log.e(TAG, "Future resolved with error: ${it.description}", it.exception) + errors.add(Error(it.description)) + } + + fun start(baseUrl: String) { + + if (client.value != null) { + return + } + + val options = RTVIClientOptions( + params = RTVIClientParams(baseUrl = null), + enableMic = true, + enableCam = true + ) + + state.value = TransportState.Disconnected + + val callbacks = object : RTVIEventCallbacks() { + override fun onTransportStateChanged(state: TransportState) { + this@VoiceClientManager.state.value = state + } + + override fun onBackendError(message: String) { + "Error from backend: $message".let { + Log.e(TAG, it) + errors.add(Error(it)) + } + } + + override fun onBotReady(version: String, config: List) { + + Log.i(TAG, "Bot ready. Version $version, config: $config") + + botReady.value = true + + client.value?.describeActions()?.withCallback { + actionDescriptions.value = it + } + } + + override fun onPipecatMetrics(data: PipecatMetrics) { + Log.i(TAG, "Pipecat metrics: $data") + } + + override fun onUserTranscript(data: Transcript) { + Log.i(TAG, "User transcript: $data") + } + + override fun onBotTranscript(text: String) { + Log.i(TAG, "Bot transcript: $text") + } + + override fun onBotStartedSpeaking() { + Log.i(TAG, "Bot started speaking") + botIsTalking.value = true + } + + override fun onBotStoppedSpeaking() { + Log.i(TAG, "Bot stopped speaking") + botIsTalking.value = false + } + + override fun onUserStartedSpeaking() { + Log.i(TAG, "User started speaking") + userIsTalking.value = true + } + + override fun onUserStoppedSpeaking() { + Log.i(TAG, "User stopped speaking") + userIsTalking.value = false + } + + override fun onTracksUpdated(tracks: Tracks) { + this@VoiceClientManager.tracks.value = tracks + } + + override fun onInputsUpdated(camera: Boolean, mic: Boolean) { + this@VoiceClientManager.camera.value = camera + this@VoiceClientManager.mic.value = mic + } + + override fun onConnected() { + expiryTime.value = client.value?.expiry?.let(Timestamp::ofEpochSecs) + } + + override fun onDisconnected() { + expiryTime.value = null + actionDescriptions.value = null + botIsTalking.value = false + userIsTalking.value = false + state.value = null + actionDescriptions.value = null + botReady.value = false + tracks.value = null + + client.value?.release() + client.value = null + } + + override fun onUserAudioLevel(level: Float) { + userAudioLevel.floatValue = level + } + + override fun onRemoteAudioLevel(level: Float, participant: Participant) { + botAudioLevel.floatValue = level + } + } + + val client = RTVIClient(SmallWebRTCTransport.Factory(context, baseUrl), callbacks, options) + + client.connect().displayErrors().withErrorCallback { + callbacks.onDisconnected() + } + + this.client.value = client + } + + fun enableCamera(enabled: Boolean) { + client.value?.enableCam(enabled)?.displayErrors() + } + + fun enableMic(enabled: Boolean) { + client.value?.enableMic(enabled)?.displayErrors() + } + + fun toggleCamera() = enableCamera(!camera.value) + fun toggleMic() = enableMic(!mic.value) + + fun stop() { + client.value?.disconnect()?.displayErrors() + } +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/AudioIndicator.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/AudioIndicator.kt new file mode 100644 index 000000000..6d29f7cfa --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/AudioIndicator.kt @@ -0,0 +1,70 @@ +package ai.pipecat.small_webrtc_client.ui + +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.animateFloatAsState +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Canvas +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.StrokeCap +import androidx.compose.ui.semantics.clearAndSetSemantics + +@Composable +fun ListeningAnimation( + modifier: Modifier, + active: Boolean, + level: Float, + color: Color, +) { + val infiniteTransition = rememberInfiniteTransition("listeningAnimation") + + val loopState by infiniteTransition.animateFloat( + initialValue = 0f, + targetValue = Math.PI.toFloat() * 2f, + animationSpec = infiniteRepeatable(tween(durationMillis = 1000, easing = LinearEasing)), + label = "listeningAnimationLoopState" + ) + + val activeFraction by animateFloatAsState( + if (active) { + Math.pow(level.toDouble(), 0.3).toFloat() + } else { + 0f + } + ) + + Canvas(modifier.clearAndSetSemantics { }) { + + val strokeWidthPx = size.width / 12 + + val lineCount = 5 + + for (i in 1..lineCount) { + + val sine = Math.sin(loopState + 0.9 * i) + val fraction = activeFraction * ((sine + 1) / 2).toFloat() + + val x = (size.width / (lineCount + 1)) * i + + val yMax = size.height * 0.25f + val yMin = size.height * 0.5f + + val y = yMin + (yMax - yMin) * fraction + val yEnd = size.height - y + + this.drawLine( + start = Offset(x, y), + end = Offset(x, yEnd), + color = color, + strokeWidth = strokeWidthPx, + cap = StrokeCap.Round + ) + } + } +} diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/BotIndicator.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/BotIndicator.kt new file mode 100644 index 000000000..282c4f812 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/BotIndicator.kt @@ -0,0 +1,122 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.client.small_webrtc_transport.views.VideoView +import ai.pipecat.client.types.Tracks +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import android.view.ViewGroup.LayoutParams +import androidx.compose.animation.AnimatedContent +import androidx.compose.animation.animateColorAsState +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.runtime.Composable +import androidx.compose.runtime.FloatState +import androidx.compose.runtime.State +import androidx.compose.runtime.derivedStateOf +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.shadow +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.StrokeCap +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView + +@Composable +fun BotIndicator( + modifier: Modifier, + isReady: Boolean, + isTalking: State, + audioLevel: FloatState, + tracks: State +) { + Box( + modifier = modifier.padding(15.dp), + contentAlignment = Alignment.Center + ) { + val color by animateColorAsState(if (isTalking.value || !isReady) { + Color.Black + } else { + Colors.botIndicatorBackground + }) + + Box( + Modifier + .aspectRatio(1f) + .fillMaxSize() + .shadow(20.dp, CircleShape) + .border(12.dp, Color.White, CircleShape) + .border(1.dp, Colors.lightGrey, CircleShape) + .clip(CircleShape) + .background(color) + .padding(50.dp), + contentAlignment = Alignment.Center, + ) { + AnimatedContent( + targetState = isReady + ) { isReadyVal -> + if (isReadyVal) { + + val botVideoTrack by remember { derivedStateOf { + tracks.value?.bot?.video + }} + + if (botVideoTrack != null) { + AndroidView( + modifier = Modifier.fillMaxSize(), + factory = { context -> + VideoView(context).apply { + layoutParams = LayoutParams( + LayoutParams.MATCH_PARENT, + LayoutParams.MATCH_PARENT + ) + } + }, + update = { + it.track = botVideoTrack + } + ) + } else { + ListeningAnimation( + modifier = Modifier.fillMaxSize(), + active = isTalking.value, + level = audioLevel.floatValue, + color = Color.White + ) + } + } else { + CircularProgressIndicator( + modifier = Modifier.size(180.dp), + color = Color.White, + strokeWidth = 12.dp, + strokeCap = StrokeCap.Round, + trackColor = color + ) + } + } + } + } +} + +@Composable +@Preview +fun PreviewBotIndicator() { + BotIndicator( + modifier = Modifier, + isReady = false, + isTalking = remember { mutableStateOf(true) }, + audioLevel = remember { mutableFloatStateOf(1.0f) }, + tracks = remember { mutableStateOf(null) } + ) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallFooter.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallFooter.kt new file mode 100644 index 000000000..d9f46c7ab --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallFooter.kt @@ -0,0 +1,89 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.small_webrtc_client.R +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import androidx.annotation.DrawableRes +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.ColumnScope +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Icon +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp + +@Composable +private fun FooterButton( + modifier: Modifier, + onClick: () -> Unit, + @DrawableRes icon: Int, + text: String, + foreground: Color, + background: Color, + border: Color, +) { + val shape = RoundedCornerShape(12.dp) + + Row( + modifier + .border(1.dp, border, shape) + .clip(shape) + .background(background) + .clickable(onClick = onClick) + .padding(vertical = 10.dp, horizontal = 18.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.Center + ) { + Icon( + modifier = Modifier.size(24.dp), + painter = painterResource(icon), + tint = foreground, + contentDescription = null + ) + + Spacer(modifier = Modifier.width(8.dp)) + + Text( + text = text, + fontSize = 14.sp, + fontWeight = FontWeight.W600, + color = foreground + ) + } +} + +@Composable +fun ColumnScope.InCallFooter( + onClickEnd: () -> Unit, +) { + Row(Modifier + .fillMaxWidth(0.5f) + .padding(15.dp) + .align(Alignment.CenterHorizontally) + ) { + FooterButton( + modifier = Modifier.weight(1f), + onClick = onClickEnd, + icon = R.drawable.phone_hangup, + text = "End", + foreground = Color.White, + background = Colors.endButton, + border = Colors.endButton + ) + } +} diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallHeader.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallHeader.kt new file mode 100644 index 000000000..3fa2aa2eb --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallHeader.kt @@ -0,0 +1,49 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.small_webrtc_client.utils.Timestamp +import androidx.compose.animation.AnimatedContent +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut +import androidx.compose.animation.togetherWith +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.constraintlayout.compose.ConstraintLayout + +@Composable +fun InCallHeader( + expiryTime: Timestamp? +) { + ConstraintLayout( + Modifier + .fillMaxWidth() + .padding(vertical = 15.dp) + ) { + val refTimer = createRef() + + AnimatedContent( + modifier = Modifier.constrainAs(refTimer) { + top.linkTo(parent.top) + bottom.linkTo(parent.bottom) + end.linkTo(parent.end) + }, + targetState = expiryTime, + transitionSpec = { fadeIn() togetherWith fadeOut() } + ) { expiryTimeVal -> + if (expiryTimeVal != null) { + Timer(expiryTime = expiryTimeVal, modifier = Modifier) + } + } + } +} + +@Composable +@Preview +fun PreviewInCallHeader() { + InCallHeader( + Timestamp.now() + java.time.Duration.ofMinutes(3) + ) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallLayout.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallLayout.kt new file mode 100644 index 000000000..ef2185bbe --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/InCallLayout.kt @@ -0,0 +1,71 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.small_webrtc_client.VoiceClientManager +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.runtime.Composable +import androidx.compose.runtime.derivedStateOf +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp + +@Composable +fun InCallLayout(voiceClientManager: VoiceClientManager) { + + val localCam by remember { derivedStateOf { voiceClientManager.tracks.value?.local?.video } } + + Column(Modifier.fillMaxSize()) { + + InCallHeader(expiryTime = voiceClientManager.expiryTime.value) + + Box( + modifier = Modifier + .weight(1f) + .fillMaxWidth(), + contentAlignment = Alignment.Center + ) { + Column( + modifier = Modifier.fillMaxWidth(), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(12.dp, Alignment.CenterVertically) + ) { + BotIndicator( + modifier = Modifier, + isReady = voiceClientManager.botReady.value, + isTalking = voiceClientManager.botIsTalking, + audioLevel = voiceClientManager.botAudioLevel, + tracks = voiceClientManager.tracks + ) + + Row( + verticalAlignment = Alignment.CenterVertically + ) { + UserMicButton( + onClick = voiceClientManager::toggleMic, + micEnabled = voiceClientManager.mic.value, + modifier = Modifier, + isTalking = voiceClientManager.userIsTalking, + audioLevel = voiceClientManager.userAudioLevel + ) + + UserCamButton( + onClick = voiceClientManager::toggleCamera, + camEnabled = voiceClientManager.camera.value, + camTrackId = localCam, + modifier = Modifier + ) + } + } + } + + InCallFooter( + onClickEnd = voiceClientManager::stop + ) + } +} diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/PermissionScreen.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/PermissionScreen.kt new file mode 100644 index 000000000..dcd56e5b0 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/PermissionScreen.kt @@ -0,0 +1,99 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import ai.pipecat.small_webrtc_client.ui.theme.TextStyles +import android.Manifest +import android.util.Log +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Button +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.shadow +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.window.Dialog +import com.google.accompanist.permissions.ExperimentalPermissionsApi +import com.google.accompanist.permissions.isGranted +import com.google.accompanist.permissions.rememberPermissionState + +@OptIn(ExperimentalPermissionsApi::class) +@Composable +fun PermissionScreen() { + val cameraPermission = rememberPermissionState(Manifest.permission.CAMERA) + val micPermission = rememberPermissionState(Manifest.permission.RECORD_AUDIO) + + val requestPermissionLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.RequestMultiplePermissions() + ) { isGranted -> + Log.i("MainActivity", "Permissions granted: $isGranted") + } + + if (!cameraPermission.status.isGranted || !micPermission.status.isGranted) { + + Dialog( + onDismissRequest = {}, + ) { + val dialogShape = RoundedCornerShape(16.dp) + + Column( + Modifier + .shadow(6.dp, dialogShape) + .border(2.dp, Colors.logoBorder, dialogShape) + .clip(dialogShape) + .background(Color.White) + .padding(28.dp) + ) { + Text( + text = "Permissions", + fontSize = 24.sp, + fontWeight = FontWeight.W700, + style = TextStyles.base + ) + + Spacer(modifier = Modifier.height(8.dp)) + + Text( + text = "Please grant camera and mic permissions to continue", + fontSize = 18.sp, + fontWeight = FontWeight.W400, + style = TextStyles.base + ) + + Spacer(modifier = Modifier.height(36.dp)) + + Button( + modifier = Modifier.align(Alignment.End), + shape = RoundedCornerShape(12.dp), + onClick = { + requestPermissionLauncher.launch( + arrayOf( + Manifest.permission.CAMERA, + Manifest.permission.RECORD_AUDIO + ) + ) + } + ) { + Text( + text = "Grant permissions", + fontSize = 16.sp, + fontWeight = FontWeight.W700, + style = TextStyles.base + ) + } + } + } + } +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/Timer.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/Timer.kt new file mode 100644 index 000000000..1b5af67c9 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/Timer.kt @@ -0,0 +1,72 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.small_webrtc_client.R +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import ai.pipecat.small_webrtc_client.utils.Timestamp +import ai.pipecat.small_webrtc_client.utils.formatTimer +import ai.pipecat.small_webrtc_client.utils.rtcStateSecs +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.Icon +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import java.time.Duration + +@Composable +fun Timer( + expiryTime: Timestamp, + modifier: Modifier, +) { + val now by rtcStateSecs() + + val shape = RoundedCornerShape( + topStart = 12.dp, + bottomStart = 12.dp, + ) + + Row( + modifier = modifier + .widthIn(min = 100.dp) + .clip(shape) + .background(Colors.lightGrey) + .padding(top = 12.dp, bottom = 12.dp, start = 12.dp, end = 16.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Icon( + painter = painterResource(id = R.drawable.timer_outline), + contentDescription = null, + modifier = Modifier.size(20.dp), + tint = Colors.expiryTimerForeground + ) + + Spacer(Modifier.width(8.dp)) + + Text( + text = formatTimer(duration = expiryTime - now), + fontSize = 16.sp, + fontWeight = FontWeight.W600, + color = Colors.expiryTimerForeground + ) + } +} + +@Composable +@Preview +fun PreviewExpiryTimer() { + Timer(Timestamp.now() + Duration.ofMinutes(5), Modifier) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/UserCamButton.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/UserCamButton.kt new file mode 100644 index 000000000..4a3048a1a --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/UserCamButton.kt @@ -0,0 +1,110 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.client.small_webrtc_transport.views.VideoView +import ai.pipecat.client.types.MediaTrackId +import ai.pipecat.small_webrtc_client.R +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import androidx.compose.animation.animateColorAsState +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.Icon +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.shadow +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView + +@Composable +fun UserCamButton( + onClick: () -> Unit, + camEnabled: Boolean, + camTrackId: MediaTrackId?, + modifier: Modifier, +) { + Box( + modifier = modifier.padding(15.dp).size(96.dp), + contentAlignment = Alignment.Center + ) { + val color by animateColorAsState( + if (camEnabled) { + Colors.unmutedMicBackground + } else { + Colors.mutedMicBackground + } + ) + + Box( + Modifier + .fillMaxSize() + .shadow(3.dp, CircleShape) + .border(6.dp, Color.White, CircleShape) + .border(1.dp, Colors.lightGrey, CircleShape) + .clip(CircleShape) + .background(color) + .clickable(onClick = onClick), + contentAlignment = Alignment.Center, + ) { + if (camTrackId != null) { + AndroidView( + factory = { context -> + VideoView(context) + }, + update = { view -> + view.track = camTrackId + } + ) + } else { + Icon( + modifier = Modifier.size(30.dp), + painter = painterResource( + if (camEnabled) { + R.drawable.video + } else { + R.drawable.video_off + } + ), + tint = Color.White, + contentDescription = if (camEnabled) { + "Disable camera" + } else { + "Enable camera" + }, + ) + } + } + } +} + +@Composable +@Preview +fun PreviewUserCamButton() { + UserCamButton( + onClick = {}, + camTrackId = null, + camEnabled = true, + modifier = Modifier, + ) +} + +@Composable +@Preview +fun PreviewUserCamButtonMuted() { + UserCamButton( + onClick = {}, + camTrackId = null, + camEnabled = false, + modifier = Modifier, + ) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/UserMicButton.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/UserMicButton.kt new file mode 100644 index 000000000..9e9040e37 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/UserMicButton.kt @@ -0,0 +1,114 @@ +package ai.pipecat.small_webrtc_client.ui + +import ai.pipecat.small_webrtc_client.R +import ai.pipecat.small_webrtc_client.ui.theme.Colors +import androidx.compose.animation.animateColorAsState +import androidx.compose.animation.core.animateDpAsState +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.Icon +import androidx.compose.runtime.Composable +import androidx.compose.runtime.FloatState +import androidx.compose.runtime.State +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.shadow +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp + +@Composable +fun UserMicButton( + onClick: () -> Unit, + micEnabled: Boolean, + modifier: Modifier, + isTalking: State, + audioLevel: FloatState, +) { + Box( + modifier = modifier.padding(15.dp), + contentAlignment = Alignment.Center + ) { + val borderThickness by animateDpAsState( + if (isTalking.value) { + (24.dp * Math.pow(audioLevel.floatValue.toDouble(), 0.3).toFloat()) + 3.dp + } else { + 6.dp + } + ) + + val color by animateColorAsState( + if (!micEnabled) { + Colors.mutedMicBackground + } else if (isTalking.value) { + Color.Black + } else { + Colors.unmutedMicBackground + } + ) + + Box( + Modifier + .shadow(3.dp, CircleShape) + .border(borderThickness, Color.White, CircleShape) + .border(1.dp, Colors.lightGrey, CircleShape) + .clip(CircleShape) + .background(color) + .clickable(onClick = onClick) + .padding(36.dp), + contentAlignment = Alignment.Center, + ) { + Icon( + modifier = Modifier.size(48.dp), + painter = painterResource( + if (micEnabled) { + R.drawable.microphone + } else { + R.drawable.microphone_off + } + ), + tint = Color.White, + contentDescription = if (micEnabled) { + "Mute microphone" + } else { + "Unmute microphone" + }, + ) + } + } +} + +@Composable +@Preview +fun PreviewUserMicButton() { + UserMicButton( + onClick = {}, + micEnabled = true, + modifier = Modifier, + isTalking = remember { mutableStateOf(false) }, + audioLevel = remember { mutableFloatStateOf(1.0f) } + ) +} + +@Composable +@Preview +fun PreviewUserMicButtonMuted() { + UserMicButton( + onClick = {}, + micEnabled = false, + modifier = Modifier, + isTalking = remember { mutableStateOf(false) }, + audioLevel = remember { mutableFloatStateOf(1.0f) } + ) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Color.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Color.kt new file mode 100644 index 000000000..7a55d707a --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Color.kt @@ -0,0 +1,22 @@ +package ai.pipecat.small_webrtc_client.ui.theme + +import androidx.compose.ui.graphics.Color + +object Colors { + val buttonNormal = Color(0xFF374151) + val buttonWarning = Color(0xFFE53935) + val buttonSection = Color(0xFFDFF1FF) + + val activityBackground = Color(0xFFF9FAFB) + val mainSurfaceBackground = Color.White + + val lightGrey = Color(0x7FE5E7EB) + val expiryTimerForeground = Color.Black + val logoBorder = Color(0xFFE2E8F0) + val endButton = Color(0xFF0F172A) + val textFieldBorder = Color(0xFFDFE6EF) + + val botIndicatorBackground = Color(0xFF374151) + val mutedMicBackground = Color(0xFFF04A4A) + val unmutedMicBackground = Color(0xFF616978) +} \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Theme.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Theme.kt new file mode 100644 index 000000000..9f5307286 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Theme.kt @@ -0,0 +1,36 @@ +package ai.pipecat.small_webrtc_client.ui.theme + +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.TextFieldDefaults +import androidx.compose.material3.lightColorScheme +import androidx.compose.runtime.Composable +import androidx.compose.ui.graphics.Color + +private val LightColorScheme = lightColorScheme( + primary = Colors.buttonNormal, + secondary = Colors.buttonWarning, + background = Colors.activityBackground, + surface = Colors.mainSurfaceBackground +) + +@Composable +fun RTVIClientTheme( + content: @Composable () -> Unit +) { + val colorScheme = LightColorScheme + + MaterialTheme( + colorScheme = colorScheme, + typography = Typography, + content = content + ) +} + +@Composable +fun textFieldColors() = TextFieldDefaults.colors().copy( + unfocusedContainerColor = Colors.activityBackground, + focusedContainerColor = Colors.activityBackground, + focusedIndicatorColor = Color.Transparent, + disabledIndicatorColor = Color.Transparent, + unfocusedIndicatorColor = Color.Transparent, +) \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Type.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Type.kt new file mode 100644 index 000000000..fb20151a1 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/ui/theme/Type.kt @@ -0,0 +1,40 @@ +package ai.pipecat.small_webrtc_client.ui.theme + +import ai.pipecat.small_webrtc_client.R +import androidx.compose.material3.Typography +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.Font +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.sp + +object TextStyles { + val base = TextStyle(fontFamily = FontFamily(Font(R.font.inter))) +} + +// Set of Material typography styles to start with +val Typography = Typography( + bodyLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 16.sp, + lineHeight = 24.sp, + letterSpacing = 0.5.sp + ) + /* Other default text styles to override + titleLarge = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Normal, + fontSize = 22.sp, + lineHeight = 28.sp, + letterSpacing = 0.sp + ), + labelSmall = TextStyle( + fontFamily = FontFamily.Default, + fontWeight = FontWeight.Medium, + fontSize = 11.sp, + lineHeight = 16.sp, + letterSpacing = 0.5.sp + ) + */ +) \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/utils/RealTimeClock.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/utils/RealTimeClock.kt new file mode 100644 index 000000000..8d951bfef --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/utils/RealTimeClock.kt @@ -0,0 +1,21 @@ +package ai.pipecat.small_webrtc_client.utils + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.flow + +private val rtcFlowSecs = flow { + while(true) { + val now = Timestamp.now().toEpochMilli() + + val rounded = ((now + 500) / 1000) * 1000 + emit(Timestamp.ofEpochMilli(rounded)) + + val target = rounded + 1000 + delay(target - now) + } +} + +@Composable +fun rtcStateSecs() = rtcFlowSecs.collectAsState(initial = Timestamp.now()) \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/utils/TimeUtils.kt b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/utils/TimeUtils.kt new file mode 100644 index 000000000..ceba7bebf --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/java/ai/pipecat/small_webrtc_client/utils/TimeUtils.kt @@ -0,0 +1,64 @@ +package ai.pipecat.small_webrtc_client.utils + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.Immutable +import java.time.Duration +import java.time.Instant +import java.time.format.DateTimeFormatter +import java.util.Date + +// Wrapper for Compose stability +@Immutable +@JvmInline +value class Timestamp( + val value: Instant +) : Comparable { + val isInPast: Boolean + get() = value < Instant.now() + + val isInFuture: Boolean + get() = value > Instant.now() + + fun toEpochMilli() = value.toEpochMilli() + + operator fun plus(duration: Duration) = Timestamp(value + duration) + + operator fun minus(duration: Duration) = Timestamp(value - duration) + + operator fun minus(rhs: Timestamp) = Duration.between(rhs.value, value) + + override operator fun compareTo(other: Timestamp) = value.compareTo(other.value) + + fun toISOString(): String = DateTimeFormatter.ISO_INSTANT.format(value) + + override fun toString() = toISOString() + + companion object { + fun now() = Timestamp(Instant.now()) + + fun ofEpochMilli(value: Long) = Timestamp(Instant.ofEpochMilli(value)) + + fun ofEpochSecs(value: Long) = ofEpochMilli(value * 1000) + + fun parse(value: CharSequence) = Timestamp(Instant.parse(value)) + + fun from(date: Date) = Timestamp(date.toInstant()) + } +} + +@Composable +fun formatTimer(duration: Duration): String { + + if (duration.seconds < 0) { + return "0s" + } + + val mins = duration.seconds / 60 + val secs = duration.seconds % 60 + + return if (mins == 0L) { + "${secs}s" + } else { + "${mins}m ${secs}s" + } +} diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/ic_launcher_background.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..07d5da9cb --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/ic_launcher_foreground.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/ic_launcher_foreground.xml new file mode 100644 index 000000000..2b068d114 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/microphone.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/microphone.xml new file mode 100644 index 000000000..ed375ff21 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/microphone.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/microphone_off.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/microphone_off.xml new file mode 100644 index 000000000..bbeb43c59 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/microphone_off.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/phone_hangup.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/phone_hangup.xml new file mode 100644 index 000000000..a637344c3 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/phone_hangup.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/timer_outline.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/timer_outline.xml new file mode 100644 index 000000000..ef0ca41fe --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/timer_outline.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/video.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/video.xml new file mode 100644 index 000000000..bce7a9ed5 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/video.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/video_off.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/video_off.xml new file mode 100644 index 000000000..c06bc38a3 --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/drawable/video_off.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/font/inter.ttf b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/font/inter.ttf new file mode 100644 index 000000000..e31b51e3e Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/font/inter.ttf differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 000000000..6f3b755bf --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 000000000..6f3b755bf --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-hdpi/ic_launcher.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-hdpi/ic_launcher.webp new file mode 100644 index 000000000..c209e78ec Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-hdpi/ic_launcher.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-hdpi/ic_launcher_round.webp new file mode 100644 index 000000000..b2dfe3d1b Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-hdpi/ic_launcher_round.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-mdpi/ic_launcher.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-mdpi/ic_launcher.webp new file mode 100644 index 000000000..4f0f1d64e Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-mdpi/ic_launcher.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-mdpi/ic_launcher_round.webp new file mode 100644 index 000000000..62b611da0 Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-mdpi/ic_launcher_round.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xhdpi/ic_launcher.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xhdpi/ic_launcher.webp new file mode 100644 index 000000000..948a3070f Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xhdpi/ic_launcher.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..1b9a6956b Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xhdpi/ic_launcher_round.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxhdpi/ic_launcher.webp new file mode 100644 index 000000000..28d4b77f9 Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxhdpi/ic_launcher.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..9287f5083 Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxxhdpi/ic_launcher.webp new file mode 100644 index 000000000..aa7d6427e Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxxhdpi/ic_launcher.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp new file mode 100644 index 000000000..9126ae37c Binary files /dev/null and b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp differ diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/values/strings.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/values/strings.xml new file mode 100644 index 000000000..41e627ddf --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + Pipecat Small WebRTC Client + diff --git a/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/values/themes.xml b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/values/themes.xml new file mode 100644 index 000000000..3675aa6ba --- /dev/null +++ b/examples/p2p-webrtc/video-transform/client/android/small-webrtc-client/src/main/res/values/themes.xml @@ -0,0 +1,5 @@ + + + +