From bc98c2e36c897dc0eafaedbeec5600236ec02753 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Wed, 29 Jan 2025 19:12:15 +0000 Subject: [PATCH 001/427] added sqlite storage example --- example.db | Bin 0 -> 8192 bytes ...d-transcription-processor-gemini-sqlite.py | 200 ++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 example.db create mode 100644 examples/foundational/28d-transcription-processor-gemini-sqlite.py diff --git a/example.db b/example.db new file mode 100644 index 0000000000000000000000000000000000000000..8e51d4adfc7589f478c2f61785ade9ce95f72c25 GIT binary patch literal 8192 zcmeI1OK;Oa5XYU?r9ATB5^RP~v~@PIiQ^64b*uDN zi3@xK#24bqiBH0vaUP1Okr3hryNcuNW_R{KznMLZ_UL8X7ZkcI8V~`da$Ct{l&1iS zqU7;ez-QJn_(Pp_@SXI3*Yb*X@h*q6GKKdFeq@sbBmqf45|9KW0ZBj-kOU+FNk9^i z1pZqB7n$WdD=V4HT|rI*I-s1B9_6$5!q%bPbZl^%&)PQ3Mju2hpy1fAoKlaVK?XzE#$`@x--c~_r+M6VVC(qs&^~b9B!fpzbN>ZAOltF7wxATg6h0R2=D+4Y z`#t+9dw_q*CJ9Ial7J*22}lBxfFvLZNCN*RfyX(uWbP8qeJmkDvByF> zf{>ok2>6is0oM%OSYOl2Yevm6^oFtCunf(pmmleRL)TkbwRpT6a~f@t5JoHpL4yE` z7Qn8LRZ4`Vi68c$OI-0JZA#b7dUayz^)0nDm@7H)@OrYt#x`AoHVFjeVg!8>_DIAS z_J=HXJ^U6#u(V9m(saw3*xtOU7B_w^6W+2uO)E1EO|R>d)!iFQrOU;+G{>XB;}F$R z4w28%S?IB#LqqUlx(Y*ro+nQ8zK=8Ts&&jWq~}*vS`Dj)W0lE1jq7T$`jg7@D7mXm zJf(xH8gXq!Tdz&m@@3SxGS}Dw7b74d67qncjR3L5NtKC#>A=Kp$M1GAKv+oj{jkG< zb-|4&5&jvSm6&P4W*s-JO$2XTQ;YSn;BmBQfGDOR>xb+-t=Oz;RkJ=>ytAa1?786Q zu3$q4%3LYhqNTs0FNTM?n(Ng~+1<{nQ=`%^eB0?<8HJx}P^e4P8fV)0& z!ZX5=Y0L#+$_+T2QkS4+REwu(K>8HoAtb>b2@ew>P0uhj%bJF4bK%dD(rOrGt*TGi bS2Akp-dyS)9y^xkh-iA(az(3J(-ZpxU)f&k literal 0 HcmV?d00001 diff --git a/examples/foundational/28d-transcription-processor-gemini-sqlite.py b/examples/foundational/28d-transcription-processor-gemini-sqlite.py new file mode 100644 index 000000000..f25c3cbe1 --- /dev/null +++ b/examples/foundational/28d-transcription-processor-gemini-sqlite.py @@ -0,0 +1,200 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sqlite3 +import sys +from typing import List, Optional + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import ( + CancelFrame, + TranscriptionMessage, + TranscriptionUpdateFrame, +) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.google import GoogleLLMService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +con = sqlite3.connect("example.db") +db = con.cursor() + +table = db.execute("SELECT name FROM sqlite_master WHERE name='messages'") +if not (table.fetchone()): + db.execute( + "CREATE TABLE messages(role TEXT, content TEXT, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP )" + ) + + +class TranscriptHandler: + """Handles real-time transcript processing and output. + + Maintains a list of conversation messages and outputs them either to a log + or to a file as they are received. Each message includes its timestamp and role. + + Attributes: + messages: List of all processed transcript messages + output_file: Optional path to file where transcript is saved. If None, outputs to log only. + """ + + def __init__(self, output_file: Optional[str] = None): + """Initialize handler with optional file output. + + Args: + output_file: Path to output file. If None, outputs to log only. + """ + self.messages: List[TranscriptionMessage] = [] + self.output_file: Optional[str] = output_file + logger.debug( + f"TranscriptHandler initialized {'with output_file=' + output_file if output_file else 'with log output only'}" + ) + + async def save_message(self, message: TranscriptionMessage): + """Save a single transcript message. + + Outputs the message to the log and optionally to a file. + + Args: + message: The message to save + """ + timestamp_f = f"[{message.timestamp}] " if message.timestamp else "" + line = f"{timestamp_f}{message.role}: {message.content}" + + # Always log the message + logger.info(f"Transcript: {line}") + + # Write to database + db.execute( + "INSERT INTO messages VALUES (?, ?, ?)", + (message.role, message.content, message.timestamp), + ) + con.commit() + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ + logger.debug(f"Received transcript update with {len(frame.messages)} new messages") + + for msg in frame.messages: + self.messages.append(msg) + await self.save_message(msg) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + None, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = GoogleLLMService( + model="models/gemini-2.0-flash-exp", + # model="gemini-exp-1114", + api_key=os.getenv("GOOGLE_API_KEY"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", + }, + {"role": "user", "content": "Say hello."}, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript = TranscriptProcessor() + transcript_handler = TranscriptHandler() # Output to log only + # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + transcript.assistant(), # Assistant transcripts + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + # Stop the pipeline immediately when the participant leaves + await task.queue_frame(CancelFrame()) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 95c8346cb5a44f8e076cc99d1efd9833515ca920 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 29 Jan 2025 19:00:42 -0300 Subject: [PATCH 002/427] Starting to create a react native client for the bot ready example. --- .gitignore | 15 + .../client/react-native/.nvmrc | 1 + .../client/react-native/README.md | 56 + .../client/react-native/app.json | 75 + .../react-native/assets/adaptive-icon.png | Bin 0 -> 17547 bytes .../client/react-native/assets/favicon.png | Bin 0 -> 1466 bytes .../client/react-native/assets/icon.png | Bin 0 -> 22380 bytes .../client/react-native/assets/splash.png | Bin 0 -> 47346 bytes .../client/react-native/babel.config.js | 6 + .../client/react-native/index.js | 7 + .../client/react-native/metro.config.js | 4 + .../client/react-native/package-lock.json | 10970 ++++++++++++++++ .../client/react-native/package.json | 30 + .../client/react-native/src/App.js | 153 + 14 files changed, 11317 insertions(+) create mode 100644 examples/bot-ready-signalling/client/react-native/.nvmrc create mode 100644 examples/bot-ready-signalling/client/react-native/README.md create mode 100644 examples/bot-ready-signalling/client/react-native/app.json create mode 100644 examples/bot-ready-signalling/client/react-native/assets/adaptive-icon.png create mode 100644 examples/bot-ready-signalling/client/react-native/assets/favicon.png create mode 100644 examples/bot-ready-signalling/client/react-native/assets/icon.png create mode 100644 examples/bot-ready-signalling/client/react-native/assets/splash.png create mode 100644 examples/bot-ready-signalling/client/react-native/babel.config.js create mode 100644 examples/bot-ready-signalling/client/react-native/index.js create mode 100644 examples/bot-ready-signalling/client/react-native/metro.config.js create mode 100644 examples/bot-ready-signalling/client/react-native/package-lock.json create mode 100644 examples/bot-ready-signalling/client/react-native/package.json create mode 100644 examples/bot-ready-signalling/client/react-native/src/App.js diff --git a/.gitignore b/.gitignore index 50944a3a8..1a11c0f2a 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,21 @@ fly.toml # Example files pipecat/examples/twilio-chatbot/templates/streams.xml +pipecat/examples/bot-ready-signalling/client/react-native/node_modules/ +pipecat/examples/bot-ready-signalling/client/react-native/.expo/ +pipecat/examples/bot-ready-signalling/client/react-native/dist/ +pipecat/examples/bot-ready-signalling/client/react-native/npm-debug.* +pipecat/examples/bot-ready-signalling/client/react-native/*.jks +pipecat/examples/bot-ready-signalling/client/react-native/*.p8 +pipecat/examples/bot-ready-signalling/client/react-native/*.p12 +pipecat/examples/bot-ready-signalling/client/react-native/*.key +pipecat/examples/bot-ready-signalling/client/react-native/*.mobileprovision +pipecat/examples/bot-ready-signalling/client/react-native/*.orig.* +pipecat/examples/bot-ready-signalling/client/react-native/web-build/ + +# macOS +.DS_Store + # Documentation docs/api/_build/ diff --git a/examples/bot-ready-signalling/client/react-native/.nvmrc b/examples/bot-ready-signalling/client/react-native/.nvmrc new file mode 100644 index 000000000..4a58985bb --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/.nvmrc @@ -0,0 +1 @@ +18.18 diff --git a/examples/bot-ready-signalling/client/react-native/README.md b/examples/bot-ready-signalling/client/react-native/README.md new file mode 100644 index 000000000..a1c0d1422 --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/README.md @@ -0,0 +1,56 @@ +# React Native Implementation + +Basic implementation using the [Pipecat React Native SDK](https://docs.pipecat.ai/client/react-native/introduction). + +## Usage + +### Expo requirements + +This project cannot be used with an [Expo Go](https://docs.expo.dev/workflow/expo-go/) app because [it requires custom native code](https://docs.expo.io/workflow/customizing/). + +When a project requires custom native code or a config plugin, we need to transition from using [Expo Go](https://docs.expo.dev/workflow/expo-go/) +to a [development build](https://docs.expo.dev/development/introduction/). + +More details about the custom native code used by this demo can be found in [rn-daily-js-expo-config-plugin](https://github.com/daily-co/rn-daily-js-expo-config-plugin). + +### Building remotely + +If you do not have experience with Xcode and Android Studio builds or do not have them installed locally on your computer, you will need to follow [this guide from Expo to use EAS Build](https://docs.expo.dev/development/create-development-builds/#create-and-install-eas-build). + +### Building locally + +You will need to have installed locally on your computer: +- [Xcode](https://developer.apple.com/xcode/) to build for iOS; +- [Android Studio](https://developer.android.com/studio) to build for Android; + +#### Install the demo dependencies + +```bash +# Use the version of node specified in .nvmrc +nvm i + +# Install dependencies +npm i + +# Before a native app can be compiled, the native source code must be generated. +npx expo prebuild +``` + +#### Running on Android + +After plugging in an Android device [configured for debugging](https://developer.android.com/studio/debug/dev-options), run the following command: + +``` +npm run android +``` + +#### Running on iOS + +Run the following command: + +``` +npm run ios +``` + +#### Connect to the server +Use the http://localhost:5173 in your app. \ No newline at end of file diff --git a/examples/bot-ready-signalling/client/react-native/app.json b/examples/bot-ready-signalling/client/react-native/app.json new file mode 100644 index 000000000..fb9394e96 --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/app.json @@ -0,0 +1,75 @@ +{ + "expo": { + "name": "bot-ready-rn", + "slug": "bot-ready-rn", + "version": "1.0.0", + "orientation": "portrait", + "icon": "./assets/icon.png", + "userInterfaceStyle": "light", + "splash": { + "image": "./assets/splash.png", + "resizeMode": "contain", + "backgroundColor": "#ffffff" + }, + "updates": { + "fallbackToCacheTimeout": 0 + }, + "assetBundlePatterns": [ + "**/*" + ], + "ios": { + "supportsTablet": true, + "bitcode": false, + "bundleIdentifier": "co.daily.expo.BotReady", + "infoPlist": { + "UIBackgroundModes": [ + "voip" + ] + }, + "appleTeamId": "EEBGKV9N3N" + }, + "android": { + "adaptiveIcon": { + "foregroundImage": "./assets/adaptive-icon.png", + "backgroundColor": "#FFFFFF" + }, + "package": "co.daily.expo.BotReady", + "permissions": [ + "android.permission.ACCESS_NETWORK_STATE", + "android.permission.BLUETOOTH", + "android.permission.CAMERA", + "android.permission.INTERNET", + "android.permission.MODIFY_AUDIO_SETTINGS", + "android.permission.RECORD_AUDIO", + "android.permission.SYSTEM_ALERT_WINDOW", + "android.permission.WAKE_LOCK", + "android.permission.FOREGROUND_SERVICE", + "android.permission.FOREGROUND_SERVICE_CAMERA", + "android.permission.FOREGROUND_SERVICE_MICROPHONE", + "android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION", + "android.permission.POST_NOTIFICATIONS" + ] + }, + "web": { + "favicon": "./assets/favicon.png" + }, + "plugins": [ + "@config-plugins/react-native-webrtc", + "@daily-co/config-plugin-rn-daily-js", + [ + "expo-build-properties", + { + "android": { + "minSdkVersion": 24, + "compileSdkVersion": 35, + "targetSdkVersion": 34, + "buildToolsVersion": "35.0.0" + }, + "ios": { + "deploymentTarget": "15.1" + } + } + ] + ] + } +} diff --git a/examples/bot-ready-signalling/client/react-native/assets/adaptive-icon.png b/examples/bot-ready-signalling/client/react-native/assets/adaptive-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..03d6f6b6c6727954aec1d8206222769afd178d8d GIT binary patch literal 17547 zcmdVCc|4Ti*EoFcS?yF*_R&TYQOH(|sBGDq8KR;jni6eN$=oWm(;}%b6=4u1OB+)v zB_hpO3nh}szBBXQ)A#%Q-rw_nzR&Y~e}BB6&-?oL%*=hAbDeXpbDis4=UmHu*424~ ztdxor0La?g*}4M|u%85wz++!_Wz7$(_79;y-?M_2<8zbyZcLtE#X^ zL3MTA-+%1K|9ZqQu|lk*{_p=k%CXN{4CmuV><2~!1O20lm{dc<*Dqh%K7Vd(Zf>oq zsr&S)uA$)zpWj$jh0&@1^r>DTXsWAgZftC+umAFwk(g9L-5UhHwEawUMxdV5=IdKl9436TVl;2HG#c;&s>?qV=bZ<1G1 zGL92vWDII5F@*Q-Rgk(*nG6_q=^VO{)x0`lqq2GV~}@c!>8{Rh%N*#!Md zcK;8gf67wupJn>jNdIgNpZR|v@cIA03H<+(hK<+%dm4_({I~3;yCGk?+3uu{%&A)1 zP|cr?lT925PwRQ?kWkw`F7W*U9t!16S{OM(7PR?fkti+?J% z7t5SDGUlQrKxkX1{4X56^_wp&@p8D-UXyDn@OD!Neu1W6OE-Vp{U<+)W!P+q)zBy! z&z(NXdS(=_xBLY;#F~pon__oo^`e~z#+CbFrzoXRPOG}Nty51XiyX4#FXgyB7C9~+ zJiO_tZs0udqi(V&y>k5{-ZTz-4E1}^yLQcB{usz{%pqgzyG_r0V|yEqf`yyE$R)>* z+xu$G;G<(8ht7;~bBj=7#?I_I?L-p;lKU*@(E{93EbN=5lI zX1!nDlH@P$yx*N#<(=LojPrW6v$gn-{GG3wk1pnq240wq5w>zCpFLjjwyA1~#p9s< zV0B3aDPIliFkyvKZ0Pr2ab|n2-P{-d_~EU+tk(nym16NQ;7R?l}n==EP3XY7;&ok_M4wThw?=Qb2&IL0r zAa_W>q=IjB4!et=pWgJ$Km!5ZBoQtIu~QNcr*ea<2{!itWk|z~7Ga6;9*2=I4YnbG zXDOh~y{+b6-rN^!E?Uh7sMCeE(5b1)Y(vJ0(V|%Z+1|iAGa9U(W5Rfp-YkJ(==~F8 z4dcXe@<^=?_*UUyUlDslpO&B{T2&hdymLe-{x%w1HDxa-ER)DU(0C~@xT99v@;sM5 zGC{%ts)QA+J6*tjnmJk)fQ!Nba|zIrKJO8|%N$KG2&Z6-?Es7|UyjD6boZ~$L!fQ} z_!fV(nQ7VdVwNoANg?ob{)7Fg<`+;01YGn1eNfb_nJKrB;sLya(vT;Nm|DnCjoyTV zWG0|g2d3~Oy-D$e|w|reqyJ}4Ynk#J`ZSh$+7UESh|JJ z%E?JpXj^*PmAp-4rX?`Bh%1?y4R$^fg7A^LDl2zEqz@KfoRz*)d-&3ME4z3RecXF( z&VAj}EL`d22JTP~{^a_c`^!!rO9~#1rN``Vtu@^d~$&2DJ0 zI`*LVx=i7T@zn{|Ae&_LKU;BmoKcvu!U;XNLm?- z`9$AWwdIi*vT?H2j1QmM_$p!dZjaBkMBW#Pu*SPs+x=rj-rsZX*Uwl!jw##am$Sla z={ixqgTqq43kA2TwznpSACvKQ?_e*>7MqBphDh`@kC8vNX-atL-E9HOfm@-rwJ=!w zDy4O~H&p86Sz}lqM%YCejH?s7llrpn7o|E(7AL-qjJvf?n&W*AizC+tjmNU*K603| zOZctr603w>uzzZk8S@TPdM+BTjUhn)Om0Fx>)e6c&g69aMU3{3>0#cH)>-E7Fb4xL zE|i~fXJ!s`NKCviTy%@7TtBJv0o|VUVl}1~Xq$>`E*)f6MK}#<-u9w0g2uL2uH;F~ z;~5|aFmT)-w%2QFu6?3Cj|DS}7BVo&fGYwubm2pNG zfKnrxw>zt-xwPQgF7D3eTN17Zn8d$T!bPGbdqzU1VlKHm7aaN4sY`3%{(~59Mt>Kh zH~8zY;jeVo$CVOoIp;9%E7sP$0*Cqou8a-Ums!E502h{ZMVy|XH-E90W)USFDzSjp)b$rmB9eaA1>h zZ<`M7V|PcDSP0lL>GO^&xuaLpig7~Y3;E3E-f@>AOliK)rS6N?W!Ewu&$OpE$!k$O zaLmm(Mc^4B;87?dW}9o?nNiMKp`gG*vUHILV$rTk(~{yC4BJ4FL}qv4PKJ(FmZoN@ zf|$>xsToZq>tp$D45U%kZ{Yf>yDxT|1U6z|=Gd72{_2tfK_NV!wi$5$YHK zit#+!0%p>@;*o?ynW3w3DzmcaYj7$Ugi}A$>gcH+HY0MFwdtaa5#@JRdVzm>uSw|l3VvL-Xln~r6!H^zKLy zMW|W{Z090XJupzJv}xo0(X~6Sw%SEL44A8V}VDElH!d z>*G!)H*=2~OVBZp!LEl5RY8LHeZr1S@jirblOln1(L=0JXmj(B&(FeR9WkOlWteu+ z!X75~kC)10m8Pej+-&6T_*l|x`G(%!Dw)BrWM*0Hk-%zF{{H>1(kb7 z4)}@b!KeU2)@MzR_YE%3o4g*xJG?EcRK5kXSbz@E+m@qx9_R7a^9cb7fKr1-sL|Hx0;y;miqVzfm7z;p-)CAP(ZiJ zP1Y%M-_+4D9~cib;p}(HG??Wn1vnmg@v#rr&i#~r$Wwqk85%Axbzh6#3IZUMvhhU@ zBb%DLm(GHgt(!WkiH2z!-&2b)YU6_KW!G-9J9i_z)(0`howk{W+m9T>>TqI6;Kuqb z|3voT4@T;Gn&UNdx+g&bb`SsFzPp(G$EED)YUct=@1m(ZU8{F5ge^GUuf~;Y&sv=* ziv8_;Y3c?0@zpo_DU#(lUdOB1Khv)>OY90tw#Z*6m~Q(nw1v2@21||3i}LH~zg2&a zRK~&B2OrDXKnKp}GXpMm%ZJ^HTRWKRcroCL_|6xZoD-#3qpC`X$a{Y<{(DFR?P~WM zQQ@VwTnF!hBK3w(sjs%RMRvk>BDzO+c~_XeFvaf`)o;ylGq9&7%V_)#L?|%aFD2pF zoisAcCNS58Cjcq8wDKX22JiM0;_|1*TYpvgziQ-IT%qgY2JJ9>qg5V>?yDuVJdArVp_*M5f^p;!XL+`CZXIz z&rC=}cLo@_Z*DU{LE$PR$sXxXn1@wOg5yi(z4XV?=*+KPm8XtGOiM#Ju5zxQZ<-j- zWUgqFd9cs}49w<*_`4A`Bw*I&f|oI<xl5> zVFZ2Nj~iRjUXAa>(fXNh^l0ZvZCj}@-|mHBAfc{{giu1V*5YbZoWSQk4n50vJhk5U z(%~pjC}zxiC;H4m8q}m=m3wS(8#hGA^wk5xKEb6D;tiW=`Sq=s+BIa}|4PYKfRlyP zYrl_^WKrE&P?=hyvPG`OPl^JBy^IJP$fDS=kV$jySp_Zfo)VztEnxJtA5%{TMQ}>f z7)(c`oDc%)o70pZfU5mSJqy0NhtDg`JF1d_Q7)jK{(ULJE=`#LdopdJKEt#k4J7#7 zHOIUCTFM<46TmOC`1i`8O@L5bv&=_jYTiD>IYC~+Q+)RoebW3r;^Iehpng2|yd;de zJ5KgeWK#i0JHt%Vh8L}%06l3tR5^>%5BOp2+sz2Y<-MfS!PB1Q+#>y2%&eMwBd@3j z=bIn_S@vrd%|mYBFpKmmI7L9WK=$|y5pIxl8kb@Q#9?S5lzDIp^6t|E@mn5>h0@LX zK5t(Gk#`NN?T}O)dwhpjGXabPxSDo34&-s^4bs!=oG}g5WIH&+s$#qjWa}Qzc;|uF zjmT93Tt3wV$xyw$Q~~O)n_sRbDAq6)VeKQ<$BnQn+=~XDTd9hO;g~ILIS_U-iVNE> zP8T*%AbYt$AGdO!n3*5rLc@Me=!J(I1z=v0T1R`o5m|{)C|RTYTVNuTL!n>uc);VY zt1hK}GgHuUkg;EwmlnFSqOS2-CBtR8u0_ij`@xIE`~XqG)j!s3H>CR&{$1(jD0v2v z6LK_DWF351Q^EywA@pKn@mWuJI!C z9o+gLqgrVDv1G?Gbl2z+c>ZjT!aEb(B{_7@enEhJW20r8cE*WQ<|85nd`diS#GH21^>;;XS{9)Aw*KEZw0W{OW#6hHPovJN zjoem5<5LbVSqE%7SLA7TIMy;;N%3TEhr=W&^2TFRJUWPve86@7iEsH^$p;U=q`H!)9EwB9#Y=V-g&lcJVX;dw}$ zvE?Goc@I7bt>>~=%SafT(`sK|(8U+Z0hvZ`rKHT|)(H2{XAd;2_a?X5K#5EjWMF~@ z=Dx$iW|qOsStpJq`5mS6o{?&hDkjLH2Omg)(og-e>X->WQU8V^@vGI{=FC9ES5e{A zptfOTbCVipp$%$%4Z3!I{EpC`i1AM}X7`m)lAs2KXqp( zxS7r0jzS+aeOwl~0r4WDc$(~!?+=hpubxt&+pyJ|MT1$(WA>^N&d@0YIPh1RcUwrD zVClN;B7^C`fzofKtfG7=oGn!WXK-ng6(+_N?txi@qgah^A0zsqx??_U68mb73%o9x8I-BGbW3+qPbqD(RL3!8Is3{2QUr@pfV7s zyDvbLe)5av)u%m{PWT>milh>L)XBGX5hkYLbwus;=c-=K&e*&CVK0|4H9Is98XSS3 z?u#8@a~?u~@IWW~;+ve_(hA~~Fpp2>DDWKD-8{zTU8$j91k|r1fqwhasxVvo0@rBl8WY}*oQ9Qli~1-fda^B`uahETKe zW2a_^&5=2w7|N;ZY+Cn99syF%rJm`4_ehNznD=O)C3=B-MC=0}tSBRwzsf*r%ch2U z-|x@x9AkL*xT>L}=7IyUlfB$Wh-7}4GV?|UtBfPb|iP*S;^5@Xl4#xc-reL)N8g-aP-H;@?3A`?b4>#KAW#~2t$Lnf@L(h&flZE%(6UHif)My{j zHKntv_d94HiH`>MIeHL*46n>b$nl0U9XiixT2^=yst zTrW!v9UQnvt-ow8GyWB+Q3N?UjTr zT*VeybJ8~IEqwnvI1Z+8zpGbPQt*i4~_e?dK-4%6+$D>w61II;f zl=$T^9g&Htv*eRMTt2s^XOjYM37Mt}HRpl9vCaGZW`UOf$bn4W{Wlk*_=dx4?P?dG zc#bUGmYTaS^iXdm$hX@@-@0;Cv{8xFn0*_Crfn}XIG@HmE`rk z_0-#^aKI@cL52NhLEZr{LQq5cDvSB8q&3%qGa}t1t3Fhd+_iON`Re{;nlv=n^uo`( zn0&8)ZX$v7H0-r zBJE^dvRs$sS!1MWb2y{NIO<_huhf+KvH2^_pqq@=u{mwQM+P=4apqt>Mv*kd^v%AY z>FL~qxn5Hn>3~%y=6$CX)ZfvZt(a3}f&Gwj8@f*d?{BSvkKx-&1>jTwdR<0H-Q_{gH z(h+qS!JO~g9}y>>(0!#1RKpoU(;A+m|2df6OmoD#K6&xZXSO2=MeK49(A#1>_cSK$ zxNTS+{T1SB0)*+{nsumSHMf!pNG5HuA1`$-Wjg9T(L@gIMhp~B|Dm}cwL*0tGV+qSmExLEP?K_cA<;ea@WI{6 za6THY@lQURt`WtlVfNM*|8R28OSRM_Trp~14J z(Zzsnr9G0C2^O8T-yW7pSMI-|lgV2}v!)DmLWT+$y6?Y4yt8nJC?JpEDGwk0%`nH@ z{@YsI5Fkt(BdW!DT}M*)AT;Xn4EeZ=kmyOWLx}g_BT+b(c&wxKra^43UvaXoE8}*&NOlT4U)?L-3@=;fJx& zaGV?(r4A(EoRO!`4x5sfDGkfqDQ5ug=R+xpr=V3Gl<*vVyB4G9du)3ZA ziDzy}JA7@I6Kg;jB>IgnL+V`q%~d0KG(c5fuxODH9*a=M_KaVXzgA)8zi9;+J+nvo zkNl=-q^o~L;Z>owxJT@rd=E*8^!|~GduhQ|tU+9{BxPfkgdK6)-C#Ai*>ZbxCawR{ zL_C7c;xY(LU=X;;IMRj<#sis39%c`>|Le8OdCnNq)A- z6tK0J+l1)b(M9a<&B&1Z#Jth4%xQbdMk#d&1u)0q$nTKM5UWkt%8|YvW(#deR?fae z%)66!ej@HC_=ybH>NC04N(ylmN6wg;VonG`mD(Cfpl$nH3&z>*>n5|8ZU%gwZbU@T&zVNT;AD+*xcGGUnD4;S-eHESm;G=N^fJppiQ z*=j&7*2!U0RR2%QeBal1k5oO`4bW&xQ7V?}630?osIEr?H6d6IH03~d02>&$H&_7r z4Q{BAcwa1G-0`{`sLMgg!uey%s7i00r@+$*e80`XVtNz{`P<46o``|bzj$2@uFv^> z^X)jBG`(!J>8ts)&*9%&EHGXD2P($T^zUQQC2>s%`TdVaGA*jC2-(E&iB~C+?J7gs z$dS{OxS0@WXeDA3GkYF}T!d_dyr-kh=)tmt$V(_4leSc@rwBP=3K_|XBlxyP0_2MG zj5%u%`HKkj)byOt-9JNYA@&!xk@|2AMZ~dh`uKr0hP?>y z$Qt7a<%|=UfZJ3eRCIk7!mg|7FF(q`)VExGyLVLq)&(;SKIB48IrO5He9P!iTROJR zs0KTFhltr1o2(X2Nb3lM6bePKV`Cl;#iOxfEz5s$kDuNqz_n%XHd?BrBYo$RKW1*c z&9tu#UWeDd_C`?ASQyyaJ{KFv&i;>@n&fW5&Jmb7QYhSbLY>q9OAx+|>n0up zw2^SLO!XASLHCE4Im8)F`X1QNU}mk@ssu*!ViT@5Ep%hB2w0kS0XQbRx8B(|dSEMr zF^e0IZ1$x}$^kaa8ZGi}y=(Rn1V4}l?Tx`s=6Vr7^|9oYiiuHlWJ&7W$}3x}Agpk} zeM0Fa;wuFuzh&67?b5ElegEwyD4ctwO6z|2^Ryh;U^}gvl|f-s>9f9hL_ybM0@xG( zQ1I~tGO7&d2be|<#Cs(_l&dG8)_#H8s7G?8-|1Fi-ZN~Kf$1)`tnZ~?Ea2SPC~w!% zN5N}H_G0#jI!9Cw#D~!7Al;b%PS%DkYv#jUfx;B3nk6lv({hlhK8q$+H zSstPe5?7Eo_xBsM+SKCKh%IedpelOV3!4B6ur$i+c`Cnzb3;0t8j6jpL&VDTLWE9@ z3s=jP1Xh)8C?qKDfqDpf<<%O4BFG&7xVNe1sCq?yITF_X-6D6zE_o& zhBM=Z$ijRnhk*=f4 zCuo^l{2f@<$|23>um~C!xJQm%KW|oB|Bt#l3?A6&O@H=dslsfy@L^pVDV3D5x#PUp ze0|@LGO(FTb6f#UI7f!({D2mvw+ylGbk*;XB~C2dDKd3ufIC$IZ0%Uq%L`5wuGm}3 z#e?0n)bjvHRXGhAbPC)+GIh!(q=}cRwFBBwfc~BY4g-2{6rEbM-{m650qx z^|{n|;_zWeo2#3Y=>|Ve0(#Y)7Nywel&yjJMC1AS;p%g=3n+xHW&&@kHGo5uu=vKS z=`3?V6S|~7w%a5 z{}=htve$^OJZLo1W}!u*ZTG9|M}ecn)6-YdK>$e;PpbW+^8K8}!6N_KMOdDCdW!;} z?sFLI8mGJntXnvi29p;0^HLaV;t1fLNND@^-92U2w4$!I931qha#C`Q2sk*fIsVZS zBna`<`##i>ropjwol`Lv8)&Aq#+2uuqa5@y@ESIbAaU=4w-amDiy~LO&Kx2}oY0hb zGjdkEmn*sQy#_>m`Y<}^?qkeuXQ3nF5tT&bcWzljE#R0njPvCnS#j%!jZnsMu} zJi-)e37^AC zGZ9?eDy7|+gMy$=B#C61?=CHezhL$l(70~|4vj?)!gYJqN?=+!7E5lDP}AKdn9=du zhk#)cDB7uK#NIFXJDxce8?9sh?A$KeWNjKGjcPNdpGDHEU=>}`HxpYfgHfHh29cAa zUW2P@AB)UO>aKdfoIqg0SGRpc4E&-TfB3Y9Q%|WAj|mG4e1$IOk1CmNVl)I9Vm4wo z3(oVdo}JO$pk8E*ZwuuQ1THZ4-TXOKvqfwqg^A=8eE+D`MRVo|&eynm{Ofwwm}6xr zi-ZBSj>L9g$p$AoVv9fu6%h7%f%`)l+O2bZ@%rC3f+-_J_0ap(NLXgyPxdw$HM9~= zFABy^XplC%j6ExbJHBu#cganl#xs`^X-w*M1U9Y{Cs%L|!sU3)rK(498T1HYtO-*t zE>i}}Q^5VijVUo+a{N20QKeZ&mUB)$2x>!>nfd_<&42MzO_oU^Cuw3W1U>C8k4Z-;I)Hwz}clprW*1#cN9Eb zc+)>qHS%7}9^t&jOjsczIIrb)IhH|7_FvnJ#3iry6`pc8JS^|zdc`sIrW~1v44uAu z4cXW$3L?~kE9>1tR}nrfv_T83-xr!;EgYul%$1fy>9C%r0(M(5`Ww>Z8eY8jc)$22 z79&%(H(PfzKGg~3+n=o!mLRb+v51(qU9bb zgq44mOQDCxkf_0mCPe6MW31cl?In&&s*%%+%XbEe{59^Z=D4z^C9H>b{DB2~UamwF zuSv;}X)m89VM~{>c0?+jcoejZE9&8ah~|E{{pZCGFu4RXkTYB4C|2>y@e+&j`Bw8k-+O@%1cfIuz5?+=-ggCj*qoolI4MOO5YF&V{*r$zYEKQldnW$~DOE*= zjCNv~z^rJMo)l+4GaQ}uX*i+ZO3((%4R}J!+$z^OMmeQ@g}-0CU`Y!IT4V!T zsH%huM^)eDsvK%fc_5tS-u|u^DRCgx=wgz($x22;FrR=5B;OZXjMi_VDiYp}XUphZzWH>!3ft&F_FLqSF|@5jm9JvT11!n> z@CqC{a>@2;3KeP51s@~SKihE2k(Kjdwd01yXiR-}=DVK^@%#vBgGbQ|M-N^V9?bl; zYiRd$W5aSKGa8u$=O)v(V@!?6b~`0p<7X1Sjt{K}4ra2qvAR|bjSoFMkHzE!p!s|f zuR@#dF(OAp(es%Jcl5&UhHSs_C;X87mP(b;q0cEtzzDitS8l|V6*s)!#endR=$@lM z@zW@rnOyQ#L8v!Uy4Lf}gWp9dR=@Z^)2;d-9604An?7U4^zOHu-y$2d#C+DDwdwt6vZ)P1r zEmnfv)gMQ5Fez$I`O{_|`eoD#e|h-ho*m}aBCqU7kaYS2=ESiXipbeV2!9|DF0+)m zvFag{YuNeyhwZn-;5^V zSd2{0Oy(}~yTCmQzWXEMFy`G#&V>ypu4f&XDvubOHzbVle1bo;(7-=3fvAS1hB{r{ zK9-O65t+fFL#0b~r6L-?q<5=RcKTM}V$WkcEkv5iL&ukW?jO^a^rU=0Cen1H^wqC0 z{sv?taDA@di!}>PKt}4{dQt=zaJRlDSS3%YCQij$@El(EeS)@&@lx_+=r1t|Q3>2v zCDdxkooWqzrf(+dORYXyBnry^vm>wyd0hE~6T;p-9~f0^4m~AUeAv={cet7m*{2|~6vVAM=vpL?8r|>+7ZfuT;*FKMLJGNyc z)!M?FJlzd>mzyrCJi3SQM$eUS@xCJioofaUwqrzeQ%S|R`Aa6u$h3~pn3ge8H;U0% z+Z~w$tX*TF3?Bia(5OK1--uI#gzJ;b5uLoH{ZFw&E0w}REn0XA!4#HLjdvE}GHCBT zMj7g$9;PwAHTUKI5ZL0?jTRutws}W@-^ZQvY+I`RRUq^H(;hro2sF&qX0$Sn8yjq1 zS-XgbgdmyQukGKXhM9c#5rJ(q^!e2^A|dvfiB5oGPSLeAt5%D5*PeG3-*&*guZuuC zJBU$e7TQYCv=P5Uu*IQUHW?0y%33xDZpbd98PO};2E)HxOQVOU|UymxHgZ9B@5W$*}2MWJa*c^h+fpc9wwZ5c?$46XDvb@ z2}v~Q+LI9-eS9J4lf0KKW+gGo70QNXC1;t@eC1Od3WRDxuCWR+h{JeQTln@;u^A#0Ge4Qp1=`> zt(XIo8r+4#xfGhRFBQT(lgt$%8A30KhUoG{+ik~fuoeR8Ud~f*o zN#9})#5rW_+dgG!l}{1c%z{6AH(Tvg3|h;u2D`;{o73i$bqh7Iop3+H*fcNREDYT_ zV_$JL|Eylt9GKs|rOxX5$xtGCZEeAQKH}yQj-e(UJp}D!_2yJ@gWOA&MM>%1!demF z{DzSMQm{L!n=px(sn{+@2(U%8ziqH>-40JBY~3gL*LpzOteyy^!}jjLw(L1_o}Uk# zkKOf^Zc3kM+N-motfgs9@a}WnlbNk!W-goXTetqGjXAXc z$y3qKU$bLO7v=B~DBGp6MY8{jqh`(d-;*ilDsa5kLsG3nql?h0gTJ>LMhtReWbRU)S)mI$^JHKjp#>5BrWm#uS z&6^i@GHwk&nGLSz%FztTWa8``W>tAC{;-Vadc3icr+*5Tpg1 zb4{+jDC;o(mNXIT&m#g)lCPKSRP?zt$jhdxu=L}y*CL>gNCS=sCl`j~I9IwR0hkQC zNk0%Mc)XPszHT|{`-Hp9ZCH;eb4c<7?i;#qszYtx_-^5xDYJR3FZ*l<8yA}Xb}g`% zQvia(gm>;D3o7NQ-GgipuW{}`$MPFUGAzrbx{1i|?cuMGeLCu){I)gxeT2lY%p5>f$g;-r^p8fOaa7MlL zOB$w}<1+naU2bU$qq8(UphBVS{il1Y%H%Ot66gsPl;7oMV}Eif_WZ)$l#gYl_f z`!9^`Ih-`#inT$_!|E=KMw|AP$5OZan1c}{81&!%*f?-6`OBAih;H|eKf;SD7SvYJ zzI!=qL9#@V=6^Ed&Vox>nvRgDbxB_G?scQ-4ZOdqdj8RP9skm?jMwcFwCnt`DMh#3 zPx|w1K!Ml)Gcv<|7Q?Lj&cj$OXm*u%PCL^ivl`om5G&#SR#@4=SD~LX(^Jcxbdhw)5wf$X(QCS-?EVV-)KgU*f@rc_QJ!#&y zOnFUrTYr6Mk}Z@%Qbo3$IlJ$M@?-X_S_aKG-u<$&rk995uEm5|lZ&I?TEYt9$7B^P zh2HP!B7$3DdD#;0C|DAv-v(3*Q|JpR9rtw@KlcjR z0u>+jpcaF#*%yK3>on*QPT$n!hVmV?3Ts*6GgSv4WmL`R|5df<*oLdRtm2wssW!KC zANH}}tLuVDmi`i0E&R1Fka^c(-X?U*iL8Ni3u&xU@Cju*t3?-7mMgv#d@i~fK9iXzdGFDTymtyi!gn^Fzx1BNJP&lM zUsmCM#g|#v+_f=Bwx2VIz0a!?{k_u&wdY!H)n;5Filb}BC~Dd zleclQdsliFY_`v=OWBaLQw%{>Irf^2qsPwfC@p5@P%HZ<(=Xl}n2EvcWSC?(i?OY1 zvC~5z*DPj7bacJde*UiO7_88zd&53d@@}-WtQqfPE7fZ3pqKF*Fq#f{D`xfrsa@wU z<*UY85uCMZSrwZ8)Zjhj&4|Xa6JbcI39UBcTjM8SJm_RGI+SF6%`K{6%jaGz3>bn} z+_X**pz=y>rP<-ElPQyC5s&80wYvX>jrC9)DWiw(CWwmOALHdL;J%ZxDSOP~B6*A^ zvA9^=p}pk1%Hw;g2LAW=HZgN5 z)~zf0COD0!sIf(4tefY|r#UNQ3*Ed-xx_2&1=P{a1GYu(heIonxLsE;4z5%~5PV+G zn75(GucB<9ey_JzfqTF@|E^G{2lv&{W8A+uCNx8}!;{`fXXNVUWdk>vQT)x8#S=20 zxtV0no%fhw&@#V3{rh`fUu(DC;I3ADmQ?4kRO|GN3w_z?IEURYnw8c~?CjFGP#-#o z6gxi=DS(5ZOw^TRNj*Ya+u14%%PLH@XN&L{9qlq7QswNCL;D{qRJt{qk!YsZZMQQ& zpL9?2Be@!`V@xFODnG)ykGOt$GdusL$~Beo#G*t!R!z>WA%1S}UVPj`)8)QQEp)R? zNRlD9@_AzW1FNeC<#_Rnxwu`2rChms6a8n8-s5H)8!6wf;y=ezsBCb@2=?%+ZjD~>TkD?9{hd{mviZq&e@@syMi~U zd&=3NKjgbW%mK=%vv}3C|XwTn{657 zbb~Af2pBjxh4)hb_DyqU?}{vGa$0wA*G2sYHC$?DOmM^-6W#0b4l|R-yYDFkj_7%~ z4GR*+&k3YxnbR@Lwhi2Y$1K&)$0tR&(no+~FJ}E%z!Lfj33|sT#!5-MsBQ|fpxRI7c%fg$8dcKMWe0Kl% z5&ro-HQiOeU6N*GaPWJz@Xp;^$)vl2N`-Y+6Y>aJpuz5qRzjJ6dWpvbc+4+Vzlz!+ zMa$YdGf{^1e)cq$COm-0*!-aHVF}nYbz{GW)v>Gr)~Kp70Mb8(Y(ZihSi|qF5 z089q9BJI!Buu9C!yR2*Y2q4kcM{t?tq@|G|_%<@ea>STGXz2%?AASW~uXEq{Br=wk z;iYtbm+uz4>eazwD!eYWHz5TL$FioIQmm#<0q=S&yGv%>(jRr+j0xVP4fwW~TW!&C zW;FK}vhuHx>NIf;<_bI%=cHBC$gQaA$55KdxcRQYC}{A?n*LFZVSxOh>9RMUq!p+1 z3b+o2kA(^lme;OnzCpiD>d8gsM4FWk<_TASAE>{y?UnzI-kfutXG!&%xG*OQYE5*F zKRZ&$x^-pS>w0-i6XiYyMz`?ph1BT6l;^LoTMlfY1M1dsU~3NdWv|JT*W!B*rE?zN zL$=&u)^hz_W=Q*Hu=D)oB7Utxr|bE&BI={s8ij4!u?rlcer>!d<3W$RcL9~X;OWqh zSOiRkO`m12Srj~HGB&B)ExJ7|u50z<(mvj`L@%c-=D=^^l(TR?pzXQK52^Y;==qY< zbRwd8@ak?QQX2^_l?sygrJC<#-Opg|dNb$inQC298xt1{gp4!Wo&@1F_^@xEwSV(I0PKsI}kIF$b$=b-aygh z_b$B~T;22GMW4NvE`H-P(UguY{5O4^L-@Y)A^35c5x&<@_XlVuj^_#=jcOblZG9 zdFXYD{dweuA(en;gvv?Zj!k?tAC0ob&U7=9LnCI(7O$!wjHZbdX?2R^6+HWEZ%V9% zo*v1!(M=0%3%Va$Tnb&|yXAO!r=M81O3%#UKV2`L?dh#%H&0!C9C)}_jHl$DG`ufC zGqzclc(&4Bj`#B)7r?LJDesZEAF2vUhtdD~;y3HR z2K}eo-2b>8-t@0;kN*oyG18CF>1w{Y zBeHf{*q3<2*AtQf4s&-m0MsH$EBv51Nj=s=Appw|nd1Yi(-DKZBN$9bAlWN83A_)0 z$4U=S!XyBuAm(`t#aW=l*tHPgHRE~MrmzGWN*Eidc=$BV2uYe|Rpi@t-me&ht6I?| ze$M(9=%DxSVTwNL7B*O`z`fRE$T)18O{B^J5OHo#W%kD-}gAcJO3n1x6Q{X*TFh-d!yx?Z$G16f%*K?exQ+p ztyb%4*R_Y=)qQBLG-9hc_A|ub$th|8Sk1bi@fFe$DwUpU57nc*-z8<&dM#e3a2hB! z16wLhz7o)!MC8}$7Jv9c-X$w^Xr(M9+`Py)~O3rGmgbvjOzXjGl>h9lp*QEn%coj{`wU^_3U|=B`xxU;X3K1L?JT?0?+@K!|MWVr zmC=;rjX@CoW3kMZA^8ZAy52^R{+-YG!J5q^YP&$t9F`&J8*KzV4t3ZZZJ>~XP7}Bs z<}$a~2r_E?4rlN=(}RBkF~6rBo}Sz7#r{X49&!gODP+TcB*@uq57EII-_>qWEt44B z`5o+tysMLY*Dq^n@4_vzKRu3We5|DI+i%NV=Z|)QAl{di_@%07*qoM6N<$f(5Fv<^TWy literal 0 HcmV?d00001 diff --git a/examples/bot-ready-signalling/client/react-native/assets/icon.png b/examples/bot-ready-signalling/client/react-native/assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..a0b1526fc7b78680fd8d733dbc6113e1af695487 GIT binary patch literal 22380 zcma&NXFwBA)Gs`ngeqM?rCU%8AShC#M(H35F#)9rii(013!tDx|bcg~9p;sv(x$FOVKfIsreLf|7>hGMHJu^FJH{SV>t+=RyC;&j*-p&dS z00#Ms0m5kH$L?*gw<9Ww*BeXm9UqYx~jJ+1t_4 zJ1{Wx<45o0sR{IH8 zpmC-EeHbTu>$QEi`V0Qoq}8`?({Rz68cT=&7S_Iul9ZEM5bRQwBQDxnr>(iToF)+n z|JO^V$Ny90|8HRG;s3_y|EE!}{=bF6^uYgbVbpK_-xw{eD%t$*;YA)DTk&JD*qleJ z3TBmRf4+a|j^2&HXyGR4BQKdWw|n?BtvJ!KqCQ={aAW0QO*2B496##!#j&gBie2#! zJqxyG2zbFyOA35iJ|1mKYsk?1s;L@_PFX7rKfhZiQdNiEao^8KiD5~5!EgHUD82iG z2XpL^%96Md=;9x?U3$~srSaj;7MG>wT)P_wCb&+1hO4~8uflnL7sq6JejFX4?J(MR z(VPq?4ewa9^aaSgWBhg7Ud4T;BZ7{82adX7MF%W0zZ_mYu+wLYAP^lOQLYY@cUjE4 zBeFNA4tH1neDX`Q|J)mZ`?;#~XzBag&Di1NCjfbREm)XTezLrDtUcF|>r`6d+9;Z2K=0gYw6{= zO`r(C`LX~v_q!oQTzP=V(dpBYRX_m=XTYed%&nR+E%|WO3PI)^4uPRJk7kq+L(WmAOy(ux(#<@^3fSK25b1mHZ&DAw`q0&a5 zXU$pWf=NbJ*j}V$*`Y zMAz4Zi@A4?iMs{U8hRx*ihsZYHPTpP)TpG}jw4o_5!ny)yKkJoo=Bir+@d$gzUtPf z76rl^DOsUwy9uARy%q+*hrZZzh_{hGBXepC05GjPV+X0aCfbk@fQWuf;3wQF@_yMe zt5AXhdB6CNa}=s;{GA3bi9jK8Kx#cdW9+*ie&)lhyA|*h09Nk?0_r>m95{nVXO$6+ z$R>+ZL^ryBs*)RkM6AqpNS?#{nnq$qo^Vt5G+ytRnl4dc&s0sMr1WG4?WRPcp+ zP;4wHTl?f)^!Gj@FV%`g0(eGv;HbO<_}J0}FndK2L|Kcxs9q1mJ&rMg$cKcFmX!S! z0vJ1OH3owS*d>`!`*;8rrX8t`(L`=H!AifKdlcO~&e#f~Gz*D+&)!2#ud^j$6ZANS!q}@cvw*7N5+0Q4R zvKIiqx03&fsKF9NtB8=DY2R$GBF zFO>1hO8{sMa4qRW4rz_ZeDmKOIy>H_iVr#{5#Sj@pJ!sj&rhsFLFP!^^K&|Dr6uLtPu&2WmLoOp+72f`> zM88yjBZc@DHb&cF31E_s3Lc>O?h=~(jh!O*kcTy{W=1>28}m0z!NXv!+39S{1Oo=094 zX=(h?=(7}XGb1D8Le$|=j;d-;;crtG&kl~$1R;+jNJ~%pbCYscUVDFEU78K}k--e# za(QZW#pp2ud*;SAz*bwBzqqTRikI2Y#5?gmB4!gw{q?IKxBJ$Ekk*C1u@L4^va%|d zg`199czf=a{W_rZV(o9cO3-ss^nlj#!JCtP7Us%{K*#UAfC_J8t8O95*4X1neL!uT z7q+4#870U_4@PTELQHYcP!d#&(5s=1xX@nu4~{P ziXP#%91t7KLLnvdo!MHcGH5gCyUtMXC>j$4q!W8-qKL+{QA?W|P_g@&o};Qr{V>;Uw00_+`9LV$n}g$1Wz-iO^%O9@tw3qx-3ufU%wo0W1X6 zd5hj=!1>$2#x-W=@#r)rb>i#BX;&5+G{ip^1}TzYa#zzvid~=DT3juEZzPd*Ptx5PlmOekc^%T@qfGKnX zVLtTc?`|*HLs@&g^HLc-XM;hT*okFVoGV>Rk7|YR#rP|>d%?%Ac6a6tD?jV(PEM2| z)!GQ%0<#4uaBClL!}ieEL#lNYchYI!%yOx-k)Hrt@v}`10WkK6dpyGbIn3J}K<9>6 z&Qr3w#HH4O-)FlVQbmE0IsYU?*2#U}c**@5bJg+B;Z3a{C!Wn z%}5?fNU7QX-m!{(5YE8DV9$RRbxu+^pZ&ZnAiN>7Ej;=f|mchq~oo_duHA zm}UoOBhc=BYSg6-FC`~!vzKFuZxq)d%0s_mkb=8gcX@+)g%YXM+P;snBBP?OLzICI z^nONGyOXmz_6V@ewl4VaqES4q;1}i2cE%ze0*luwQ@4j=-woV5=th~qD7<$}vxHqH zki`K3_K?tAp3?w8qw7CdG)(7lggoq>PPlkt@rNqVm`Ycg!CT9)9T8abyZIZA;Y;5m z%X*dax+I%)X7Yjc(a(`}0da228T?%A)(62CEkfr13$PzqKi>>_-(@aRUSr2JRNn||G!L%}1dKJ|E9+0HUy|x0-9#8- z__=}bb&@;)o<6PQ+SsWesX{>caBlo2%~rhkUU6n+Pfy5N$X8vK18kZm*^~XJsG(og zBO`Kur%3CE5}R|r$by?(@1|{;bLg+dG6WvJ5JO>#SNDdi)Mq0e&KQ?o%pyICN1`}n zIPG++itoD%6Zjho*jBp)LaVIDkPL41VQx_s+y{K#ZZMFUJN!!59D>C?pv3!jpgav( zrWmF`%6QG9&{*|Y2TOEg;yXX+f+FH}@zJ?z;cQ;60`OsF+Pun!-_^Oh_aQkQeRK|! z@R;}3_d5Uqj>@W;{SAaq0{e2oR($}c?m}x>mw3U&EK8p zbDNT;)(io|2H)fID;xYi(7M`Pl2^igo1pxecivhQoZrDJYYqKXg7)kPm6M}H&wk?1 z|CR)0PYBK27ml4L*mD4!ulgjD!q2H)&b>^b(Z}^4enh{P^oa<(*DW{p)=!K!Cf2yxArAy8esW_t$!wO}OC;g>-Y;p?(8K5Lqzo zVOhL8FZn_oA~?Q9?Wp}%Z1Q|bKd}2%!+#WJCx^^$C*0K6QZ2#Lm}2_VciwAguz0^a zyw?EN>H_b-HZ}3A`6@(yG~8IYa)emU9NjV=esnMsEpL5I0ZtmYfC8%y6>s_lxxw#E zG^q&>1%X%Rq$(&YCp2v6OnGR-mI-$;?ekV}$>8saMk6~@idK;{+s(Zq?`iUsro#Rn zzK=vUonDa1DE+ob8@-xJ^13dF>)CrThqq%v97t^q4e`&PYde{8V33VaZdX`=oBAPu4=@9clN{P5AM&b z`|?IsKKKQs>6f)XqgFHWEv{GF=(s$!WorDO7lh60_n?q_z;I`mZq z*dn<86V%zQ*m>k6jwwD*+Tvl&G&c*s)!Qmq5P(FqOG?8SR457Mh3XI}o* zNHJnfNc3rddr4S%F5TL`3ttEi2p&B*92mBV{y_fFcD~9Cc1oH&eyi!@W)XDmr!-Lc}2ziivlJ7K)m%-)5hd*#%qjqpv-I0wp)Ww;Zmhe}i%+uMaYSzlf15j7cS4Lcg zSw_~_f!|o?!98lFa72N~m5HV*@680?k@kjT&o_ld&VK=i#LoRgmXTJI{t}u-HdRZ?xP84*Y8~` zqFW_yBG2VbRtq|$md@m7E{$t7b^3%Cqa|@prg-_BqkTptrIu-ROancLO)(0 z`=1nJO?$p%(=%NhuS`x@r3G||Oy!YPtYHd3F8}Gpd5? zgBlTI*{@j)(&e2)r%evo5bP~_(UYOO{MQk^fQqpvQIEd=s`Y7!rEyHF6#dd&lqXBj z{|hLWB%YCqcVlq&AE8P_$lodI-p~4@dR;nHMQ2FmIOOL`<)D1t5VfCd_YzcanOlBt zsL8m#o5134a;vzx!oLHR`N~~sP@WwvT?bz)a<^pV!b6r$f9^=S!iu>(V~l$UF_QW@ z!jio9i1}8uto)xGyTH-HFBncUqGi4lrD{Q`&u+;dL z7?|h3?1oggBM*H{DI5sULUT1H*YkzV_qLG^sc%iIgZTIw;OSOeyh1tMAY zSE>_9do_gknQA?7{grd7)rmnvoMHyAhTAnruXGW5CH(TqWX~?>l+3`Z`IZ{MAO_}t z>z0mi4wXAv4ZRp4DOLP=OH9o7w>!9tx#eDG2oy4Ma3!FI|DH(Z`MZqlPjidSN?!+$ zxAP0oI8On(1j=wbLHW9&CxWKM7y*dfaz2%0e>3Bk9$HH+poGt8IM4O2Zp!L+{o>)TGM-lB`>PR8Dne1b=v{V}GsGFDR6 zL?jl3X>eP9=IXDRx^qg$yDfIGM{KhS@4j*WHp6TdG>Mie2RHg82( z!YwvpPJtaPNlyo|V5-ByJ~FNdS3jtrR5LFZZFjc~l%lkvldKPru(A4oET?;Mo0KeZZgt?p`a4@) z)CnT%?S_k4DegHCHilm~^F_lg&w*-=5wnY--|%|j;2c`kM4F~{#!A9F)TLy9i5Om! zGf^3|Fd`_!fUwfTJ2E~!Q?Nf4IKX|HVM;0LSu(H^|202t;=Pkd%$wl(mvzH4!mEbw zygM6z8hzkanzrS;p+34V;Ahu&2H1nB;i!W~D1yw={CxUbmC`pccY_aa!KB#G3x?Ji zjkKo#t+c@lLa%4C|1#`FT!RHCmzUmffD-n|KTh5?_aJ_j@Nf4G@ZKA5hRyL~KE=D;$L6#A z+anClym(vFCUa6`mh2H+eCQ}j7N2II_7beG;%^FrtEsL|yur#E`@#U~)2`~Y^efsA z&Upac9Y>`9d312?bE^)0sxhayO07&;g z#&4bUh`Z(-7Y*$M_{0jbRs9@D@;s;4AI~j|qj`T1G9)vhRn0lBf&; zDThp@IKRj>^IItes}_6lK!YanIoN&LGLU&fXeWbwO$Lw+3`D`~?+tZ)+C3D*F4VD! z!YA~jLKQc(iUKMbQ${@@%PvI=Cvet*TcTe`3Tm9?Jw8D`#1kU0%T!+yTD58D#$S?< z08SIHoPJ5$Fu7)8-82N`9ssG(k|}5@(`$kkOa^DI=sjZ>mJDIzT@2*l#~G!|Y;P30 zEuj{><|Y7e0`>g8mDh}S)d-(egD^KCCcoEcx=L42Y*7{IQPA_2Gj63jC*yH7VYxse z^WgiuLu--n2w?CMkhX~&mpdQ?WAV5g_oGDJALfosHq;QF2`+9#-&$?d77|K|-T`aV z+KtI?WJ6w|m{mH^#phJS02_?+l7+Op8`d)%&%CXKh)>}rVP{1RNQ;v^0vU&c_mg}) z=~Xr1v*?=v8`h%Z(4W5)bGiKujAq3i}g-nmv90otzcnAI&?}v10NoRzG$vHYtyd4DyePWNt^4l%sO^^H!E(f~f8VWd6 zaJO8ZJ&I;+fTqUsn|B1gu%75Zzq_eGBQ(ZuR)Zt@d4&PdgiG-=F~!N8!zgM0#=p=> z+GPqp`i^As;$u*G^A&%^ML+kf0E*Dj;~-lx&ovlnsXlm+u4shDPz!rV$sP&RKi|8G z|6ruV{hm;FVq8i|l0F6a1wYu8{yckALq*+Y>?Xe)`jeFxXP#11gM(6xUBeSk{Uk!krUo5_7H>e;Dv&W$_2jrFH?#*z2jY zI#JyAOQ@r-f0EX@5RWJ8!L|#5xZB3zS2t_qd=bafdoDfGk8lF3pL8KAZ!a4!!pgf83>i5Pu zYMyimE!m+Pmb_Cldje-6xU_|0Y~>W12^QzJUQ%KCfn-h(j9E~e3Rza5+0iCjw=GkR zllb*}Z;86cW~@;2#H$^c?SJjen|Sl%_P;(afLk#HkXSF6^#|7u~~%Oy-b&-M3mB zF)Nw4XIen0`tv16 zUQginofO=-m#!+HAyx5_)7k><*g@oL(=yTyqlA8~)>yHvh1y^rUuUl|# zX@i}tPv7iUsqQXZG$9MxrNW8?H{CBD{?0gIv|}eNLWrI3|6z_KZp)J8kIAx3`nI`v zt!LS*vFdaj6)Dg7@H4xJox2zl%!i(imn*s>~@mV%AwKd#8KUFwB& zsSP3wcW}%>|F!f^RigSket-v+*WKx%61S80a{Wkv_#Epof`lZKNR<`w^~r~xkgQ$3|sxDc|{U&nVydhl3 z5zEN}oJ`pV{udB9#Pgu;WrF(!CAP~yte|3PJ3KnMU4zxuhn{w+$U_6zeNK0}-V(8T zgBs86T&@CVG+5dDki6y_0YK$NCZ?s>68}OCmdv1jjBwgApk%Vl5O&WmNnmUbPR9p= z8=TL5VlG1b?Z8?9uY5Fb#-(Ca&__o^EzC02_O!n$pmUEcluV)@_mE8G_r7g{ z_dMXFp3`5VcBcz&2MP)FotYrnziA%ADhbT`;&Ak?>a(iE$j4wQ3*>1=%u=6@W^d-C z%A0mJAG1qSL9I{~*5uT(0rwc&$7OB58ZO&-S@Fq*eJO+;gL|V0+B|VwE|{mlwy&vl zgIqxW`{S9=(Z_^TBe@wDxibSgU!NH4kui-Vtf02zv`cDBj-yuqg+sEjCj|C`%bCEz zd=kBf@b^zG#QC+Y^taq&f>5r6Jz;_Y0JF+M#7-rxfdn~+_XuFj7@zDz7Y!k6LSo$4 z$wm>j>f*QauR^_q@}2~WpSig8*rvl1v^_a%eD5pXhgbDkB`mompqC=tJ=rz?(E=S*zcha14B;fw`=0=Vl# zgMX@BccXu%)OHr^5;@K=bbFX5Nwh7X0Gt`DcnnM4LDq?(HMn}+Yi>c!UV>MgD~62( zz*Zgf$8KU|VoDT#%^svR|3%G4!?Vu%0#YboHfZpIV5L%~V?g6=gDp91Zq2Vt2(x1M z77X|ci>WCA|J04*{}gkXhJ5ILR$)pUeJ3mhMt&Xtgx`FX(a=dzs9rdk8u90I*_@`_ zth12y2|+N)Lf?KMI)~=XJBIe%q~Mol^c#HbRX7E4PlS>4x)3$T;RmP;F(BMKK*SE5 z{)0t5YoK5m;t(td&e9&^*&9*FyHA05x1VDD!sk8c5ktSwKpC`#vG$jPAetb*=iBy$ z>&Mp?mGMJs`6l^9tOa09&^^SVUc7i}h&4SyPuUxD)YFkzn1md*nE@dxAxDv_bBOk# zXqA9%{Ai@0-zGeif6w7I41QxK3U;xSpq=7%(x1Iq)vdNoU}xemV0yJ zp7HDQfyym#9qDVe6<{;O0bJ|9IPfYkoIxYRY=XToDSunStmuT3fFT64FNWDKgmGvD z+f6=CH$a|_tey)ajUTUAI=(O7+LKn>f5AQEF3Bh7e8pbYAwz~5egE7&ptm+z-r ztWoekP40Rl7K4-YzWjX{be8rm34X7}$`P2iORL~tixDmlq;Z(fG2o+6@qWrhOStVH zbFcjxChq=9_whhS;w4xF7=1W?>Tc(uzAY@zJVX0>TUFAI4CAZ({12O=K;08G;HA}m zTle>T!oaprs}9KTCixt#IrR`=L^qo~CFr$2!*6|hf=&oCk!lpxnBpJVeO(9`3TWUz zZDza?g3o_-DtI#na}{pxV%bgz{6@2-t|V?A&nt_S1jF1s{BopN-!rP?!q3KJq+J4X zTV>T0fuo^!)nIXJJRwXu#an<$St-rAHVvxLg<$z_;7-Ff&?=hkh+PKb3LYhn3(357 zDnQd1arx>TLs}B3|G?tC_R!SP-r zw?k?T@6*IVnPNzb5UjxT#9LtWdM#V~D+v|Cun;5jN}Nb=>u(MG@@Zs%8>2HGlbMu= z`%Pbj7}DG~>bwy~&0C>?Y z=Ebap803V9nrSLWlB0m#wf^lDz8jeR{RNkf3n(pvhmRn~{$~@9B*CW6Lj1A~xEO;^ z=ahG9j{u)sV1->1D{F1bm&T)d}DZNCGRjEBpw}K1i|b z#T=G>O^6Zw1^7m}Pk2$Y>SfknQS)zt2RC1|i)j${u&nn!|=9;ZYe-{Wb@? zRyg;gyZDsCD0rCvVZ-dYSgc(1$yY?0eT+#-*^ln+xfo+$?4hj+6b{e`mEB*rvx2qX z9?~=^hk9F~>6E?ocXN-Dq-h~r8RbqKX;HY|qIb9lTy|SyZ-7#NpBFz*TM_5lQf9M) z);F*BGk}$qK~up`>nKwFp)PWhrXcOSCYx=j@i-CFkcVdP^uHo)A%YWvm0DE2@HETU zHjUOU(KtnAaHMlwCX7(*v>3IOVPEjZz+L0v-eQCA(6r8gK#Kn9L7Wid&nszI!9PyL ziTfR#&;G2Z3Zix}9E2Ea>R=iYV2mF=G#icUe)U+t1`aNHMD&N(-zKfu5JKNrNWA;; zD(VPWTDdrNo)%%s&&My{$^xWo@;@X(z~dLj8Os#?z~^thrTkOw1PN9%E_P5O4h!NO zBy@|K!p=CRg$#G8$@PhaK*yFm_P-3?xkYFr>*QZc%4{)AGZ8l~^-N}&7=a{dk3!~)!n3yks4(~nhE0wleQu)VTDwl*>Uk^-2Gj4kQ*l>vLAU^j$%7@IaFaE8@0 z3+dWFd@ab3WmUHBX`ruH0!@0wF-_tc5a;j6>m8^&Or>Ib!PR}jU`GZs@`(21VCOIA z1ghU0)IsLDEE=pCSw!gou?-)uI-XmTlYlMum7H#9be#y@S9Yzkk7BU1QZ-%oZLqu2 zECe!NhNpcOm#t+zq#vxuop!(byd(5p^ORt-5ZJlP1>6k*rca9CEfu}`N%b_KCXTuN z_29!yXf20wQyU?cgyCEp%v3?v;9+k1&6qSv(3%$MwtE7O0!w`&QQ*PpCwIn>7ZS7# zqrh~jK--svvT)WJUVaF=}_FZ?L%^AOmN)&-7wBK+d>6 z)}kj_AS$2c9{zGy7*e%GJ_O?{zo2PRrvuWC>0Ol<1q1TH*1chmD!BE<9YRz`@BHBS zC<7RUL#|q%;MW1K$EC-?^h5=Afdb$jVoc9$sw3x@;iCh7avo={xt8I<^m+8XJ3Rpc z|D)s#sNWp|b2q9miZm(EN)T9H-0LLVVLF)G?2qf2mgP5 zk-yAxE#$J{9`irn&WLLP7>oYxSiDE=r<*xqd{b<*Fac1#h^}mZLF8?uaH737@S)5? z>|mi?h-%CRaDIZJFNLvadCv0#^=JqF&qvu4;^Jl*1aV~Jo<(d+q__;9qV=NkHIeB?H;{gu+oLz=pX zF;2vEjY=KRwZD8^Xl(r~SzZKg;hQ$cIk@4V5FJ&&zppbTVfzX9W#IGh;0|*zK6*!T zpVtA%`BBB#-4E*KKz^cZ@Q>y?V0rq7`|W^xl7JRr_8JNy#b168_X^}&7`uVG7m!-X zdqs0_z<-QbrW>Sh4pgq;$FeqW%R@7GuT2Eyv{V>ix=B6Fo&UDQ?G)10{SqOk<@&ww zX6~c2M}^&27F2e${pMltA2fUS84aKHJ6b;o;l3fQfxDO}0!`y{;y|`@ zMTJNy5u`k)Jyip@30b2^MBYS?0Q!P}Bzzmo)_12HaLg}2QauF+2MAk;99YN{Y*83D zZahhIpNPMe5iAJ*A^%!QcNS!$eawnb>8GD$z475a`<4D(qVqsAhyq`Jm7GSi2e+gP zoZZev?JNDqcq!I818$!c$n3&bY-&{xy#T=$>z@r@MpxX}15`o8%Q|ypRnc)yFg`zb zWW9EwA~ib=3R(hopPP_E}og1_mqyHwHqH`>JPK(jK3U+6qr%&EDiuevSEe=wQ=GH}5$N zo5U^;$A2(Hjg;Ki>2wE64xb{|(=K}k8qidag5Dlwhd&hyXk}1ytqnh8&9D)IgPgLM zZHrDnH3OjQm6zS3?Zh0@@93aZ@)S0>Wig43rR{-;;{qcu8eeNA*Pr0F3cT5#IZnE+T~Z>)gy+e_Q$xsj*}TIUz5Bd`7LREo`%zq zT9a88Gs%pwD{P1JIx3n|(r#^f$4|RK_8Ja7pofd^UT5hx9?4Lcgqv^T1$bM=^(We+mGxRi6*8Ipg z;PPw#RQki84bK<0I4w3#gH}D9pW|>1Y>?KhgQ5}|dTv?B9?TlQ^z{75CZFW=<_Yvs zGzfXrCXku~zp?>6_-L`L7Z<{vOv|UCkkYAr0b!rE;4MoA*gG^lK92~tQjF1&*Oq}) z5O0s2K8c4+EkT9>vbF9wwN4eh)z|SKM6=1!$Q^MvGy4c_-0VYPY8~lndlVQk$)e#u z?PQF3bx!BCZ4XWU21kp&^m1HC91tf@k#0SOtg-t9I-lXi-_<;~kJgJixU?RcU;8{7 z@)M2QFejGga0u$h0H0T1rng*P(&Y3{_=a5$ObI8(ZBCE`vD|cn`e&;Jht7I*#T7|V zr$|2v6jZ_1FXA7C81?46k^SBW&w|+^m}^XK;1l1dnS;HitpLUEC5yk7|D#1rm?Z) zg&P;AwTWL*f&ga;qusIEptBAyKKyDj)tEeHpILiMNAGN~6M%P(ZqiPZ2TEH&*-F!f z6~&;}Uz=BW9o6<(jv3^1t+b8E#)LeuErSpReL2(q{cq`vD+;`nG0LaBK*5{QAOcH7 zUKNFR$i479)BYRD_P7*|@&*MrBmhP*pNl6+GX^A1J$kv%>K_n~mjpa$ofX^|jMZ-x zhR+JM$3>Lp3}V1pVdP;Va@ykoNZwLOZg<<7ySZ~ zVrYV0HZ*9ithjz<&v}cP%0$YlV{98R;>_9Cy*(vQ+gCL;J14v1to%<+flFbW0%vbr zo_5p^37EI{dMt4zhH^la(|_;q+!WozZ17sauRU;7a943PDIaP@9w4n&uzcHB$~xZKw$x)E5L>JU$XZtC-K6W9ZQDGil8&(C<^w!V^)6 zNC_}mvjVLH9Ej=bB?$Izl%q`^GT~`|;*Ev9ne1t|>bP;Q`32zS)~`B*DaAd}^>p=r zROYm=E;Q+1XXAUOsrQpBX5Bdcgt3vE5&ZF}asB)Am#G@)dB6Onv9Ob)O@Q-!^zy19 zXa&8d*mDufmCoK zQy(&#k4XGEc*e3Ap5veCHM{#fs}c={uAEz<>Xt!6JVNRrI_sm?-_};^HMAzv6he zzJ7i;H0!YLc4>+P0rtQQE>!bWxL0|w* zjxBAUBj&B>tGyH@JR$r^n(7VekMfOhLK|84th-9kf1JC`pRBJ&vco>0PeDG!zJz`u z4g++no(Q2fpf`%q&7jW%54KY{k>Dut(#ugdbN|U5xZRe70mzQorRg=HWk=iP6OC2qnOWDytmOau8PU9a$_gVr!b=s}mk=^LHAN zhF;wBXZf99rLWu{1tLWK$^{Ew0%_h$OlF}r5pW*?0=>w5=W92XjG73Bx}Be3oxeg} zRkV&?DhK1y_5}Js8x}cRmtea@uSF8NA;9!K&?+9b;T|F2CvT+4zo+z06rq8?KEZbQ zddUG7i`dQ5F_|wO(+GzARU`@HENgRmDL>A3f%H>CqT=hTS}Lzn-y1p4DH8?G_2|n! zpyv`|xDlg^BDgt-#MQfDS^3@q)5L{wFvaoEgIBJUkdiqAA;GdN?`xxt4~$)CyLcOB zi4}vO>Sy34#@Y*Sz6#40mRhLg%XSVt`cNQ>e2GI3hb6?=QN5+4K zpC%y`n~>&je;bM?WJtOA#1L5lFI&=Khe{AEABsK~@kXuHA=Lh1?k3tU=o&mvuTjm9 zmWMOfLn>OF(#pFlN*D2DRB z$7c_YE;}Qfn)l!J)Sp}{oohJ8q%C9~j|7^m-6v$I1rfU{#h2C-EY=eCpqSfEG=0h| z5%I1`VOP1+(tk(ACyD!%`X*7_&=2{&-%RPrK#rp=_TH4T5_1u{p?FcOYIX| zbam;>yyqKFzaTY@vvKH7%3fMd5>K7Hf1!``V7EA{ z1wfp4Pd!A;Kstvm^z=AAQ1*5zEXWGy2d^#@?rfFeY!((vGw` zDdT0qa^$BC;Gifg9Q@PvUrwx3;fP1DOkGH%a>_$x80qX}tQ$WJ zqe865Jb3J)%JpLfw}t%onQ4aI-(#IaXaw4%-Wj zXg>WbwKSV@FpBojDzRtfkBig2*_t*vo=bXyIR~e^$P103Eb$Pt+CW70YAj z2_gq57u5l3KlPY-`|l|}%PI9MSgD17lw4kCb?wW*&EhW0PM;6Dra9|#Q?C66l>%!g0MA-f46xZaAU@`@OSeBho_TBL&2DXRGdheZ~P(Z)}XJq2Q8k=q8N$` zL;S>jYc@wOBwOe}X9xwDqor4g`L{f4FEpuYgH?i0pUe6+hH{yNRtR=G1QX0kgH)dn z-gA@VWM%~2QX#znU+mL*T@=@v&B{d8La-YDWGrFV{t}w*l#8 z-8?eqS=B}mIRCXGtM~Uh!7C6jhqjwxd3qg;jmUmql_zVIzej$q|KOQuKS>LH_iO>! z0=pZ|T^wbx>dF+n`hh?MX4H4-%n6Zd9&9?WSBt>!g`QqQ> z+xI;;rbR0~ZERT1-|?FBAjj(P10exmQ)oM>6!UAl{(@=qiKoHbC&7ivr-yQmUkmmq z%*fv%Z@LqtC7oz^dYMobXqf)7$XW+1xInOVZtBl#^8-~= z&Y|KAqijRzdGE0*3-K*(A{E+KDC1$wAXVdylLr{zT1oub<7J-e1dW{R*oeDV#2M96 z&Iu%*@Z@Tm1%nTu&fH&(7Hl&(jI-qP51t$R}hJ{Z~{i+tbob)(Tr zZUAZs`y{LrcqY&RJoxQPTcft01g4pIz>Hn=OMxH&BKtqJsb<0&ZX&FPl<>jE7jDQ` zpwnujjafn{#H)fL!|FiApOcyY0DC+;zXOrekddL+Z~89FHeTykiP?athQ^tIZ3HoJ z2ULxy4orq4KEHK>-fM_YX*k~^%3nJbL2GECl6s7~5y(Q5ZK?wOnaIe^2~P*qtV6(V z1&;i}eS%2vHI@k<53C8*k%dEYdE^TZif;Jdy&Wb`4-~M5ix!&n4z6IDcJ zvt)%^3k3MK4AmT7z0dE|qTaldwnj6~l3bq-X|iAr?+Gu)^;NSbN0cIUg}S)0*AMg2 zYHjzT)5WyI1XJkYZR)zqDw8UAz4cu9Xg6dU*%CZ~>20c>Y~yD?^oI6%+u?H0VQKwA zy70#FuKY0~`-2uy2}&cD%wE4^Nj_-p zRhJ9BP%vMZUr*6p(T!7A}v3+URVm6+e?B9Q7i3|P)NaorWDmpz;PX(cJ> zs_kx9aqq|7+_0P{a^$`{LjE+~%>$i7SV^j45KN^Oxx&G&d5Tqp3mdp8MIUUmPa#(x59Rm$?~Jh*N`sHcsBBY~3YF4KF(k=0&)Ao=sG$!j6loq>WMrvGo4pt_ zV+)DWC?5$$VGxOIX;8w5!OZXR{eJ)bet&<>eeQXm<(@P5dA;s)&pB~b@8zq=k*{~c zo+b+Tevv7!NP6JD%7%AOs(V&|IPxsbt&!1pqdFp^TlK813HicpPm>MQ1F2%`LqB1r zzNi_M+VX?0=`=z^S*pU!&kUPN*naNY3BNQddunqPbsf1*bSt5Ur49S@8~<@K;caS! zHf8q++8mVo(EDf>o7!x-Y=sqzJiJt?>}v5#mla&JBMMYaHoB~asR6bYlOuN|h_R?? z&O~~^GZtRqs-nh?^O)Svt-~4TMhQ)eH04F?>z{1MB*r~YAlrxgsR139W;MNnuJAJ} zco#7P;jt*eaxQ)MQRs6ewODwL61f4@{Sh;Pg$_0)K>T@%p{wYHhgV&3IPNn>*Agog zd>k^bhS)T5mawZ}@B?Vuf=ntXvUs-&^Q8F2z7?DyEG9!rF5v(<8raq`BRp9wtK}

_m_Cz!aI|OA~=>rPyDZB}LviY`DTRyq;E+O1bb*mtHP+eDp`ie;@gD)I~c+6GFbPa%hM z`8Vex*~}cS+digqY0sJMuZM`)j&b;BN&8Bf8ycw7yWTmLRzF2`&mV!i;_!0GY1hGp zb*$&h%G&BIe^cNQG&UZZL;uTN8%^xvNkkx~^#*AkS2X%ziIv8gqo$-Nk*@_^rPWH^ z*L)RAHm5TNw>h1~z)`GS!g!lHyu<>rZ>9iOrAIRH!X2`(0Nu~%Lxif$TC5$#DE+cE z{ijLX5#>7=*o}4n?U~M}J*BAU9vkM+h)#@@4!X98>sImyC=SSCNgT*sNI%C2T>i<-!9=`VB~MoE;PLJfXms7b`3UkFsopktZsUu2`1dq zLkKAkxB;K`WB#D)vXr>P;vI^hlReihTzq^o^ujke-_P4>d&|7Z>G0neSdVpD=_A{p zzaXC1y}rJtmP2<8MZ2q_YZJL9G7Oh;K{yL5V|e}*m1NTIb3GA>WrghgOgWuW{3aYU zC!vPfD%{X@ANAJ&0p;vM@vCuDDUKM~vORWNZI%l6eB+aw;A5p(Le52ja>c7Dso?Z& zwJa(*Ju3oD?8P4uRoM4M$N_2sO2~Y$I{|HGih=XE!=%b(>#B&zHELo519p)LB}gf- zIcriktD7O1*bNvLRB?xUzAHNJL=zjS55!G$oTK{=ZsKKXWsUA>L407$9?hfeuNv~+ zV(7Nu1QQsdH@enfB8Y2~QO~5;=if?cz*gq9X|3Oj_Vr;ouRHdF_LpwG7$hWA?kw3I z7lNtHprmKTT;3k$nlzOWd^!OqefbPJs~VbLtR(+^r?&D;fs8LVlbz?b9l`FSq~E(Q z91@`=0oM3ougBzcJV0l?;+o3fAH7d^yD$I5@`-MzfvacD@$=fV=KQoICRXSms6$j*@>%B4$Zu&2iJZcpZYc6IalE1 zvefh96Nz{OLsVyVDL-r{ysURGx|WF#U5f9I>~y(I5`<}kCXXnY+n?H0FP$I_-U7NC zxGwSeTidqo))zxLP)@I5(L~*=60Ol$Z|zvxKIIeB@$eRugHua)KcSQG)z^+&6VTUW zGtS?*TVEaJklp@53!^@M0ri?zw*fJk58rQwXay8SlYr?8f8V)T5>yKz;CSB*aYb_tKPX(}k z<-Nmh>UaB*isssB>l(Sc?2X_1yb(&R{dv+c%5t+gBCN;0xu5V?nJWM1H61Xu#Q*ew zJ3g<6)$zcaK4}DZ6IW4tG;oOLZ6<<;6p{b;!^tC7(Ks^) z7)I|ml)Sf?8KO4675nLqP{t$9E@ObSbK$D%tRu=_g_8-a-qXAKb8gT2ENXawopM}4 z0`lHRiIa78$mX9-^xSbw7iByhx3cEk`BBmpZkY%zy)f+zaG@Bq(IQtnzo z%PE_dB+x4QTfAxUhdM?2aBnQt7!^jLP z6p1kMLr{zdHvBSSTdkwCAXC?&5(J9{m-Ddn%kR(4`PhTobU%IrLb8Xe#eG)?%W0Dz zCiC}6s*q#m0+iHJhxXXVNrcM6jX(nHy~;=~xk4PSZ&~V2j?k zG|`DtuOZxpw-AY`^ORuoHM0{}8K&Q|>4z}_GxXGN26MhH(*yL)Wh#Wq)~aU7Y+-t> z2Gi$X&&c{>T-F`5Id&^R_U(!2wJTKOCLLzNOV-BSUQ;j8Q_q&Bo)TCfrbifrN`A(C zsH8<9&qKAN7yoI|fj4+LZmmiVQ< zr)G;VNGNJ!3WxTKPt)_?T-;#uwgw5u2GX}-upj0;v5T$T^D>^-KKl#8xUn$h*i zDKNN+<#-{d5?`yhYH`5sJC$>we$z~cVgB&3Jlr7Xs@bI=O}lU<@hcjBqsqiK(ddWR zYH?T;6}Jl8x@9lZ+iv&Fx08o7jo19{-!6WPLCH=sPP5mqNwP(Pe7Qa@-c*=m-8&6YljhO=0g=sdnhY>(3u~b(HH7@hHN! zX_EN{NMW6@`eU4I(!C1BI za8t+(oEN(5)x_I2Q%qwX2%Ga>6go|O}1S`eIgR_1yGQ?Hs-gyHadT(a8-+F!f z*)M+!Jx-xzC>i(}?yZ@6l485#m1y7R-Cf2u5bj1IZk^rTLEjINCq>OKTR9g$^`6)* zr9)BhS$FoZ(+d&QTZ~+`h&Q(?vO6>Il=h8HlDRsrr0>_6OD&&gzv9_NO);lzCZ8Y; zlZw$=iRH{7R#O9Q@WEj$xOA^PfS3a>_!E8cF;wGL;mDCQ%|Kc%DHEo5d}1cD zd9eexRBf?fEF`B65$6Z>3Q1koOhDvF+{lM&T=_X1q^7>_Ff1P>l?AE0dR;LShNmC~ z_@Lr)p+XNXZDGu8g})2-Jq7hry0Tg?gDg&N^$nqJ7WBcLE6LH~-@}7>Bc25)q;?>m zMU(z~brJ_7V&6_d4=G+9NFt`doaw#pgaxaojM?Vx*@f62rL3DlsW{2CULK+K7og#3 z1tLqeluZc3rCJ1e?U}8P`xKTNeNolv3Z6F}{ zWeYeL>MG~?E&R4;0^cr$Wc|YG3@A#FrgaMsbmdV3bC}}Q$P@fl-zo{zxaBwS_AGkq zh5l*L+f{%=A@|J)p&zkGt#s9UIpjVFDi)!dk;Gv~FMr2WL}E7gO}COZB2n_I*t8Vj zl~Mg2vDV1*ulDL2MLtTP;{;dY(}*G>GCZIrt_Zmyhg|i$2r3A~uuAfsFH-hIvE{d} zc&&Z<1O~v)g+GgFvnx*d-7o$FX$$q;LtkiWyAcAxOL(F+0K0mr3qK5xu1vhe6A`Oh zD&31jfrychVu37ZscaUNdFcD86P-1XR;NfIWx=OV`q2?e8sy4sa ziLnwCyu#GvqAVK?w-V@l#EA~_=;_r!jb%*J<7SdkL`W(*(1!n*aYYNEX`-zxnAW;g zhsNcRs*9+1v@LRq1^c$V_{VPNgOIc8l@vbTdXU{|a9}xQ z1j!X9x2p_NmI=RgC}3bMC1@tid=-wnJef4(FMPWecsB5oaJ{RH9t&D)2u;^xYC4c! zOu*McDTa5XGpeG+iAFZEzz~t|lmcC1?pc^bM7XP#}O^uD@>2uHf zvY@iHgUC7+G!Du~M)<3e(0 zz6vYN92GBHwcKV=9C*E+{BCQE!>Re>8P6m`yiMT;GrqX;4=+9h6yc zcumctv&^SaUv@5ZWTN5r5yLX|cceP_gdt@WSE43Q*656Q>d?GpFTo^s~$(q0a!#*Y0^2DTl?R*d#Ly|?u@6<(g3mi!=$zFfeZ zv$uR~_T9qh?LQfRk0swkGBA@x#u}lsAu@vCyW-uelR1ZORH@y28R591A;ewXIxt!- z_FpjlQ$LCN$&0}W;@x1HmiZlhx=-}H6*1C2chKjlM95CX;y){Eyu&5Z>s*@AdtFn} zMCi$NlTn?0W0GAd;urGp;xO|Wuc2pVNKR;WDXOE<9|bSvf7CX(sp4EETTrb1oEpmc zOBM`^2Jlm_*`+>i5_+U#G2wpt&gMBQ%x5<8GlS+u`vrGAU*YlzaodXC-kWq0>q@_f zn5zMiqn8{>*#AD@W0DC>26`cvj{oli-hCX6>?l5MjfMU*;QyH$gE0WW`&~tyL1z_C z#zZrwk#?@a+?*z)mFq$h9WQcp93kMDOGtxP5rgsMKfnJI^lzee!T$^Tfk^zHAfD*o eYX2uFQ^E?}>e@W{JrCL6z=m|hvgm+s%>M!WQ(8m- literal 0 HcmV?d00001 diff --git a/examples/bot-ready-signalling/client/react-native/assets/splash.png b/examples/bot-ready-signalling/client/react-native/assets/splash.png new file mode 100644 index 0000000000000000000000000000000000000000..0e89705a9436743e42954d3744a0e7ff0d3d4701 GIT binary patch literal 47346 zcmeFZi96K&_XjK_r7THgZ=)=sY}ukdVw6J7XJ~gi6RV z#!d+_#@NO%)0pRj`~Lo(f8lwq+jY5I%;&wG_c^a~&g-0y1QR3OQz!UOFfcHj(!2YY z83V&nW(I~6&; zF(jiN^m|L+!Uf(&`suOcKb8H<#Jdj6-1?y&;5J~8X2 zz7CuJk}fVIaFPY~et#fWJ{T*j#nWee)9-McpR-W6OkCGj*gu<&Tv=bu3J1H0#ve0mwiSZ6 zR0Vwj+-m(w-WooXk=Hkl)m~qjKbT<&y0h$2gl8Qr#(JfoEZLZWVuB->i=`_OmFa@N$0#y%&3Gs?}-cn2#GejXLZ(_t6 zc>YO^T8Mc*haZ7l&}5__*3NNJImJz2C5V)Wq;~DsRz@FNxpJ509*pVqDsJ8* zjk&L{KPH`Lw3rG;gvEKuLm-f(4zCJg5DN}Ma+_oXYAU`w>C5i<;R_(HyYF>s2ZE=; zmCHdYmMwh~_g$MJBJD)l@jL5tREr|(@{pd*KV2RJ{TBBh02iSWHF~hy8{YLs_GfXQ zl6*S=X*Y;>9XVHoZ#~W|u18z$o$?EIXrF1sL57;jH)?ge1jO|1sMZqWFI z&$Ozre|eSx=*Tw=M{OA#ORXu7sKVi=%J|c#%44Foy%@^6fnLKynVqs^A zlblnDh40s(ZrIq`Mi~me=IoJ_&YT5yWAOrhlZLC?@$&Ez2 zgsRNCj|U=r5BAXOQEy|}Rn`QkcLjg1jyR@bijVO9Jg|Wmi|EkOZH&D?AsXue?8ZCM zIl#E?x4Xo3&q@B`K=0lILFZOCH%EY8=LkUJK}FVrjwYGieu)d0M!%Tl?Y)MgL@Do4;Z{ES-&>~<0JurBK zBc!EMyhbWA3;4iMqi19_4f`_iXH}wn5;i7qJk+Nid`S$hRo-pufjAQ!@4AKr;@nzq6|GT9LMxDfqA!Ic^)H5#tgJKB z022aBPRC=Z2(Pv1W3C39_G+(|>%9)||2HYWNwFX2_igh}J)rGI&J}n{MYBe9mR3Mb zO?kW38JhomIMD?@;1eEx6U`AR@=T2Lb;#sb|KyB}L*+~K4b`sRe%dIue@)zmN&9MY zfQ{NYAnds1*9U9p#!LWGAlBAR6<5HTXC@H5ym_xx^=ubJQ>>NF9h`*Qxg`JuqB`TN zfJwBfhRRk`fOX1o0#WEI6wR-j%cfY55u)ZpJL_$ct3CC)%aoa;v4=X;mq1#6l|a(t z#vf;i!({ARHyj5A5c)cgC-@AF1_IH`uS67>r|1zoR-TU9OyNly`&KKK29cCRE1ft% zUhbcim?=N#!%AEWSRto=0%1vt@Fwd5Fmi%f{7TPsXyRMSkQAc*J%2CQ($fETNRP3O zH)_JN?DMZc1Wt8bXYMR;r#`oBHLEI&Cnt&IO7j#q1Oj1+B~>4Li!3j1y{DZsA5Npy ztkAXdEgekvck}ank(^Mi#0AXel@|u3#aY=)c(-ZJ;2AT^=>mmfMNiH}XRu^c^CE z_#36;m87NTl>iKpQWcJwjRVzF-T>P1_I>_cf|eH**jsrR0*{r^QH}o7_^-Qg_w-x> z@amziZHEEiN=?!MIMMB?nPFuX=VUdKVXS~J!!Fz87la`b4fs(tKN_)KhnnDKJ zL6|y+lLbVmuRo7Zd>c)CuO8WyD9_E>x1sUPFTq<{M-l*KiNSI#|Ky<}8z!=C;z;XC z-3s6KF;KyE4CYYhUckd@vsXz39MN&Nzc*>4l;Heu}k4&#E ziWEXPF>{Z4g2xk3J$t~hNhj{@y$9`!Q<3kapFj$vJ7pi~Wf1@l7tIi7rto=TMS#A( z5$iv+3j>kWVyM`S|LYThFsCRIen}MguNOw z%gl&b%9vj!xZd2cud^q<@&$d+ynVT%J}=);^3ztikO~6NKrk#a$$PpnL|l(A;cK4FD{N zi`57?;U2xi?T zBf5&)crbse?2Z4@H0L^8D>s_{X(|}H5~Dn1+XQF@gE&|2++Q4GTX52ExHed!L&*^B0azpeu!a9XuMHX{b&M!monL+>QR!DW>6J%bs#d@QG;{2YEo5Y(^V;Uy z_b_1qCEf|3;9iHmuGY95K{bnX7xa3=-`mF=o3?L4=9R3>c=4mL>B#bz{#SeUWZv?0 z=KN~};zrBgYL+nvThul&KZEWEVP|W-y}cPR2_$}&STL(mApmvKJ<~J$X4q5Hs;B)< z2zC8XG(ZSDGCX}5fI+FWsbTyn4H4;{n*E!X?ij*{AgF!A%UUgV1oP)^=;?8qoFDcd z#g?mHMJx1268mZ>*8tZI!nW1e(wyt0RIhQq))G}VpHbmv9WmDVzbjCy6uC=K50C!o zxBqxI8B1Eug2Uo-5W8pQc(QliCZzV_k$0E21Cijy@@1e0y+*e3pmvg03@y@ zE+fj^8~}40LIFm0nzc{EFT<6d_O&J|>Cn3Zejru8I@*CU^eH0N57pLmCBh*IoH>uT zC?0Fls%m#o$T`k@U|#_P7TDRmGITo}Oa!I4S!Yg}WuhzHt#?lWTVTXkPscN2#-@|7 zaYccM>wZ80^r3w4v5H|iBL3$~bHJ2cX^@T9XsLcgH(-OuncX8qPB1IU`DssCFag%< zmTy(5k-doKxNl7aBAZOWIHvsSHElqkO3UYNb6QpKWq){AF}YAH;H+nBgeB+{b1X2d z>Rfn!yDDJkDGpl}#fi=wgd@$p>1&lJ7=O}{Iu{E8>Gww2>(Z0h%0{}|+DPWgk|($2LaYkVi1EqD))Ngy$!?Ey_Khw=N$ z0*>LrfiNG=fipoI@PGEb=ZJztU+<|21z=DLF=KlMJ2zm4_5;FT06CGWu2!NR2eAwR zbOz1gYQ0;g)<1&;g4q~H!I!3*&s`CKwL$eom8B(_m6ZJICl14gPoJ8jl?}@^^A^>C z$e~861#yJ}o#Dr2o&fN$;e3IDk;as{y1}~ zIOpr&NqB!Ur0Kw`xMjG`U-WdQd6b&BS}Fh@pT4R_q|LwI56OVz8UNp$R8MF19Us&3 zS60R*XFAojP3f&ySju?(O`hwK;74Q40TUAIfu~u3=mW#u2Z$$&fU9gjf6EtDF+pfI zR>(O(93TSF@ii1xj``j9>hX;IoPT)!a(VCs|EE#}zT zG>Ep-VHUDPViBnX+&5r!H2A=Zf#{A>_%w9_&BuDp0?Wfj@Nz(4(f);b>UE>5t0Jh2 z$iA3GR1smNAj@*&4l?7<(jttw(tj;fIEBhz@8zJ@WxoP=+_94^acKu0J^L4#Lr{6` zEkFdc|1K-dk61T1&WjGD5P3yZf_`6)=MahZtlJ`IHP|4tT&=f{4X_Kr?eoPJWQ@7{ zH3d;XP-K}r@%*B=efZB$36}2)nxw|}Q~3R;+dd zxYETNK0Q5X?@07?y`&@!PocS2=%+>6QCi7rv8G9PWCo$re7NQ$0+P!yW4=1~ zf)8K)9CZ-dT8)EHL#(%>&CZ}J>uq+C0~=8R-VxF6<6j^^Kn$U5Hej*telk7vNy@J35f3j0sxz|iKjNS&DRS!qyxgn!+Z8Zkxmmn{TMY=RYR zk&-3`y>}nv7qA_k=o2j@YU$D7p>e>SVObgt=S!O(+6$)vnL1H=8ouhEK|1M!Nh5UiycwGz<5I}w%9 z52C4Gf1_2SWzuYXN<=1aL{z3tldZus3c_q%E*)X5cjpEJ{yeL`WW#^VFKxZ#iqW*9 zaH#Xid*onzn87_wn0_4q@8R-(B$r7_py^gS|J?Y-Ms==^%hdbMQC{(wZY#by=j61d z=*qO}>s{aYR4u{ailpkG@bKO7^--Hl`gZeHggvi|e=-K&{fn=t2wAbW3g<(){7DT| z>)PbQxg@8Zouhrc9ju*9pX-m^v3=GbpDu1(+Mkr3m7=Ni^WlBk;#bE2%F3c4C{H+= zrKG5GlQ^dPz7Jst)#1n3j^&{FZ28Dd4>CU<3uRt4OsO+)OtTv_rLS7tx1I_<`W zn!!jH0}Co`PkJfZ&l}Y3DZs(M!>fSq+xB9HHLT7cMBw=P_&Jlm z8}q@G@ooT;*Zoj`?q_Bc+#?Ky+e5{SekLaoODCd2>J%FHoV^_GIZz*%S~w6$%X9@A zjc!2R)GXEeqclipA0vRNLw~7`qs*uwnWx%v^JmD*5o@$9vdFvcUDJqEO{28k^sQP= z!+yNGwyCDZ_=R!$P>=&GvyIGKG!%A>?is|YOS4?Ux8HRTsHoD1(fiBPZ`$yHMEELG zRbZ--E#kTUO5VAIy$e-Wd!`Gw{&1AEi%fo{=Ih`O}Q;qlcH}(eQ&0 zqNA#@w6rAQ9XrRQ#n#42WTxso%)h=Cw)zWOIq3bTC539HuC3V;(M$t>VMq1Tor4T}G5vGs=!G+@VMKa(@=-alVmaxCRLy*QT>nPvo+srM>qhj; z@q*&OwPT(>)MyHYJjl11$LHUdtV(qeyr;Qo#oyERe0hVkQ=%R5T2uJRqd5BI6en0g z^tM*AcNz2=yKZ82#f_6G)PmGN*{%*h6gffu8cc0!yJ(3jqBpk?KQu}UXm01|wBmR1 zN=C|cby*3x_$8y|Sh}qQT^=O&%ITDLM@QP>IPQ;)Lx#w!#{KJU@_jR^?Ak+CFw0~z zS6J7MNCDG&IA;Od`tIM++Y9S5t`|PrLa4ndb04llVSFZCi-wP1bf<~5i)qA<6R?O2 zVaffa9@g8rmfh~)sE|(g(H|Z04ss_r5m{+>I(EJ#J(7*)TA%}+&yUoFScNsBC?$9% zOh>$KjAQxA#1+nOHFLP)iB?51_v(mZT;#&IsVJZ1+J=A&b}H-vkRH=^phXowiE>7VLf?&+C}WXjH}A+Oc!Ei^B4tQ^a0 z8O~(vXLs;6l8qVfB+57UjiMzReRE*x*NouN*m>ZjH`+h%Xm-UoCi`=-E`&43Vv8gt zcin*l(qgq_yS{B6ja>@Ykhc>JTZ!4xHZljM*kfbDz*VZ5qwV;pdxM!P1S zb`y3d;&lmI4;#4BP^WeE>Ch1UK!a9iMn%7+NOu%(cVdc1|BQWWbW)(f!i8j8YwK|A z*RLLk^@kJwPtUuWszvUGxqfbxzBW>spg8?jaXMD;*1~%vJ5%pN-#V-`W1m&Nn*X{N zw?fX)o&pZ)J^2$VK%6lZKo`uRg^26xROp{QO_UvZGIPqKsJiGOH2I?3yHBIn`CXi; ze#CLooN=^oswLu76|OrNN%B~V!|P`?c-(w9Hk=eKUxjt-@b zs!T7d`pvERPC8HcCy&X6=&CB^qpk_0t>aNgbgh)^F{o&PwZ=TE+PV6jWNUKx=HQO@ zND~25>TrGU^|)j1T2fzBS03$~zDUeREg-_RzXIk=1y2ui0Bmfy>dtxgAJ4q;rz&eh zw@x2@6bQuxdI$6B;AjH%B_Swi-4rr&+&Yqm!%giCsx4X|-j6vWS~R`h`xAZzdXw%P z5@*KcoBdrOtpI`pq?f=G#UesZ)`hwR?y#)!u{#}i6dN|*qy;uAsaX7)z5O_qD_`1` zLt4s$`qpqW$~-S$nfn2uU}yYi^xW3Zu;k9ZBDRh=LzQD^A!9@CcRmr=jw8a5frINM z1jxTJJ@b^`dQ+p0rPn?qsLwV27b~AQo&8QV((Y)Ommo!ZNAcv3vklt{d2Gy7Dym#~ z?t4Jg=?BBEl9v1x4(i!n?YY#xDNk#v1dx!+EjURA&ToGkV}@&fr$@`xSt&|DgeE) z!4{a~o?`|3OCiTM)Ps8>2IYKt_Lb=RZ0AXO-=Z^1?Bb1+$IVZTATPCk2#{@%2^F47 zfO?}6I{s>&a&AAQbk6rI%Y4f0Q=Yc~CeihHxSjKe_blVJlT05*??rN10?$G*Hc zC{fPWv$yZ$TA4Ns_vKIi^7>#t2YRGhVxJY!v-XXyQ5_-s5z}i2TZ;vs0y5PbexyS> zgRFlqxAzgEvcT^yRILFL>n*%e) z&JaTI#{bK>?t!o~GCd$}d_sNBwYmh(D<9uj8?&Tx`z-F}JgOZBlFW#}UX0=6R_?g{ zyM!X>*c!p8N~xp!sj_UXz5iM_K)Z?p=~W4Tuh}{#b9+Nf-hnai?8iND4hmM*R7*K-qJv07|pE=c%X>~gyg%LyfGR4PQ zfl2_y$*{5j38(;Sqm`0;z%Q(D;{l3*sO$N_*I6C2c_+6~XV&MI17yS8_jg0m(ZR(T(%gmGxaE2r zBc{4`BEg-NWrE<`t`*P_DA^OC+4t};6)%S`cLVdK%UAD}d&zsFYU49AYa8%PM(&j? zu`XOEuSo@S7)9n`M($OA??uENlmPM%)%D`X8~}H%O}8{k`4@Q$r_EF&H$D%nUcEJI z0QELL7VA#!m*ra#%vR*H^>KwQ+Tnn;`~iBy{E#2=a-K>@i#6}ixbObXVjp@J0 z8C7u(b=p7df*b&p@a2Mk*!7z7oe(eM`_{WhvC8g+c7)vRU!wpxTSl()$E3f$38c_F zv26-aS>1&~{{ZwMK z0=`D$mRAclD6tvXSbR6~>tR9ZwG|8n@OD5<>@eOFob3jhbw*G{dL(xXS({!ntM1dD zWtvksFLyfeId~CfaDrv-k-*%D$D~9LC`J@ezi;pfWLtsQ2rPdQn??SKFNgp+HXD|j zt4D~<0%`p%QDrnMa}ju|Rk?9A$4g-SqrJU!_9BVw49tM0C7lGO7+v|K!iZ^q58umY zV=iq5&ptr$JBSAejMe1u0@&m|f+nHlKxPdF z0GDfZhSWb);4sBj8Cr-%%dop=hk#}y0OpID$rC#i;WwkQ_qvS-8kmTUja>fle4tTb z^v0n|tOIvd^!7cybZZe8LiHB%{W5BuHUb>=1vRvuBp3Z1*Cd`ksKSIcsxz;?5_Ky{<0me8J5dP59-XU8^K;x6J zIFpHkEBj-gPmTtl24)A)bi^(k@5B{xU#?W{$EC+j04gd47*xB3d=e5l^SmezHrWGt zHk8d1Gwa|!wkmi~{K*v`iDPA^zmvlIuQcEq8Yjbp2Csf((=F930f{P~zBTk7@O%v| z)FPpqIqHGM*qc>t_23Pdjr|vn63v3>KJuV%yk^!O^rwamaupg$FiA%KhOp_I_Ai(} zE9z3cqng@LisR#WF88e};qyrnv-M~rg!k>p_M?Rz+;A1GT~@5lSEX5!?RB4Uz|D@(o11})N@$^4&|TL+fge#G#wrGqW( z2Sen+t-%~fjuWB%)PPN>!Mk-zzxB2=9;< zvR5x>VY4hax|De1Cwpew%WqvmPDm%wbg{3n;^mGb)Wgm}n0jGD-C#)3KBIqHvc9dL`a1jCG zNYP1nRk%~&&)^%OolY0o%K^sqk-A28s`nAar!j%(55UDf(daX>I?s20cI|s=QWK+W zg>=}vlnT0%mp;Ld>d^v`uCLwR@y1tZhb=o-h}!xDllvcXHe^7(6Y(cjcT7w~fuNTm zGR#@s_6UwMN}I0^G;z28i6SX|^9-woIP>JVtn_koz=Fy1IJR{@uJX>Z4{X>rz2Lle z{+-a1MDMGSSHLLg*G>6Ow%o*T_?z{-A2CSw-1tJrP55{7T4A`$0o7&aEN)z$R=4SI z#QKQcZ+@ zyyQp7dJ6vU={u^ClgmW9II#Ug7L}e{9A1{j13>up%b&#Bz6h@YT5F z)M6Q!atd|S|EEfL2b0AGX4~vErW*@o{--QC{2pY?ce1j`fJfETo=5UNj%_#zknSHc z4ayf)IekttWwl^CmF0q4?&KP>#FRcgKP#Ber&>iK%zX;nng=Xz3ss4tovMV2 zKL!dU`;pZC=+KhhPqI~0)1h+t-62TM$-g+myaI1VQq260<+u6whK{ODf}`p-)3Q|f z1W8EBmn4)B`sSI}dfv{1q--fFPlJC*pI&=`eKGi$h>poe-YeAzuHMRD8fFHfP0Uxti5?gZT`?$d%n4d@*$8H9AA~n z%G!QbV0LdZnl<8JbQnd2gm~OI`R!eMpJV+iY;4wbPBk*W(n+|nFZpUuWWE2sttOC& zhOA67>s}?jj}@!c!vb$ospvDzecm(8vu&>^)5C?U$rI0Hf<=|1p{EKR6^sktXmJ9U z9`far%E#KLvTIu<)6L4>9^44VT>E~%Q;dt%{=S}?d3$Tm%TQeXcSMz=eDymtS_bge z*;!1!2j!9g3^$(gB|O_oDX+1mY83se-+%nO+fz_X>Dkl@wQ2|zC`+Xg7rwiVI|k$c z?%(KK^oAKrth)p5>5t&;tv|^SRpN*JT3t5VX3gNj-J!A;Am-gPK>&R%o|Z@7g#_4x zA%yL=`n;#OX~?qh>*ev-QwXg^*C(@MxQywC0_aTT^VC5ya{R=8ePZ;_C(2-D-MRc$ z)kP=A>@(vAwGsi1>S650zEjg}_0&7L$HhrTCx;fKIR)F^JvCYTyisB|=G7w$j9r;c zAgzhUokH34b#H&FPPv^s%1)^SBLC(r)Uke-ndVEhU61X*IxvC)!r$f6VjMk`?RH-X zuU$N_YUx*24u5!JQ^Zfmgd)Nx%v4YKE-yY-)E(bd5xEfA`!oC$pgBcOszHyZvflY0Kj>}fHZ0F&=X!t`=yYtwf&CpMo| zmHZR_A^bOF^Zr+FwrfE5K+z^YE4zd4(8%8W>J0uMsEM;pObGVLn3O&FdX6WUi`C7V zMqb)AZq}K+rLON$Yd?2Hs0il&8p#+0NZJl{+PQ2ssHYl=h?t1;_D7mLiM-*`1^TMxcaRFS*`q? zKza%+J9OtSF%4p{q`)HKuV3g9R7lR#jFA4DKKF%Fj7&A?4ZBIf>bIc#{cs^4K2g4b zf206%n$V*ar#~idT>ZE?hzfxx;CNb@U7FcyJH|2#* zedq+DqzYc;8K`%u0E@S-l18x`z-3}vHONmvso0RpZ0rGq^ofrMRMg}S;aPODxo~&9 zRk#|k%hRP~g9((N#Ngo5KSGJa4MD&E3WT#RT3+ zd=>Y;!=H^6ADQ50^{WFZH_Y|9NQ*s=i3d8fej6Z}W3w9l2|)Q%2U$~2nIC-6@cqn* zzPZgAk0e@%uh7WB(b>gEI*^YAgu3M7Ax{K2IB$;cb~pAa*Kx7hkGItesJHuT7fk3K zOF3B?7siERKh!+{Hjz^!O#|Q`Pl_aszd=qZs%_o3&yTxq5v#REX`B(W+pp z!~3Wa;>KSjtbECP0AG9BPYQQ(8RE{f#<6`$z{p zip5BF-?QV`HeghMIUkUqcv+_!Ha=p^}uJM#qoFL*kWMEk2B(-M99~WETPI zC7H9ZV)5f5;ZLr>6RE()&$~vtJgj|gb%{NCRYO>>xwiT$Sv6$jT%3-XLw+f)<~tCp zt#&-t5x4TEm9PV|I2wo9{?f9MM|fM`suK7D&-`n#Vc z^(=3Tl8m$~s(4~Xh3|DMQVKUcOb8)VsyQ86Hw z&3xIUL{9mU;^brYoV+yerP1bU1pi!`!oeharZr0{X%vG;o1Z*LhO|#j?Mn3zQ4k;3 z?tWgzI@R6Eg2;*H_2_Hmd6CH$MBb?ObkH%yi2NmdX|wfuPfETeC6qc-1RfZK(X&## zLB{1+d6a7H$5qBv?}zl%+L^sSnz@u;LuCaeZCGmXP`kNTnu8VEeus7gm)-JV5A44d zg~K)EuWgbn=wgdRNWU+@y7hF9?8dG99x7`W$=;iJpTA}!Q$AB3lmr|79q!jj)x<6> zS(I8JmT^n{1)s7rfeHnTEK*#(O7;9k^`k`cQxpAxqM3^`zfAk{=v6$Bug%H3MPKfx zI;6_U_k5Kp9*@?j?=PW7%6E+cy&m`X3l59BvqfbhnlJpQKep6F`Zlo~@4EkJ0sWu_ zZF_BeJwWl(IGNxn1(Su+@|LP+^7Ffy_S;C7@Z{2Ja@$tZeyeM{WW7=-&{a6(OT3%* zkh<|85JE|Ax(rR76m(h}AFuWQyjd?W_fT8|_OtfA6rB*fUzTw5^(8E0u~>u+5|gon zx4b{*Z;#$@P2MrkpNZ^j|I^d{$BELU33Q&y=oi3b^a$GPH-FQCV*exbS=P4S-wW@^ zBz!S_9OHR=J6(EUE2=VC8`HaVzej_q{%UbMf#j`M~ku3Pvnc{6qE1~Hi-z-|XPBsqTY z{(9k7J%`SkCC*#K2uAlXJtJbw{mHmEVW|`hzOaQa)mxga^}J5m1^TRR0|hniZQP{u3} zbpHB#^{OxT+EyD#yY~GtgeW22O5cTs=GF+2MO)Vg+X;E79B2+uKuD26%y&cA*PkXdl3HaJr&w+lKfe^TFMjH zt39gBAa2j+kA6(hL_taO-lckx(gIp~vv5?q6s|4TkD4d17%kZ~DE}_{MoRn4Gdab2 z)|2gm?LG-|%2UKe9hV2BR{)DUH05{B=|{KA$|@NrT!!c7=$3hS;Zm}kMi*tr)i{|3 zG@Uq7q{3y@M^p!0(9%64)BNpHiT%l2H`g;+S@+wMyWD|x#jm-8?ik|s9fMNi zt4klg`CV%E%qhE?7b%j{NY=3mO`J=8cyZ;~=69j!=LP)v6@48Evual^*jd-#c-SB5 z4u;>q8W2eBObf=r+)KQ^=RYJ)O4ha&JQI2W0$HnCB5jvQ2)a#A>+R{5hTE8j{vhJR ztj{v7ztBdvZ-o=n9iEk;ZXbAUhRAE2li>3nt)^mnbB-qPtM?f%b6+K`>pO(cXXtmx zwi-ytG*4lBu#5If%6*`xKOCgFs~;}**%h^|<~5)r@|+r#-Y1N;M8SMvoUfZq;i`h} z0ZBQ^Z4e2K`wvRRf=scq%JLT6A6qWVzx3h?MjOL*DYQLm$&34Ege!D@6k6mYBaUHz zZ8(wCg{R@dCrcvM%)LJDJj;0FWj(^!v#Z<$tJ&{G0iIFKeD- zo9C4}z5Ipm+*30eiegRLO)KjTv*Txlu3o&}_0>w!rQ*+q4xB-{Ckf7gZ3oW@1~H6>D5rd?JwDtZ8MQN#3S2z8*G=##Inf8!YgG@E}kVt zKTL0p|16Vd8yXhJPc4FLk=g=$OSx@tz)x;XpC@XYox5`6O+`5$$%_f4B9&XI3*pHF z8vf@aS&gdw2|U{5QXk}~E;q-yrC<2|p}&JZe10J}Hd@tm>2=%wOBf7V=jMh~u*@yP zdL;u#g!JMc2DMOw!%`E-Rh%S7`{K!W5m=gYuV*Hw76)RgN|N|ncbp{*qb-_>xpEx z*#^&o>x&~_$~`{Z_J@~-*Q-a+DpknUi-9vAPU}k?XYSdShBq#+K#;CfM>9?T&~HbD z@*NPq*FH@bIH@ZU4#+xyXR7q^D2fc8U7+oPghOtNS~d7{jSo+u%-GLa%Rru3))&wB zx~``EvkdcBqw?TNc7tZkOA{z6Y@fHZ$9%_+FVFx=h_$;4BmL~ zWUXRj67-+w3)@!-#W)VM@tB<-)ta%fX-LJl1}PWb3qaq^5XF}M^Zf5m5oO*o%Qiw* zII|yejF<@Oh&|YK#;g7hR8K#?h9*5eoILL=^d77Me8; zYHw4i1FsaN3r64mS76#=BhBDrVyoVKLdCMX2dmUTlU(x*w~#N*;{`MwFL_!&oQAR= zq@6&RtTmkwj1XuiT4wNsxn35!R8wc`d-+U^qe1%`4f@nc$RqUIlMtLr>lsk=tL|Sm zOXIMWt=H)~{WsGm0T9<7PooZX z=2iFhJ+1xmDp<>S3Cv?C`wb4>^ZWVfzB*M1z!QSARjQ5D42pl8C@QAHCEri7#msJa zcFC~HYeCkDC+hB_sQ^q8E7h?U^tqE#a>tecX)jP zNadBXm}I=pGP*sE+vNG2N&z=oSOl(FzsVvDp zSIPW!R*tZ&CFdXW#)3%u=^;W81yJZF#Xr0Zv@ADDVFYilh zp4z3S5#9Xi3lU>9mR$CFw?h9f-WLl`)M0-;G*+?wi=sVtXvYl2pHDKo#3^ldiV>R< zfZgF^9KVRlo?y7#nC@B%+D0mGsQ-%0I4)I0l?qF1&IZp&n5QUZ;DRt6+W&x7w$}Kk z<|##9=Z?74rtiPhl}v@MxG8YHq-~Esg}yamz0wm{5-T%ThpT}~;-CnkG|w|V5PV5L z!CkT{&qnkLHcSo_Ye>AD9n^T&%tY^hQs>6YZks$G6@B-kX*Ci`EJh!EV5X|Xu_o#nO9dHN$TDf~W zqi=8;jN`odF_4_%lH#G!p{mt%N5mP>(FNNOfuk`Bk8cG(Q8ZPs-hUy)_3oT<23xkz~DF~cDVUY?!ftTH{&oy z#P@x`M##ud9kDr4P#JMBT{u7FA9Jl}^5avjwzrXU81`)n7!nu83$xz449Z6{;^C~{ zCQuTv>6>x4^2lc=mmxnaC}6Xl%#a#lko}xo&r=sh*kKgIAojO>b)TwSLFRjvsvjMk zLF~**2yxn$#Lb=px1&~r54Og~wcs|Y=X~ERo&G6C0S}}@OV1N)ocaFw+qAXsyT`)~c1C_baOzO`9u)j$w4s0EEqlzY8P48d=0?B9 zz^@HsY-y@I533GMtb01P2YxCzOh}PO5tY2-^;HZJ!yWC051cz2Bf4*M43}3be%?Dd z!*A<6w&ireMFqs__9RBXXF(210oN89j+}NDx{c|b|2@RP4B69|V&~PH7XG082J+7h zi4pRxPyohOr?0zl@ISMrc(y4MsNXMheq&|AL2_2oO3ginUO?r{x2=6t&iK>-zAXw#5U`J1$w_m1&Y0W&eWTgru*H9Zlj%&9(iuQkZmTKf`u1-8Q8!3RDt z0fM;llQ@MsR%UJ^0b$|=i?U%-;-jPiwxS07u^h;?cJAreI(zpet z?^OHDU^qx47hEZI%D*YTJBs;dUgeUsg?lqqi^xys(*NB42T@rclS9TRi|`|Fxc(1;e8km+Isqs*feghdk1q+>5F4w;J*Vg?gli z{QX%m`z7-9B=?=BCA}2;RYrkLRG=Q7=dWm2f6MHlACocSN z0_J)ZlVWd?;Xt~Usk=wImC$JQAM0{2g1~YTj;(?xJT{Fpk@S1#`E+oq&2(m zJL}7hJgiTX43EVY?eTFxRg@R|1d?h1a;twd<>mdHJxy=WsXFJj_xKq8U~u4N(6PP; zGda6j0g0ek0Kml1>{%x_J9VPjp9YKiCD#bjm19KrWy)}QONxFjZ<{Si)8bB=`quIZ z-_vBD+#kyyOe3G@x&?n(vjSq|mY)SFAw02x;!uHJ=3zZ*Vu&H#;U6WrQs~l5hxeSG z`oyHIvJlJe3xbI9J@oikZh0)xx{_0EM%)F?jHs}|B5zj#j=qkfeQQGxXl4CJC*&fw zMe1%kS$l%uKB`W5x84uyV!}NBij~N!!JlPK zrM%NPmh=g2l-UxJbx=V9!b6YH@``Jb+nof+yPlW}Z!@)I-TME^%ip}TP;xt9Gx$MG zUsZD-cXH%Ic7E^En#Cv5qM zh}B^2Yhmv{@3y@PTGQ9o_aK#XCL`>97f5`#J+IcVjDMg$_B6-(caH*DJ0rfcpm@dO z;!TPn0e7$qWw&LQ0-nPurKvHFA5ZVO8Sxvj_Dkbv=P%woxH)aHv8TaWrFYbVG@Ptf zPWp~)8}CJt#@egdf%1Cd)TC!ylHP5Rhe*Dcn5t7!n|Mm?7!mOx$dtcz;+`u!bns|%!{AJs^$fNe6TAZcLddvl_?5(4<+h)~2@j1w=Qi2IHN@G&(t%KSvAaBc3nu4#X@iZr%AJNKc8^24S< z>|!&U8~v0+0cmT*;#EjUiB92Svs>EtzpO8JvfbI*z4>^*n}*>Li}+}-MOi1<-cxa` zQld^zt^8IIlLcJ1f^!RqMOxKLo7u;|D{u}&lmEpV(L6ZJ&FQ!=sL=3d%msd-H)c*mz{Ng`Q-+0~(SSJ`#v zPk-f8D5>rgbMTCNT`W!DAZs5r|7mRCEA|+2ePv|&I5SzNWJpa|;xz4#mz9pHevG5} z50d@y!GlNNhsFv4Z#On?Rey~fApD*3HS;7fhWlwJSX9}aCsskK2)k{aoe&UD#AXkjjCztII`W_hw2ng`zsRS>dYVd8> zqtSl;2-sPub?>)-yGQl)8btfc^0iLM_eu(OH+_};gNQ`$)i1l?nkpjW48F$AeoLY4 z^#EM>G;(>gaa=mx$IWSX!=aXvFpa&_GX({G^^$9BDwc%8%5GC|4s? zwHW@?P+Hmy*@LXT#Iy8&nOELR4{uYf5c*kwh?MV#y4MGe^j}8Oe}%uUTdb#Uw9e86 z>n(TsJ=30(iQyVbgqxR1DRpi9soz#v+4Z}2Vrr=;B_}hCc)~nC! z7HzP2&3?SnlKndpr9VPl4Cb>|)he#sw|3`N73B>Db#R2W#>VS5b^tRqR(!aSH z@_H}wqipMtJZ%CCn}JUk_?gn7>8-p?t7|M1_UJzOV?+x&w4Sn~I!qnoneroVgs8R} zpxx~vRwtWK`8OXfNH62}mVfEdo&TTq-uxZv_lqCzRTQ$lNcN?&z3eIb+G1ameP6Th zMwW&UlA@4(4cU!-tRpExBHPGVvz5V!7>qHWn|Ob}|H0?FK382=^#jkD`+4qjpXG5L z=iJ-b*z=G!Z421q5&REI?S^)%;u7m5Mu3xPtRIqoQ|-bLNN!9F`3_ z+62asA^DiXkgkCsOD{d4ZO?(EfXt5t%Pywtz7A|<6Nr1of;ZSz>WA4`cwAt##5o#q zhnL58Cx>7l9%RSf5SX!?t3)ia=X9YJW_%%f*{%>6p$FA=hz$Lv(Ux-XWoy6v9)_Y_ zH}o)TAAW5G@~bWgvm3Tdfhd~}rbIPhDP}MVj6@N_W!U^k41Q zb7r+iQMdFg0H8nLj5gXm{I(UAo1Uu#{!z7{CQ)~YCJJ{+*!k(rQOxZMgt@`*BDzz5 zk7JzBkUj|Y1`;N##B=6TeI_ zSqP|MBflHCDPf0HheNY>OZgg&D&t6_O{aDZV zlm**5yS(+gHCej4h}=_i8vcGh|Ih$Xmfrgc23PoH@<5tW-lPN#1f&4Ozr3>2k_SUq z^V?`zCY+=3K`W7QLuJ)kJ^v!T(bW3NBF$=#aLqzn@u-VhBo1Y7Qe~6bc6SAsO*RK~&|2zq^?ClMAp7fEjk-(&lfU~?pqcbByph2GZOQIbv`_^-3J?C^fn zwv_&p`%%Y6KlO$warh1Dgi%HkAxMzQaz$vrE62ELOhr0MBPOEF%s=4R17~&;m&*wTmq{v9 zg}dr-zFTAMOXAe#*X=0bB32`Lo(6~JcJFnzP2I)3g->Et{p;V5yiXFz%2Im{y|X6D zn#pdV8-=cDWG(qqbujI(6nnnVE*X`h&a7jq=?y-C;c_>K%yJ6LYIVho3^0iys;|p#WTJ5r%Y7yFH{Xs|PJ~V+e>F6`GQPGRPw_f=Edo3Y za6Cz?Fl(ed1FrVQ^K+xyf^FwI&X+y4>*B{zorFf3k{uqUe4dxV!%gM2aSlbzX@E$* z8`4~Pf2P#$`QVS=m|Yj8w$i7^`!YC9p2^XicR$#GapFharCOma29mCIh)G9{0aS;v zG9=Ki5SA9VEqfB~5&zJCjRcTr_1vAZ7ORw<(z@Fs9x;BzuOCRK^(hWMl}QWUgi1ij ziDW+)|58Bn}5bnZ|gD%chnf2 z{%2=K67IE>ab5NoEh*Xq(5P1|N8)_U$9+JN<5Pce_X8$%rHwz5E zkaNneKm7|rlKrxbK?+yX>3Id?ya&7WO8%Sq0=&>=$KCf(DC%e zI6RL<@=xyU@1;FGEs!VTF?~@fYZ0~6@Fgzl^57;f3usv~()JEs)MIZ`9l3d$Ms@u7 z7CN{z`}m0*1w_iZ5#%91>*k`89~e3Vs1{%!d*fc^W)`{?W*n)0@4fEh%(@JmnBH#j zoaT~0QrFv8>NF)nNNd^Vj4krCR(1e4=Rkr>k zRd>Yrhc-@wul|C|fu~Cl(K0HNTQ%k1xo1Ijxuo_Pf8|*hkfb_7dp4G)!$Pv6V>I(U z4aV4+LFzpEg6eZ{@|Hjt$B~wu;Zk)P7B4rdPdnhz@2e-DR|J_oNUQxCKM5F-ehG@4 ztt&kTAoh>AH~n$$g+B3LU0ild?W=ER#j>2Yb|NxcC2c{VoF zfb@$`8=uFVxI zl7rd-8vnp_-H3?@R?J$dK10 zX%W-vHRE6oUW4#oMFJ8H=DtG+vDm!+2awq=@ES#5;be%zI_aM>i%(7g)!vtbZ(W0a zjp|mcA9Am&A)!P?|4!7=B)gWDiN!))FW<>{qFCOr^3Hj?A`>qhLUWx*)SN=MkU_=uGint7+?-PJGR@PPr0Fq{wYI-}uA?C0?n*gj=7X8uM{6H* zHmAl9!`2#_s2?gc$hq*JZXiRnxcjvo#n`T7(ymBbt#v!@w{#Pn21@RRC9J9S2r>R5 zavmYNWPi+@l&LEqO6ooL6{CIke# z*YkN(6!?oM2lSk-xu@6Z2RJt!_G+@8y~WD!J74C|Pk$Qy1IWtVZ%tvPPG7{Ey(4Nz zly;aLU{nlW=RPc61%d$B)BQ-aCEw)T8TEuZS$I#IOyXH}B*p0|a%GwLEr4zGC_;5* z2~F5Dh_4NDyZ_wqL0V?MMid4+B{q7_UP>mD7=?eg^1Pn+BkAnd@xvJ{dGn_ycmQ`5 z)RvY0omi8(h(Dp~dN#xLl3ELId^{8vB;jjA{0av9z?uB z3Jrypc}B*b;xScnbzj#M!#+54QWyw|(@oS-;O^dbs;}I-a;@3OTZt}}zdHJ-n`#Co z5&=QPa|zOWRNaGk z_RA5`XOwBi`Wc_x+fQ|2ndq9nMG#=vx+0(-z~Sa zgz4kjcsd{5L!Nw)<~O-&ZRyd59w?DnRG?;b@X!@%mU-!|Z|?^!O255!hy_79I5Sozhq;5~hp*9^uzn>v~HS ziXv_|sh>~SOUZMxTJ>23-^)Rax;YK6j}QD{IlsPYHcXLWM@9Qe+}WD_4SlmV=F_HpJA9n$$*`RH-4wEp>d)#OQB=&%(si$v4~L%Z>A5hB&x+20 zs>T#qM`Nc!`pngLkFL9t-k=LVUYRC`IQ7U6`q`@y`bMmto0hax^l5s!C9WI{_5DtmZo@H}@6Lu7wOgL?OG|RL@p;`zrj}?@$QFW@ z0dtPekkz!mx&C3*nSoYM@3_GL)IUMRi!_=7tQ&UkwYB-v>xF!`vd(pExhHv#f4Ujb z;T$R6XMwXGvka3anvmWWWTm2wS?BlA=}di@a9Rp^o-z&U@J_gPbfcRwCyS8iYn;o< zZ1kHqoywxg)bSDeC6~%zo}(@H#^LV@4!t@;!dQK8EhFb{p1WltU1Wu1!Ey?~uAZYwbL zk`kZnFK5c+WXb%^InLW^S{=VsaelJY??${Bt0@{39x5o45QYng;?uR5(4xmnv!cpk z-kiw`9FZM-bteB~R zp^HVkF291bn}km+2=_~|Y7fR=MPuR?VXuw3jO~o2&|$NC4gBon9$9*m)j9$th_CDF zba_w_p{Fm;wsJP!p&zL*frxl6Em}nI} zfXL2jz0ZA%fllyH4rp)$96Gkpkyq+aQ+DZRrXkGTw;SC%E#uij!`}%z$19T3I@VwH znt+x$7+**zRba+MtF`;7?tL4BhW`N+LD&0$*-?p}WO|I5isr33fXgR9!xz|6m6C}Y z<(*2{71!_2O8+rh&97}xu|^>1vUV&qW)e!ZS+SIwt#Iw2|F3eqDbSX9Mj0t`<-ZT5 z^RtP8Wz^5{CJ$S15~0(A6}J_ocnidG+$|phwm?<>`keruDKnXg8#NoE50Z~sVvcH0 z=3&--GezjRt34X&g6%7OHT`^*O_W3r>nff^=t((!Vhc@HsHgU-o7`>sku)z=Mx==` zn^*Lzs6lY8r5Ljocle+SR_4odWKI?KlT3A-cE}6Zg4Ez|Ut`m_c6cdPYVsmoxbvIG zBBeh>X z_X}C}fD<@)FhFxH?-&{g-t>Fq};-;mN46&B4O5TP*>ry8c%m2x*f>W)(s|=@9Qu{ zW3?0R3@tB++64P6O36I+05wCu+AmeH3bci!7<_{#>?{q>ar}GT8NzW=RUn{!f^BRtm}42Z*lmwEc-Ld;!ksxGT>L2v3QSJhNn z;6i*7R5O_zIRoD*<=Zy|KDk+dPP?W1&1mc~E&a?HZe4%d3g~O=-k~}F?x44y?Lfb4 zk>{FH;!Z_jWm_>$Z?0hFooEvbMAp4LMl;Y#a?pfeOOj{X~l7ht%f z!dRhv5DBY@*9I2=)#Zexm0PZsGRc5Jh|Ij99D;Kkp2%baG^$-fn> zRDL*2t#4aTNWQ7VU`q3cMN%4jpB~`TV3RZWQ_9`&!dOlFl|Neb(#g(l9uj5KdJiA?EA58k^bk5LxGdcb1142_ zO7zdsWiPi~Bl%)shuVQu%CzPoFM8Ci9rjOEJ}h(Iheyv%WUctFHwX|OyHm|9H{+>_ zVT4@w3slV>yEdpD_8ol3EhL5fzfqk!CGDYIHQ@t0K|Awt^TLhmvl=#y`%eG`v{ZiC zHJkp?9l7-@C8>I$gi3%y7Rm4289)>6LJxID=S$Q)2#zc5p_Oa|_R-~o3GeXGiOG4) z_!664cf+ClULgX*K8lqpsiggu(~g(-w^SYoyza5tK2(3ehj}=pQU42rQU?3J)9ldH zotRzbQsyXuS}EAa{pwlgY7*=Vbq~-iY7hclItp;L3CEpES!iEFr(;1p_qGLUJJbpT zy^KpM4mOQ#F=FKB_Jqw+eZ(1lTV^`ce$mr@&#oKB!gCP0KOHLEHwRTXDA_;MDZ7qS zaakoGm_`x15(MaVl_Mwah}<+dv99ZrMu`oG<#L) zL?N1ImHIa29Z-0ck!|Oao8;m3DssXHnfvnbWj*usoYv*@dbCKw8w8^;Vu(Q(34 zrgQRzhikO?x}ILTA-6c~TAu%+S?@_zU?`u0O{+}94%g%ZbwtQr0Zw_|(eo7s#V#UIc6`#vEgD~J$Kbnsn$I%OmnX|N*qL;YxT1d-51y+HOv z?2SOHL@c}?+bmJq-hM0OKmXP7>e$`(<8=NVr2+dv72q7_M4nT=+gC-&!}i76xMHe^ zvo_i~4MA5kU`DA1)!3gsA{ocFZDnI6Qe(ImRE&q#Kz*`OT96sA7}*5*e^6e2yF~^2g$y(b8|T4=A6i*6xaC zOh3;^s*wec4krqCz+KJ*(*mFxI~-X(B2})!+y)m;oXVi81&G+HC^^@I-^#zWGvi!? zidT9h-MCFM>dFneAsw;)-oEc*@ zyv>>$R7`n!d5YAn?{FB`d2Uk;GyUYGu5%}()eS#^P@Kz0YQ5K+Yc6Fx2?q22ePOLF5z@Vq z&;YxVVHtI*-gPqohrSV`v1A5mvmB^mHU=#)O8;<;+;9OG<1_^tbz{bbo*)5 zG{C&2;r9VWwP1aVyDx{7m>F$WdwW0dyC~}G_KHT-_MM8HPNx#D{9D{7u^buq*zm-% zV4yY-=BS71g-YRcr%d_)cR1u zT@bhp8}m(${GlDcGk3PNoic5p`ttn>D-DUd*|!D)&Y|-VKB9grnVNQjw^V`sv+>o| zE788=4N$Mz3Q*Kf8F9VgU9ypsa&X+74giae7)WnOIP)4n`|QlXq#Q4AmI-@S@fxJg zm1%UI*3y6PQ9F~&(f!Tm!#C4Me%`b{$>1LN*=98!=u$F%t!fqmlYS^;e%R|jUi%8> zgD`=#G{E`eqyL~VwNV~W+i-?zWGr99o#$SKO7=s~ohqexwTDLzybezUA^)0ioB5lJ zAlKw%Ef`HASQoQH_W2$i?*;Vgw4D!ty+C=%Ir{0{ya#uJ9Zut|PFh#eVLfe2_n&@} zDu#4M*<2rJD(fh~F?B^OOz`XSSs8uT$s4P`EmAn-4NZ@Jy1Mu$o>ruwMOXcbflOSv zrX{HMJdvj^=IobMt`GT%PnRDt{<0)-UvT853pG*jBpn-~oF2SRty$*pCe}Jo1X9bB zG?P~?Wstj~Sv#e$LFslz=4kj=-{BH6A2yt!Al?A~dBHJ7Z>kwDZRs$R9#uyhnIU=C zUii3e^vs#JH$krT#r+Xzr2w54QkMjnCKf6#XCfUwY%xt7HFyMuzboeRLUmjL^k&l> zD^rHlYm)_ka+KVrikR)+RCFO|CS}{%}k@x31RZHPWcUOHjkT^GCAuQS+i~B+f%|j0!iIDNj}%=%LOPC#n`1K+h6idR>SR#DnFT7riF8~Dm&w~ zwO8`(jDGw-@$?jD%S@G9D)#-n)5CH-VAbEDWud!&vi98752gcy%0=(qRPt4Z<1S{; zlnIqGjW}7s)6iz6Ysr8?8;HFy88YNCx;A|`(z?sl^$t?R>+*>?Geu1-Yt5)5-b&F=ipBYLDH;v_H6Gsl=6oSM&Bodc z)5d=S8IPZ%MVISVOAFz`iz9L9v?+`}Egle4-MVw*)r)=OFqfnosvPe|O4W_6Axcxr9j*Q@6x z7i_qU4WRZDvaGwg2M0XvMPr-4`2~vp1-0DCYg^RkzkL5=a2~&pc>qlxdGa_K(+lG0cayDn@q`vq~TgxP7v z8gxdcBqQs_1NwM534S7G3L;^*h#%AmYVWHmI@SE2JlW|`J6FTEpFA01V|>AW5A$Ps zm6kRt)C{NH8xq?Wvl1 zkB4)C))8B|Jl;!54sV@p?iD@sOTb)@4Vxui<9zKyL(Q}kQ({Ct<_*zQFg-78_m8y& zlpoDGmty!i<$)Y|X3>eKkK!4tZL$w&G3=XxH^omYvqm4yq6xT_v3H30;Y9;Ts*z7j z@=Ar~tWf5IfutLCxG|^pcOziP;6nX%VRz*d(*nfeZqoG&M3^%r*cW?^D8?sCpE2?&ALp(XBRmb6=9r#&g} zJ_M!obMT8@N*eZwm0hwVBf5by;=5>ec*uJ*>8O(g)B$!}3tb7-!@k-~a?9V=2yBs$ zHpOV9d+k2oE3`6kz>WDJ&mx znnLohR7z6?gBUIPV`X(iY~^zDv?@E5eT1%XQwt2k-z%N%a8ueh%;tLkRjtq0D?rr; za90aFOBATS1|KQk8D3SbQU_bSOm`Y41`-D)M%HQ{Jqln0>d*Y1GtadD)wa4Sfc&-R z3G2|ozW;Ng6a{5HH{f70GmlvH;aIBzGTDapi|K8aEZYoSK~)Z8@-XWV6A=8``xR>_ z7fS9-1%E@#=1{vsX)@#{xwk|la1+{ci3J%;Oj3*e#g zxU5e29?u6mbLMr`+ANQY9^Mtn`Unb>!vg-Ch)(@%fafj1w<96iLQTPa*64VPNXq0} zC2)p>?n>svUPuIN_(VMN)rYUrjR`}5X@!a%P%ypSYAc_UPu3@)6$;j>3IxQ+P5s%1 zg(N+hFzM6n;a~)t;4wwCdkV*!HMBiEiQ2foOO`2Y;5&pzh;W`eJ~9hZUU!A^mm387 z6tp=~UyyYixS>Md{g4jr{Z|u{7ICMhOR)QRS~=i^E_{$aKrB-nc6jgWtZz4bG7}sZ zU)_Ek2Thtzj8hcJG4G2gA)D-|dCxAX{q96mO)>QZDA=1OfODw3J_mkUQ~CwNHKOpJ z02sO@#VT2wvo_au_T)Skhs_7f+^0piV*&lCt}D6N)a#pc_O(lsFB7fdIm*xfJ=+mL zL$o9-Cnr>Q0_(3IjY@T)O}F5{MZy^5e-iS3eX75K|qk7jX1ov+CD&q%la3!Zl$5?H(A4m(nQ6o)R54d9+6j0%z*=#vIwSp z7MVZXuB}sU=DU+o(-#95R*M=AiRfX$JM3?%$DYq@#)38IX~uBr7xbS#7o{49gYRdrh0NxIxvlTufGDXNcm? z@6J#sNu7j`?QFU9fpI=or>7^}f!NA0apg|jyh!zz+&gqB0{k9oT$4l>Y!)cG7J~2Q zWe`Pys&#l{akEJC0p6sD)zg4vhl)o&r@#AEw=DZk$ud20$h=E?>7DjQxqrB*-Mt7( zd_=L{Q?q@^i);<j$T+N9kUlb01#DUwN_TvYSyPVHlD&QWqs&mI=WYdQ{8&fR` zcA_PI;_hoxm)WpH_WoPbSa;u>LU%vXGmaIWKP5b*j>p!Xc^m+k*08Bop`at~VbS5E zsh&h;m{Dl&c2qz51t4GdG)PPraDS%~?^$eKFZ3yaed93#%*>khgGJ$#5*RcXj%u3(RBcV)fRA3g>_+7k6&61M2)HSW zVfA5*3a#H~f@HNx1Gsz`aAC#zJ7h+Yi2HIo5P%mVOGq)>D>y4mb0@Pb=64Gx=gTqx zrjrBiEI`7@I&Vmnz}mifpNAI*2g1#d@b!H*_)gHY``e#0LMi*rsEFC$tUi$daBpCp zE<9}2fUX5U0&p{Wzg;gh#0t7Dx8jSb20%Q~r3ThXW}?nu_uyUm?Pc8ijo;8pRA_s% zJV(kh#kx@r?$&k_I{n zi7n(hK^vEPfZbK!PcMMQ20x#Q7dym#3B8!@Gc_yK1gPDN581s5Sv&Zx11Q#xt6pic z?P1XRS8ZhAv`Cghg`Z&Pm(F&h6q%j$plo4C&~!|8(0WU#Pz#C&?f4Szxv-|wlY`E} zn8nR2q>aMo<+Hb;wU+!Qu(Gf1N-$LPBBV7?3FaF3qR$ojJ3R$?xDt_HZ7nObOZ7?e zid~d>hTYTWTo|g(4S7bZk>x%~Ul<0)_VT)uFH5sZ7nj)EDZvyptFh%PzSd) ze>`4vtP}=KnJ0&(Xmr`4lKT+aU5<=J4xf|DhDj@5Rhzd-n9H%D9Lm9uLjtLEtwNhx z**|e%DAxP~(l9U;3}You{WqIvh|Vi)$`SuxG^G6%mMxGf0edx2CjraTw9uwLT}y5^ z|6*lpx>)`&svmo^X#u+arXO9u;=WOTkaJ}B9?LP3s8jP^$<@rXr{SXIOEd4etHEs{ z`VaGkN1|$pq$tB&EW45FOCDNz(hbf==1BkiciP->`MDnM1m4Wxy(Mp63Ce}8E15)I zqG_+yDjZDi&2lGNrID1u_8vP2VLgdm^A)wUR26Pgezm_Ul<2dKVZV>;ws^QrtH(MY z*s1cUo!~6RH4cgB9@#b#Q#)*JW_!p&xVU2al238Ft-YX9IC^e{b_I?2j_ZV#!h-eW zb_j0~O9VsO{ZKCl0U?*%oB1E>+~zQ!~Fem*ho9U6p!*8-PQs1p`yx< z-Uj**qkxW?QMp2B$a=8u+HQF>HZi|X!E)8|85FkL%@_)un70p&&t8;8{gfiStxW7= zt>w98gQ~L3>Yp8u`UdI@V|zI&bWpy}TT-ugro3nLV6QTvWhENf4|ioCIqe2W&jm3- znER1BTHvt*qg%U8&;N1B-2Jwc$`P!_c5nX6OwjbKGo!>vcZk6JQw;1-@df|P{rOMW zk#0oU;hN0Ke#3KxjA&M<26Redv~iC@j16jGVTEFW9~y~u9k8zq5dI@MZ+ON<-S--Mkugt_=ili;~cS^agvDlL0^&gV_u8}4U-2Ixyr3MUd|*e!mc~c;sfEheRtf~ zUi2mzkOj}EOu}-5 zCi}@+M|r9BY3GVpwB-ynIT%8m%nU5_3-h_#Gs3K^7)f^W6-7vD&fQ9r^dt_)_bZCL z1UDDdtZn3sZfi+d-_^!|D-!UYW$`&wphOjTgPJ@7j!BKnc=UN+4x zqeY3E-=Pzr76d0_%O~v)2R#x7UH73HZEv-EU$c=s*sk3$ZVUUtOPz$=09B_K6!$nJ zgZhgugp2xrVh{zL0qma|zXx^}*=K%ZBx#NwW!M#DOc_D0k`P6399WIa<1s702*ZXP zKUBhUnI6)+wGbNjn+MF2u~L0xpt-?1T+yrX8g-JlMHg1&c_|F@8*igu!axuDBffu8 z^wJOGZTHe+k1eHypY50ft&{o|pzV^W>)V#WlNNCM!(K{g;5mci@MxzQ>0u_F8K4%x zi)>glq<@jZ6c78FFrNrxw?ZX5uQe7(+bu&v0ymlMYZ~zT*iZsi0*`A)c`^x_O^3Wl z7U{NPzE>=TuosoITw)2O$X^`joKyBIfyKPnZ2}1(>5P>e@Y3-fR%~*JLtH4P&7jiK zb9r0gFd8r3)Rj2=b$j{8{#MRI%lySrnE8au3qJD)+j@!EXjvFRp|3C-V^Mox&fPRJ z;2rAMlgE-_gsP&%AUO4t$mH{vWm|A|UqeDR>wR1{m*&?-cUT13AquN;@4w7El>QR@ zpjg;V2nt;snt}y4DcimO;%zJIzsh!hA))#Kmf9ZwvFMPwrURG1#NM#S>I0>Hb&r3!Oe2O}#Nt3U5rM=^ik`-87 z_UXL|)`9H=$z>qQg#|R@5{2(|Rd87ULAP=*p>`B1xRF*#iDJ$#${T7hpm__kKx6=b z34M|!l}PKaNZZp~XOq?y^KbVrkcb_KRJ;-*@02l+VXb#3ID+|5tbz$3+f@KryKMZ) zvemf9a`b4?!jjs%SHK&(tAx$|+eAWC3nFb54r9MbveO)_57MbK(SQwrErUSR+N6Uu zZl0hoglZrqx^WZ(S`vjXf`pqClzNWjeTG-Ino>Rwd^pCR6(m5M)W2J2od=j@c#2rnpU@s9|7phc0jVfrm+9SXynv<7KjSC_CR)GSi zIlw##axiA{F9_6Dluk**K3kY|!@Wpr)ktefqHraY>qb?x{4fRveSDJs=QAL>i6H$M<*-6#nv8&cinr7?>C<=l! z9zBaV`7rDA00tuY-^-+14(z=|pU(kk4iseKsP!4Q^usGn2E7XTE`*h9&j+wkSwvm&tE8VhgTOfA(~x>hOA{C^FLsF3*ime>-r3WZZlEa|#A@=eky64CFki%X_bF z*rKVKSxdt4A)T?_*qmB{?CSVHT7akl2C=pN_Ef|W97dvlqq9;bK)B-7mo4q~zAeL? zmwiC}Yme0b5Fyrx@(!N~up}S>>n8Sc4;!4tarerJeye+BZXh@q+Xdv(-DMEjO9K-3ApAEzGvgALfnlbLbArFyrLd{u#jYC2_ zy)qBO=XWo5&TWvHa%O?j)WV24kX2UP7F#zdK)KGZFj?xv7F;}g`u+D4SAyNmv{%V7 z;CN9)ccQh1Uny=}eCtd@@*wwi)hF~IqR%@VfLDhzQgL@UPNb~}UGTdPfr^lX%Q(I8 z(`y<<2gdh7R=_l-%SeiNy(_8lL}nRlkdX!>SiaKn?b2t?6nopY1;vA81*pANI1`{i z@EC#AEAz4%+~CUi(E-~Q#A$bvhOXe|bVg@LiG1VCl0Tm8kWEBK8n)Ska1Mc)(RM9J z%H@H{T?ums0)5S$Tj52lJOM$V?KbhU8c&fZ7FRTLy1k?k9kXpdw#zFkD;0Ih z56s$zy~9;ND#W;rg%4l-34lsw%4m3#2SKHh`JfS8V5tG@kRT&mduBOs+Wj;O-o`mj z(-Jvi3}{y$4l|j!L)J|P&TuKwVn`^p~6ovlb_H3Af&!2M~uX=xk*N=Z&j#4_s$!1^`2M6eVIF=LmbN zwE5iZe@5h!&3TY@+M)0n&M*8B7^^kOj_w7$P#)^fijmeKG;UIHp&((rGc*9Ko;Sbl zd~(l;>=}L3mz^RGH@Ho&)mBsjU?6vYivz5Hk7%pb9rpmWgK$R8NyuRq9}ZsqHg5=9 zp89jc?HNVVY>8I)x?6-aX7H6!{}P8&1zQrpoRM!pkIJ?uM=N3=HpTL*7lZR_0HXMfcPv1&>>K8;o|`pM#npPnp5go63Zre~Mcj%@ZR z`Z;9nwUf*t3GMzlTr{KPTHwpF%m<7+S@_(YN;J@EhT|@*H%G3deP+v$U|I>TgyeUA z^=LkM`4n17b?a4_Q1J>lSMh4p(A8+de@?%Q{e6oh;DJ&7YL z51OlMS_e!Fcbh1+as~zio|d$(~4|_hnn( zF@LNQc;JA=*G57V;lmF3R0D53KMxJIoxCH-w^3kC-Vjv}$`oSg7(ltX0B8-SViHh~Z} zdLbc1Id*{=?iReJe)19T0ov_iBJOtVev7oTn(L5T9_Z~Lcu70>kd4-jEyPTyC`ouc z*q4QEN7UiD{JtZVm-Fb64?neF92$|}Qp);c4|AlUm1u-nWry{K5m+;j#!6tB&L>0w zP_SVZ%RI|iY@ZTGYUpHw|7lF(1P1!{YV$Nc5ZNV61L1@3_oM(o83@rbfc*p&rhmJC z3WLUa8z2&3u@~cLr@{V1kL;3P%?D```$?u#{5naX=?0+cbz0kIeH8g(IRt!uZ+&&O z_w}P=8lf}ZfZg*z20jHLQ%ADH-h~BG@_8Cl&VfdUV(-4w5SrJ7PoNJ2Mi4v)zjjLt z^kQT2KY(M&o%oSEPZSR>5IqX;TMtLj8y>?qF;}QROL$~~u>+<48K!uKGZw`a&k#2-g(^S^-#|Gr`RTwZ53? zmJU4XFiY$GBU|zIzoMlb;Fuy>fYm+S=0xB`3s4mt3N^4xKSx6%(TWHy+A8)Tlb)=m$j?DNO<(z5;$GO z#LhG1HngYEJ8x*OD?=rXJ%D z92ytY#umnLloy=&$TQ}DiNxpSEpaK;58jz&KyiENEkQ`UZZ>BD&`)%81n|2*7wl~Y zWbi^wl2zO@ja;}3K38uXKhC8Z`9iZYB{`Xd=tib&;O6)HMW6W>L?Vt_*~5U3z#Xn- zFHcqMBm04Fe#;s1&O|TThW5JYeHEC$e4*<2GjzlC$3MxNgFsVF_Zlv_2k6qTAXCmM z;8QM3i5Znn1Cy73&Q+7L{67(o9^o4&kqz(MNXdQA`nVg?*l zW8Fwg|4|eqHq?V20Fyve=r4?&s_(Tl-M+)HRkLI*N}5;DKJ6?YVYxs+S+zb71}_Ll z+Y=q7ATRtj_su{ks<%_T@Gf0;t={{WSL3e-r}3LsIX<>}H~SeylefIcuC6XL zI4MVF7s)!!Q6zeNn2~G#!YQ%%|F&M3ZT69$KKzojUbC`9y_ee{Oi$}S4 z;fkchMn*=$MPfrQlJj90Gb<}cDe04lb35Va83}RmV)b5*Cy2TsQG|_w$BwsB3KYtc|@ zIZMoN&P$xK$8&9SiAsVJ)x@sc6({|N>&ZCzRiF}|hE@s-xq#*(;X(wjgWs& z-ieDv=CW3)RUgf`+mJRYoaA-}`8;%5QcS{XhRJAU2)BkEuT>D zJ?C!(%x0)Nk-^_Te%-w$jFY7Y&9kAyOp=C!~YMCKzF|Y literal 0 HcmV?d00001 diff --git a/examples/bot-ready-signalling/client/react-native/babel.config.js b/examples/bot-ready-signalling/client/react-native/babel.config.js new file mode 100644 index 000000000..2900afe9d --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/babel.config.js @@ -0,0 +1,6 @@ +module.exports = function(api) { + api.cache(true); + return { + presets: ['babel-preset-expo'], + }; +}; diff --git a/examples/bot-ready-signalling/client/react-native/index.js b/examples/bot-ready-signalling/client/react-native/index.js new file mode 100644 index 000000000..9d93586db --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/index.js @@ -0,0 +1,7 @@ +import { registerRootComponent } from "expo"; + +import App from "./src/App"; + +// registerRootComponent calls AppRegistry.registerComponent('main', () => App); +// It also ensures that the environment is set up appropriately +registerRootComponent(App); diff --git a/examples/bot-ready-signalling/client/react-native/metro.config.js b/examples/bot-ready-signalling/client/react-native/metro.config.js new file mode 100644 index 000000000..9430b0f9b --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/metro.config.js @@ -0,0 +1,4 @@ +// Learn more https://docs.expo.io/guides/customizing-metro +const { getDefaultConfig } = require('expo/metro-config'); + +module.exports = getDefaultConfig(__dirname); diff --git a/examples/bot-ready-signalling/client/react-native/package-lock.json b/examples/bot-ready-signalling/client/react-native/package-lock.json new file mode 100644 index 000000000..ed78f865f --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/package-lock.json @@ -0,0 +1,10970 @@ +{ + "name": "bot-ready-rn", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "bot-ready-rn", + "version": "1.0.0", + "dependencies": { + "@config-plugins/react-native-webrtc": "^10.0.0", + "@daily-co/config-plugin-rn-daily-js": "0.0.7", + "@daily-co/react-native-daily-js": "^0.70.0", + "@daily-co/react-native-webrtc": "^118.0.3-daily.2", + "@react-native-async-storage/async-storage": "1.23.1", + "expo": "^52.0.0", + "expo-build-properties": "~0.13.1", + "expo-dev-client": "~5.0.5", + "expo-splash-screen": "~0.29.16", + "expo-status-bar": "~2.0.0", + "react": "18.3.1", + "react-native": "0.76.3", + "react-native-background-timer": "^2.4.1", + "react-native-get-random-values": "^1.11.0" + }, + "devDependencies": { + "@babel/core": "^7.12.9" + } + }, + "node_modules/@0no-co/graphql.web": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@0no-co/graphql.web/-/graphql.web-1.0.13.tgz", + "integrity": "sha512-jqYxOevheVTU1S36ZdzAkJIdvRp2m3OYIG5SEoKDw5NI8eVwkoI0D/Q3DYNGmXCxkA6CQuoa7zvMiDPTLqUNuw==", + "license": "MIT", + "peerDependencies": { + "graphql": "^14.0.0 || ^15.0.0 || ^16.0.0" + }, + "peerDependenciesMeta": { + "graphql": { + "optional": true + } + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.5.tgz", + "integrity": "sha512-XvcZi1KWf88RVbF9wn8MN6tYFloU5qX8KjuF3E1PVBmJ9eypXfs4GRiJwLuTZL0iSnJUKn1BFPa5BPZZJyFzPg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.7.tgz", + "integrity": "sha512-SRijHmF0PSPgLIBYlWnG0hyeJLwXE2CgpsXaMOrtt2yp9/86ALw6oUlj9KYuZ0JN07T4eBMVIW4li/9S1j2BGA==", + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.5", + "@babel/helper-compilation-targets": "^7.26.5", + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helpers": "^7.26.7", + "@babel/parser": "^7.26.7", + "@babel/template": "^7.25.9", + "@babel/traverse": "^7.26.7", + "@babel/types": "^7.26.7", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.5.tgz", + "integrity": "sha512-2caSP6fN9I7HOe6nqhtft7V4g7/V/gfDsC3Ag4W7kEzzvRGKqiv0pu0HogPiZ3KaVSoNDhUws6IJjDjpfmYIXw==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.26.5", + "@babel/types": "^7.26.5", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.25.9.tgz", + "integrity": "sha512-gv7320KBUFJz1RnylIg5WWYPRXKZ884AGkYpgpWW02TH66Dl+HaC1t1CKd0z3R4b6hdYEcmrNZHUmfCP+1u3/g==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.26.5.tgz", + "integrity": "sha512-IXuyn5EkouFJscIDuFF5EsiSolseme1s0CZB+QxVugqJLYmKdxI1VfIBOst0SUu4rnk2Z7kqTwmoO1lp3HIfnA==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.26.5", + "@babel/helper-validator-option": "^7.25.9", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "license": "ISC" + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.25.9.tgz", + "integrity": "sha512-UTZQMvt0d/rSz6KI+qdu7GQze5TIajwTS++GUozlw8VBJDEOAqSXwm1WvmYEZwqdqSGQshRocPDqrt4HBZB3fQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-member-expression-to-functions": "^7.25.9", + "@babel/helper-optimise-call-expression": "^7.25.9", + "@babel/helper-replace-supers": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", + "@babel/traverse": "^7.25.9", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-class-features-plugin/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.26.3.tgz", + "integrity": "sha512-G7ZRb40uUgdKOQqPLjfD12ZmGA54PzqDFUv2BKImnC9QIfGhIHKvVML0oN8IUiDq4iRqpq74ABpvOaerfWdong==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "regexpu-core": "^6.2.0", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-create-regexp-features-plugin/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-define-polyfill-provider": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/@babel/helper-define-polyfill-provider/-/helper-define-polyfill-provider-0.6.3.tgz", + "integrity": "sha512-HK7Bi+Hj6H+VTHA3ZvBis7V/6hu9QuTrnMXNybfUf2iiuU/N97I8VjB+KbhFF8Rld/Lx5MzoCwPCpPjfK+n8Cg==", + "license": "MIT", + "dependencies": { + "@babel/helper-compilation-targets": "^7.22.6", + "@babel/helper-plugin-utils": "^7.22.5", + "debug": "^4.1.1", + "lodash.debounce": "^4.0.8", + "resolve": "^1.14.2" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/@babel/helper-define-polyfill-provider/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.25.9.tgz", + "integrity": "sha512-wbfdZ9w5vk0C0oyHqAJbc62+vet5prjj01jjJ8sKn3j9h3MQQlflEdXYvuqRWjHnM12coDEqiC1IRCi0U/EKwQ==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", + "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.25.9.tgz", + "integrity": "sha512-FIpuNaz5ow8VyrYcnXQTDRGvV6tTjkNtCK/RYNDXGSLlUD6cBuQTSw43CShGxjvfBTfcUA/r6UhUCbtYqkhcuQ==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.26.5.tgz", + "integrity": "sha512-RS+jZcRdZdRFzMyr+wcsaqOmld1/EqTghfaBGQQd/WnRdzdlvSZ//kF7U8VQTxf1ynZ4cjUcYgjVGx13ewNPMg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-remap-async-to-generator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-remap-async-to-generator/-/helper-remap-async-to-generator-7.25.9.tgz", + "integrity": "sha512-IZtukuUeBbhgOcaW2s06OXTzVNJR0ybm4W5xC1opWFFJMZbwRj5LCk+ByYH7WdZPZTt8KnFwA8pvjN2yqcPlgw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-wrap-function": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.26.5.tgz", + "integrity": "sha512-bJ6iIVdYX1YooY2X7w1q6VITt+LnUILtNk7zT78ykuwStx8BauCzxvFqFaHjOpW1bVnSUM1PN1f0p5P21wHxvg==", + "license": "MIT", + "dependencies": { + "@babel/helper-member-expression-to-functions": "^7.25.9", + "@babel/helper-optimise-call-expression": "^7.25.9", + "@babel/traverse": "^7.26.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.25.9.tgz", + "integrity": "sha512-K4Du3BFa3gvyhzgPcntrkDgZzQaq6uozzcpGbOO1OEJaI+EJdqWIMTLgFgQf6lrfiDFo5FU+BxKepI9RmZqahA==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", + "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-wrap-function": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-wrap-function/-/helper-wrap-function-7.25.9.tgz", + "integrity": "sha512-ETzz9UTjQSTmw39GboatdymDq4XIQbR8ySgVrylRhPOFpsd+JrKHIuF0de7GCWmem+T4uC5z7EZguod7Wj4A4g==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.25.9", + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.7.tgz", + "integrity": "sha512-8NHiL98vsi0mbPQmYAGWwfcFaOy4j2HY49fXJCfuDcdE7fMIsH9a7GdaeXpIBsbT7307WU8KCMp5pUVDNL4f9A==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.25.9.tgz", + "integrity": "sha512-llL88JShoCsth8fF8R4SJnIn+WLvR6ccFxu1H3FlMhDontdcmZWf2HgIZ7AIqV3Xcck1idlohrN4EUBQz6klbw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "license": "MIT" + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.7.tgz", + "integrity": "sha512-kEvgGGgEjRUutvdVvZhbn/BxVt+5VSpwXz1j3WYXQbXDo8KzFOPNG2GQbdAiNq8g6wn1yKk7C/qrke03a84V+w==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-firefox-class-in-computed-class-key": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-firefox-class-in-computed-class-key/-/plugin-bugfix-firefox-class-in-computed-class-key-7.25.9.tgz", + "integrity": "sha512-ZkRyVkThtxQ/J6nv3JFYv1RYY+JT5BvU0y3k5bWrmuG4woXypRa4PXmm9RhOwodRkYFWqC0C0cqcJ4OqR7kW+g==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-class-field-initializer-scope": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-class-field-initializer-scope/-/plugin-bugfix-safari-class-field-initializer-scope-7.25.9.tgz", + "integrity": "sha512-MrGRLZxLD/Zjj0gdU15dfs+HH/OXvnw/U4jJD8vpcP2CJQapPEv1IWwjc/qMg7ItBlPwSv1hRBbb7LeuANdcnw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression/-/plugin-bugfix-safari-id-destructuring-collision-in-function-expression-7.25.9.tgz", + "integrity": "sha512-2qUwwfAFpJLZqxd02YW9btUCZHl+RFvdDkNfZwaIJrvB8Tesjsk8pEQkTvGwZXLqXUx/2oyY3ySRhm6HOXuCug==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.25.9.tgz", + "integrity": "sha512-6xWgLZTJXwilVjlnV7ospI3xi+sl8lN8rXXbBD6vYn3UYDlGsag8wrZkKcSI8G6KgqKP7vNFaDgeDnfAABq61g==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", + "@babel/plugin-transform-optional-chaining": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.13.0" + } + }, + "node_modules/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly/-/plugin-bugfix-v8-static-class-fields-redefine-readonly-7.25.9.tgz", + "integrity": "sha512-aLnMXYPnzwwqhYSCyXfKkIkYgJ8zv9RK+roo9DkTXz38ynIhd9XCbN08s3MGvqL2MYGVUGdRQLL/JqBIeJhJBg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-proposal-class-properties": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.18.6.tgz", + "integrity": "sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-decorators": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.25.9.tgz", + "integrity": "sha512-smkNLL/O1ezy9Nhy4CNosc4Va+1wo5w4gzSZeLe6y6dM4mmHfYOCPolXQPHQxonZCF+ZyebxN9vqOolkYrSn5g==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/plugin-syntax-decorators": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-export-default-from": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-export-default-from/-/plugin-proposal-export-default-from-7.25.9.tgz", + "integrity": "sha512-ykqgwNfSnNOB+C8fV5X4mG3AVmvu+WVxcaU9xHHtBb7PCrPeweMmPjGsn8eMaeJg6SJuoUuZENeeSWaarWqonQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.21.0.tgz", + "integrity": "sha512-p4zeefM72gpmEe2fkUr/OnOXpWEf8nAgk7ZYVqqfFiyIG7oFfVZcCrU64hWn5xp4tQ9LkV4bTIa5rD0KANpKNA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/helper-skip-transparent-expression-wrappers": "^7.20.0", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-private-property-in-object": { + "version": "7.21.0-placeholder-for-preset-env.2", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.0-placeholder-for-preset-env.2.tgz", + "integrity": "sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-decorators": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-decorators/-/plugin-syntax-decorators-7.25.9.tgz", + "integrity": "sha512-ryzI0McXUPJnRCvMo4lumIKZUzhYUO/ScI+Mz4YVaTLt04DHNSjEUjKVvbzQjZFLuod/cYEc07mJWhzl6v4DPg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-dynamic-import": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-dynamic-import/-/plugin-syntax-dynamic-import-7.8.3.tgz", + "integrity": "sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-export-default-from": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-default-from/-/plugin-syntax-export-default-from-7.25.9.tgz", + "integrity": "sha512-9MhJ/SMTsVqsd69GyQg89lYR4o9T+oDGv5F6IsigxxqFVOyR/IflDLYP8WDI1l8fkhNGGktqkvL5qwNCtGEpgQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-flow": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-flow/-/plugin-syntax-flow-7.26.0.tgz", + "integrity": "sha512-B+O2DnPc0iG+YXFqOxv2WNuNU97ToWjOomUQ78DouOENWUaM5sVrmet9mcomUGQFwpJd//gvUagXBSdzO1fRKg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-assertions": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-assertions/-/plugin-syntax-import-assertions-7.26.0.tgz", + "integrity": "sha512-QCWT5Hh830hK5EQa7XzuqIkQU9tT/whqbDz7kuaZMHFl1inRRg7JnuAEOQ0Ur0QUl0NufCk1msK2BeY79Aj/eg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.26.0.tgz", + "integrity": "sha512-e2dttdsJ1ZTpi3B9UYGLw41hifAubg19AtCu/2I/F1QNVclOBr1dYpTdmdyZ84Xiz43BS/tCUkMAZNLv12Pi+A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz", + "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.25.9.tgz", + "integrity": "sha512-hjMgRy5hb8uJJjUcdWunWVcoi9bGpJp8p5Ol1229PoN6aytsLwNMgmdftO23wnCLMfVmTwZDWMPNq/D1SY60JQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-unicode-sets-regex": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-unicode-sets-regex/-/plugin-syntax-unicode-sets-regex-7.18.6.tgz", + "integrity": "sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-arrow-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.25.9.tgz", + "integrity": "sha512-6jmooXYIwn9ca5/RylZADJ+EnSxVUS5sjeJ9UPk6RWRzXCmOJCy6dqItPJFpw2cuCangPK4OYr5uhGKcmrm5Qg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-generator-functions": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-generator-functions/-/plugin-transform-async-generator-functions-7.25.9.tgz", + "integrity": "sha512-RXV6QAzTBbhDMO9fWwOmwwTuYaiPbggWQ9INdZqAYeSHyG7FzQ+nOZaUUjNwKv9pV3aE4WFqFm1Hnbci5tBCAw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-remap-async-to-generator": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-async-to-generator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-async-to-generator/-/plugin-transform-async-to-generator-7.25.9.tgz", + "integrity": "sha512-NT7Ejn7Z/LjUH0Gv5KsBCxh7BH3fbLTV0ptHvpeMvrt3cPThHfJfst9Wrb7S8EvJ7vRTFI7z+VAvFVEQn/m5zQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-remap-async-to-generator": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoped-functions": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoped-functions/-/plugin-transform-block-scoped-functions-7.26.5.tgz", + "integrity": "sha512-chuTSY+hq09+/f5lMj8ZSYgCFpppV2CbYrhNFJ1BFoXpiWPnnAb7R0MqrafCpN8E1+YRrtM1MXZHJdIx8B6rMQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.26.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-block-scoping": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-block-scoping/-/plugin-transform-block-scoping-7.25.9.tgz", + "integrity": "sha512-1F05O7AYjymAtqbsFETboN1NvBdcnzMerO+zlMyJBEz6WkMdejvGWw9p05iTSjC85RLlBseHHQpYaM4gzJkBGg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-class-properties": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-properties/-/plugin-transform-class-properties-7.25.9.tgz", + "integrity": "sha512-bbMAII8GRSkcd0h0b4X+36GksxuheLFjP65ul9w6C3KgAamI3JqErNgSrosX6ZPj+Mpim5VvEbawXxJCyEUV3Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-class-static-block": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-class-static-block/-/plugin-transform-class-static-block-7.26.0.tgz", + "integrity": "sha512-6J2APTs7BDDm+UMqP1useWqhcRAXo0WIoVj26N7kPFB6S73Lgvyka4KTZYIxtgYXiN5HTyRObA72N2iu628iTQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.12.0" + } + }, + "node_modules/@babel/plugin-transform-classes": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.25.9.tgz", + "integrity": "sha512-mD8APIXmseE7oZvZgGABDyM34GUmK45Um2TXiBUt7PnuAxrgoSVf123qUzPxEr/+/BHrRn5NMZCdE2m/1F8DGg==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-replace-supers": "^7.25.9", + "@babel/traverse": "^7.25.9", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-computed-properties": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-computed-properties/-/plugin-transform-computed-properties-7.25.9.tgz", + "integrity": "sha512-HnBegGqXZR12xbcTHlJ9HGxw1OniltT26J5YpfruGqtUHlz/xKf/G2ak9e+t0rVqrjXa9WOhvYPz1ERfMj23AA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/template": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-destructuring": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.25.9.tgz", + "integrity": "sha512-WkCGb/3ZxXepmMiX101nnGiU+1CAdut8oHyEOHxkKuS1qKpU2SMXE2uSvfz8PBuLd49V6LEsbtyPhWC7fnkgvQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-dotall-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dotall-regex/-/plugin-transform-dotall-regex-7.25.9.tgz", + "integrity": "sha512-t7ZQ7g5trIgSRYhI9pIJtRl64KHotutUJsh4Eze5l7olJv+mRSg4/MmbZ0tv1eeqRbdvo/+trvJD/Oc5DmW2cA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-keys": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-keys/-/plugin-transform-duplicate-keys-7.25.9.tgz", + "integrity": "sha512-LZxhJ6dvBb/f3x8xwWIuyiAHy56nrRG3PeYTpBkkzkYRRQ6tJLu68lEF5VIqMUZiAV7a8+Tb78nEoMCMcqjXBw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-duplicate-named-capturing-groups-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-duplicate-named-capturing-groups-regex/-/plugin-transform-duplicate-named-capturing-groups-regex-7.25.9.tgz", + "integrity": "sha512-0UfuJS0EsXbRvKnwcLjFtJy/Sxc5J5jhLHnFhy7u4zih97Hz6tJkLU+O+FMMrNZrosUPxDi6sYxJ/EA8jDiAog==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-dynamic-import": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-dynamic-import/-/plugin-transform-dynamic-import-7.25.9.tgz", + "integrity": "sha512-GCggjexbmSLaFhqsojeugBpeaRIgWNTcgKVq/0qIteFEqY2A+b9QidYadrWlnbWQUrW5fn+mCvf3tr7OeBFTyg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-exponentiation-operator": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-exponentiation-operator/-/plugin-transform-exponentiation-operator-7.26.3.tgz", + "integrity": "sha512-7CAHcQ58z2chuXPWblnn1K6rLDnDWieghSOEmqQsrBenH0P9InCUtOJYD89pvngljmZlJcz3fcmgYsXFNGa1ZQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-export-namespace-from": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-export-namespace-from/-/plugin-transform-export-namespace-from-7.25.9.tgz", + "integrity": "sha512-2NsEz+CxzJIVOPx2o9UsW1rXLqtChtLoVnwYHHiB04wS5sgn7mrV45fWMBX0Kk+ub9uXytVYfNP2HjbVbCB3Ww==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-flow-strip-types": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-flow-strip-types/-/plugin-transform-flow-strip-types-7.26.5.tgz", + "integrity": "sha512-eGK26RsbIkYUns3Y8qKl362juDDYK+wEdPGHGrhzUl6CewZFo55VZ7hg+CyMFU4dd5QQakBN86nBMpRsFpRvbQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.26.5", + "@babel/plugin-syntax-flow": "^7.26.0" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-for-of": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-for-of/-/plugin-transform-for-of-7.25.9.tgz", + "integrity": "sha512-LqHxduHoaGELJl2uhImHwRQudhCM50pT46rIBNvtT/Oql3nqiS3wOwP+5ten7NpYSXrrVLgtZU3DZmPtWZo16A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-function-name": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-function-name/-/plugin-transform-function-name-7.25.9.tgz", + "integrity": "sha512-8lP+Yxjv14Vc5MuWBpJsoUCd3hD6V9DgBon2FVYL4jJgbnVQ9fTgYmonchzZJOVNgzEgbxp4OwAf6xz6M/14XA==", + "license": "MIT", + "dependencies": { + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-json-strings": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-json-strings/-/plugin-transform-json-strings-7.25.9.tgz", + "integrity": "sha512-xoTMk0WXceiiIvsaquQQUaLLXSW1KJ159KP87VilruQm0LNNGxWzahxSS6T6i4Zg3ezp4vA4zuwiNUR53qmQAw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-literals/-/plugin-transform-literals-7.25.9.tgz", + "integrity": "sha512-9N7+2lFziW8W9pBl2TzaNht3+pgMIRP74zizeCSrtnSKVdUl8mAjjOP2OOVQAfZ881P2cNjDj1uAMEdeD50nuQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-logical-assignment-operators": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-logical-assignment-operators/-/plugin-transform-logical-assignment-operators-7.25.9.tgz", + "integrity": "sha512-wI4wRAzGko551Y8eVf6iOY9EouIDTtPb0ByZx+ktDGHwv6bHFimrgJM/2T021txPZ2s4c7bqvHbd+vXG6K948Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-member-expression-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-member-expression-literals/-/plugin-transform-member-expression-literals-7.25.9.tgz", + "integrity": "sha512-PYazBVfofCQkkMzh2P6IdIUaCEWni3iYEerAsRWuVd8+jlM1S9S9cz1dF9hIzyoZ8IA3+OwVYIp9v9e+GbgZhA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-amd": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-amd/-/plugin-transform-modules-amd-7.25.9.tgz", + "integrity": "sha512-g5T11tnI36jVClQlMlt4qKDLlWnG5pP9CSM4GhdRciTNMRgkfpo5cR6b4rGIOYPgRRuFAvwjPQ/Yk+ql4dyhbw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.26.3.tgz", + "integrity": "sha512-MgR55l4q9KddUDITEzEFYn5ZsGDXMSsU9E+kh7fjRXTIC3RHqfCo8RPRbyReYJh44HQ/yomFkqbOFohXvDCiIQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-systemjs": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.25.9.tgz", + "integrity": "sha512-hyss7iIlH/zLHaehT+xwiymtPOpsiwIIRlCAOwBB04ta5Tt+lNItADdlXw3jAWZ96VJ2jlhl/c+PNIQPKNfvcA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-modules-umd": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-umd/-/plugin-transform-modules-umd-7.25.9.tgz", + "integrity": "sha512-bS9MVObUgE7ww36HEfwe6g9WakQ0KF07mQF74uuXdkoziUPfKyu/nIm663kz//e5O1nPInPFx36z7WJmJ4yNEw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.25.9.tgz", + "integrity": "sha512-oqB6WHdKTGl3q/ItQhpLSnWWOpjUJLsOCLVyeFgeTktkBSCiurvPOsyt93gibI9CmuKvTUEtWmG5VhZD+5T/KA==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-new-target": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.25.9.tgz", + "integrity": "sha512-U/3p8X1yCSoKyUj2eOBIx3FOn6pElFOKvAAGf8HTtItuPyB+ZeOqfn+mvTtg9ZlOAjsPdK3ayQEjqHjU/yLeVQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-nullish-coalescing-operator": { + "version": "7.26.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-nullish-coalescing-operator/-/plugin-transform-nullish-coalescing-operator-7.26.6.tgz", + "integrity": "sha512-CKW8Vu+uUZneQCPtXmSBUC6NCAUdya26hWCElAWh5mVSlSRsmiCPUUDKb3Z0szng1hiAJa098Hkhg9o4SE35Qw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.26.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-numeric-separator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-numeric-separator/-/plugin-transform-numeric-separator-7.25.9.tgz", + "integrity": "sha512-TlprrJ1GBZ3r6s96Yq8gEQv82s8/5HnCVHtEJScUj90thHQbwe+E5MLhi2bbNHBEJuzrvltXSru+BUxHDoog7Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-rest-spread": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-rest-spread/-/plugin-transform-object-rest-spread-7.25.9.tgz", + "integrity": "sha512-fSaXafEE9CVHPweLYw4J0emp1t8zYTXyzN3UuG+lylqkvYd7RMrsOQ8TYx5RF231be0vqtFC6jnx3UmpJmKBYg==", + "license": "MIT", + "dependencies": { + "@babel/helper-compilation-targets": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/plugin-transform-parameters": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-object-super": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-object-super/-/plugin-transform-object-super-7.25.9.tgz", + "integrity": "sha512-Kj/Gh+Rw2RNLbCK1VAWj2U48yxxqL2x0k10nPtSdRa0O2xnHXalD0s+o1A6a0W43gJ00ANo38jxkQreckOzv5A==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-replace-supers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-optional-catch-binding": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-catch-binding/-/plugin-transform-optional-catch-binding-7.25.9.tgz", + "integrity": "sha512-qM/6m6hQZzDcZF3onzIhZeDHDO43bkNNlOX0i8n3lR6zLbu0GN2d8qfM/IERJZYauhAHSLHy39NF0Ctdvcid7g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-optional-chaining": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-optional-chaining/-/plugin-transform-optional-chaining-7.25.9.tgz", + "integrity": "sha512-6AvV0FsLULbpnXeBjrY4dmWF8F7gf8QnvTEoO/wX/5xm/xE1Xo8oPuD3MPS+KS9f9XBEAWN7X1aWr4z9HdOr7A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-parameters": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.25.9.tgz", + "integrity": "sha512-wzz6MKwpnshBAiRmn4jR8LYz/g8Ksg0o80XmwZDlordjwEk9SxBzTWC7F5ef1jhbrbOW2DJ5J6ayRukrJmnr0g==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-private-methods": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-methods/-/plugin-transform-private-methods-7.25.9.tgz", + "integrity": "sha512-D/JUozNpQLAPUVusvqMxyvjzllRaF8/nSrP1s2YGQT/W4LHK4xxsMcHjhOGTS01mp9Hda8nswb+FblLdJornQw==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-private-property-in-object": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-private-property-in-object/-/plugin-transform-private-property-in-object-7.25.9.tgz", + "integrity": "sha512-Evf3kcMqzXA3xfYJmZ9Pg1OvKdtqsDMSWBDzZOPLvHiTt36E75jLDQo5w1gtRU95Q4E5PDttrTf25Fw8d/uWLw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-property-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-property-literals/-/plugin-transform-property-literals-7.25.9.tgz", + "integrity": "sha512-IvIUeV5KrS/VPavfSM/Iu+RE6llrHrYIKY1yfCzyO/lMXHQ+p7uGhonmGVisv6tSBSVgWzMBohTcvkC9vQcQFA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-display-name": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.25.9.tgz", + "integrity": "sha512-KJfMlYIUxQB1CJfO3e0+h0ZHWOTLCPP115Awhaz8U0Zpq36Gl/cXlpoyMRnUWlhNUBAzldnCiAZNvCDj7CrKxQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.25.9.tgz", + "integrity": "sha512-s5XwpQYCqGerXl+Pu6VDL3x0j2d82eiV77UJ8a2mDHAW7j9SWRqQ2y1fNo1Z74CdcYipl5Z41zvjj4Nfzq36rw==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/plugin-syntax-jsx": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-development": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-development/-/plugin-transform-react-jsx-development-7.25.9.tgz", + "integrity": "sha512-9mj6rm7XVYs4mdLIpbZnHOYdpW42uoiBCTVowg7sP1thUOiANgMb4UtpRivR0pp5iL+ocvUv7X4mZgFRpJEzGw==", + "license": "MIT", + "dependencies": { + "@babel/plugin-transform-react-jsx": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.25.9.tgz", + "integrity": "sha512-y8quW6p0WHkEhmErnfe58r7x0A70uKphQm8Sp8cV7tjNQwK56sNVK0M73LK3WuYmsuyrftut4xAkjjgU0twaMg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.25.9.tgz", + "integrity": "sha512-+iqjT8xmXhhYv4/uiYd8FNQsraMFZIfxVSqxxVSZP0WbbSAWvBXAul0m/zu+7Vv4O/3WtApy9pmaTMiumEZgfg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-pure-annotations": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-pure-annotations/-/plugin-transform-react-pure-annotations-7.25.9.tgz", + "integrity": "sha512-KQ/Takk3T8Qzj5TppkS1be588lkbTp5uj7w6a0LeQaTMSckU/wK0oJ/pih+T690tkgI5jfmg2TqDJvd41Sj1Cg==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regenerator": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.25.9.tgz", + "integrity": "sha512-vwDcDNsgMPDGP0nMqzahDWE5/MLcX8sv96+wfX7as7LoF/kr97Bo/7fI00lXY4wUXYfVmwIIyG80fGZ1uvt2qg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "regenerator-transform": "^0.15.2" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-regexp-modifiers": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regexp-modifiers/-/plugin-transform-regexp-modifiers-7.26.0.tgz", + "integrity": "sha512-vN6saax7lrA2yA/Pak3sCxuD6F5InBjn9IcrIKQPjpsLvuHYLVroTxjdlVRHjjBWxKOqIwpTXDkOssYT4BFdRw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/plugin-transform-reserved-words": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-reserved-words/-/plugin-transform-reserved-words-7.25.9.tgz", + "integrity": "sha512-7DL7DKYjn5Su++4RXu8puKZm2XBPHyjWLUidaPEkCUBbE7IPcsrkRHggAOOKydH1dASWdcUBxrkOGNxUv5P3Jg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-runtime": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.25.9.tgz", + "integrity": "sha512-nZp7GlEl+yULJrClz0SwHPqir3lc0zsPrDHQUcxGspSL7AKrexNSEfTbfqnDNJUO13bgKyfuOLMF8Xqtu8j3YQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9", + "babel-plugin-polyfill-corejs2": "^0.4.10", + "babel-plugin-polyfill-corejs3": "^0.10.6", + "babel-plugin-polyfill-regenerator": "^0.6.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-runtime/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/plugin-transform-shorthand-properties": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.25.9.tgz", + "integrity": "sha512-MUv6t0FhO5qHnS/W8XCbHmiRWOphNufpE1IVxhK5kuN3Td9FT1x4rx4K42s3RYdMXCXpfWkGSbCSd0Z64xA7Ng==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-spread": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-spread/-/plugin-transform-spread-7.25.9.tgz", + "integrity": "sha512-oNknIB0TbURU5pqJFVbOOFspVlrpVwo2H1+HUIsVDvp5VauGGDP1ZEvO8Nn5xyMEs3dakajOxlmkNW7kNgSm6A==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-sticky-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-sticky-regex/-/plugin-transform-sticky-regex-7.25.9.tgz", + "integrity": "sha512-WqBUSgeVwucYDP9U/xNRQam7xV8W5Zf+6Eo7T2SRVUFlhRiMNFdFz58u0KZmCVVqs2i7SHgpRnAhzRNmKfi2uA==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-template-literals": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-template-literals/-/plugin-transform-template-literals-7.25.9.tgz", + "integrity": "sha512-o97AE4syN71M/lxrCtQByzphAdlYluKPDBzDVzMmfCobUjjhAryZV0AIpRPrxN0eAkxXO6ZLEScmt+PNhj2OTw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typeof-symbol": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typeof-symbol/-/plugin-transform-typeof-symbol-7.26.7.tgz", + "integrity": "sha512-jfoTXXZTgGg36BmhqT3cAYK5qkmqvJpvNrPhaK/52Vgjhw4Rq29s9UqpWWV0D6yuRmgiFH/BUVlkl96zJWqnaw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.26.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typescript": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.26.7.tgz", + "integrity": "sha512-5cJurntg+AT+cgelGP9Bt788DKiAw9gIMSMU2NJrLAilnj0m8WZWUNZPSLOmadYsujHutpgElO+50foX+ib/Wg==", + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.25.9", + "@babel/helper-create-class-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.26.5", + "@babel/helper-skip-transparent-expression-wrappers": "^7.25.9", + "@babel/plugin-syntax-typescript": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-escapes": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-escapes/-/plugin-transform-unicode-escapes-7.25.9.tgz", + "integrity": "sha512-s5EDrE6bW97LtxOcGj1Khcx5AaXwiMmi4toFWRDP9/y0Woo6pXC+iyPu/KuhKtfSrNFd7jJB+/fkOtZy6aIC6Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-property-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-property-regex/-/plugin-transform-unicode-property-regex-7.25.9.tgz", + "integrity": "sha512-Jt2d8Ga+QwRluxRQ307Vlxa6dMrYEMZCgGxoPR8V52rxPyldHu3hdlHspxaqYmE7oID5+kB+UKUB/eWS+DkkWg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-regex/-/plugin-transform-unicode-regex-7.25.9.tgz", + "integrity": "sha512-yoxstj7Rg9dlNn9UQxzk4fcNivwv4nUYz7fYXBaKxvw/lnmPuOm/ikoELygbYq68Bls3D/D+NBPHiLwZdZZ4HA==", + "license": "MIT", + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-unicode-sets-regex": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-unicode-sets-regex/-/plugin-transform-unicode-sets-regex-7.25.9.tgz", + "integrity": "sha512-8BYqO3GeVNHtx69fdPshN3fnzUNLrWdHhk/icSwigksJGczKSizZ+Z6SBCxTs723Fr5VSNorTIK7a+R2tISvwQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-create-regexp-features-plugin": "^7.25.9", + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/preset-env": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.26.7.tgz", + "integrity": "sha512-Ycg2tnXwixaXOVb29rana8HNPgLVBof8qqtNQ9LE22IoyZboQbGSxI6ZySMdW3K5nAe6gu35IaJefUJflhUFTQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/compat-data": "^7.26.5", + "@babel/helper-compilation-targets": "^7.26.5", + "@babel/helper-plugin-utils": "^7.26.5", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-bugfix-firefox-class-in-computed-class-key": "^7.25.9", + "@babel/plugin-bugfix-safari-class-field-initializer-scope": "^7.25.9", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "^7.25.9", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "^7.25.9", + "@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly": "^7.25.9", + "@babel/plugin-proposal-private-property-in-object": "7.21.0-placeholder-for-preset-env.2", + "@babel/plugin-syntax-import-assertions": "^7.26.0", + "@babel/plugin-syntax-import-attributes": "^7.26.0", + "@babel/plugin-syntax-unicode-sets-regex": "^7.18.6", + "@babel/plugin-transform-arrow-functions": "^7.25.9", + "@babel/plugin-transform-async-generator-functions": "^7.25.9", + "@babel/plugin-transform-async-to-generator": "^7.25.9", + "@babel/plugin-transform-block-scoped-functions": "^7.26.5", + "@babel/plugin-transform-block-scoping": "^7.25.9", + "@babel/plugin-transform-class-properties": "^7.25.9", + "@babel/plugin-transform-class-static-block": "^7.26.0", + "@babel/plugin-transform-classes": "^7.25.9", + "@babel/plugin-transform-computed-properties": "^7.25.9", + "@babel/plugin-transform-destructuring": "^7.25.9", + "@babel/plugin-transform-dotall-regex": "^7.25.9", + "@babel/plugin-transform-duplicate-keys": "^7.25.9", + "@babel/plugin-transform-duplicate-named-capturing-groups-regex": "^7.25.9", + "@babel/plugin-transform-dynamic-import": "^7.25.9", + "@babel/plugin-transform-exponentiation-operator": "^7.26.3", + "@babel/plugin-transform-export-namespace-from": "^7.25.9", + "@babel/plugin-transform-for-of": "^7.25.9", + "@babel/plugin-transform-function-name": "^7.25.9", + "@babel/plugin-transform-json-strings": "^7.25.9", + "@babel/plugin-transform-literals": "^7.25.9", + "@babel/plugin-transform-logical-assignment-operators": "^7.25.9", + "@babel/plugin-transform-member-expression-literals": "^7.25.9", + "@babel/plugin-transform-modules-amd": "^7.25.9", + "@babel/plugin-transform-modules-commonjs": "^7.26.3", + "@babel/plugin-transform-modules-systemjs": "^7.25.9", + "@babel/plugin-transform-modules-umd": "^7.25.9", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.25.9", + "@babel/plugin-transform-new-target": "^7.25.9", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.26.6", + "@babel/plugin-transform-numeric-separator": "^7.25.9", + "@babel/plugin-transform-object-rest-spread": "^7.25.9", + "@babel/plugin-transform-object-super": "^7.25.9", + "@babel/plugin-transform-optional-catch-binding": "^7.25.9", + "@babel/plugin-transform-optional-chaining": "^7.25.9", + "@babel/plugin-transform-parameters": "^7.25.9", + "@babel/plugin-transform-private-methods": "^7.25.9", + "@babel/plugin-transform-private-property-in-object": "^7.25.9", + "@babel/plugin-transform-property-literals": "^7.25.9", + "@babel/plugin-transform-regenerator": "^7.25.9", + "@babel/plugin-transform-regexp-modifiers": "^7.26.0", + "@babel/plugin-transform-reserved-words": "^7.25.9", + "@babel/plugin-transform-shorthand-properties": "^7.25.9", + "@babel/plugin-transform-spread": "^7.25.9", + "@babel/plugin-transform-sticky-regex": "^7.25.9", + "@babel/plugin-transform-template-literals": "^7.25.9", + "@babel/plugin-transform-typeof-symbol": "^7.26.7", + "@babel/plugin-transform-unicode-escapes": "^7.25.9", + "@babel/plugin-transform-unicode-property-regex": "^7.25.9", + "@babel/plugin-transform-unicode-regex": "^7.25.9", + "@babel/plugin-transform-unicode-sets-regex": "^7.25.9", + "@babel/preset-modules": "0.1.6-no-external-plugins", + "babel-plugin-polyfill-corejs2": "^0.4.10", + "babel-plugin-polyfill-corejs3": "^0.10.6", + "babel-plugin-polyfill-regenerator": "^0.6.1", + "core-js-compat": "^3.38.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-env/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "peer": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/preset-flow": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/preset-flow/-/preset-flow-7.25.9.tgz", + "integrity": "sha512-EASHsAhE+SSlEzJ4bzfusnXSHiU+JfAYzj+jbw2vgQKgq5HrUr8qs+vgtiEL5dOH6sEweI+PNt2D7AqrDSHyqQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-transform-flow-strip-types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-modules": { + "version": "0.1.6-no-external-plugins", + "resolved": "https://registry.npmjs.org/@babel/preset-modules/-/preset-modules-0.1.6-no-external-plugins.tgz", + "integrity": "sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==", + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@babel/types": "^7.4.4", + "esutils": "^2.0.2" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/@babel/preset-react": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.26.3.tgz", + "integrity": "sha512-Nl03d6T9ky516DGK2YMxrTqvnpUW63TnJMOMonj+Zae0JiPC5BC9xPMSL6L8fiSpA5vP88qfygavVQvnLp+6Cw==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-transform-react-display-name": "^7.25.9", + "@babel/plugin-transform-react-jsx": "^7.25.9", + "@babel/plugin-transform-react-jsx-development": "^7.25.9", + "@babel/plugin-transform-react-pure-annotations": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/preset-typescript": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/preset-typescript/-/preset-typescript-7.26.0.tgz", + "integrity": "sha512-NMk1IGZ5I/oHhoXEElcm+xUnL/szL6xflkFZmoEU9xj1qSJXpiS7rsspYo92B4DRCDvZn2erT5LdsCeXAKNCkg==", + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9", + "@babel/helper-validator-option": "^7.25.9", + "@babel/plugin-syntax-jsx": "^7.25.9", + "@babel/plugin-transform-modules-commonjs": "^7.25.9", + "@babel/plugin-transform-typescript": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/register": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.25.9.tgz", + "integrity": "sha512-8D43jXtGsYmEeDvm4MWHYUpWf8iiXgWYx3fW7E7Wb7Oe6FWqJPl5K6TuFW0dOwNZzEE5rjlaSJYH9JjrUKJszA==", + "license": "MIT", + "dependencies": { + "clone-deep": "^4.0.1", + "find-cache-dir": "^2.0.0", + "make-dir": "^2.1.0", + "pirates": "^4.0.6", + "source-map-support": "^0.5.16" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.7.tgz", + "integrity": "sha512-AOPI3D+a8dXnja+iwsUqGRjr1BbZIe771sXdapOtYI531gSqpi92vXivKcq2asu/DFpdl1ceFAKZyRzK2PCVcQ==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/runtime/node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, + "node_modules/@babel/template": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.7.tgz", + "integrity": "sha512-1x1sgeyRLC3r5fQOM0/xtQKsYjyxmFjaOrLJNtZ81inNjyJHGIolTULPiSc/2qe1/qfpFLisLQYFnnZl7QoedA==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.5", + "@babel/parser": "^7.26.7", + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.7", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse--for-generate-function-map": { + "name": "@babel/traverse", + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.7.tgz", + "integrity": "sha512-1x1sgeyRLC3r5fQOM0/xtQKsYjyxmFjaOrLJNtZ81inNjyJHGIolTULPiSc/2qe1/qfpFLisLQYFnnZl7QoedA==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.5", + "@babel/parser": "^7.26.7", + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.7", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse--for-generate-function-map/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/traverse/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@babel/types": { + "version": "7.26.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.7.tgz", + "integrity": "sha512-t8kDRGrKXyp6+tjUh7hw2RLyclsW4TRoRvRHtSyAX9Bb5ldlFh+90YAYY6awRXrlB4G5G2izNeGySpATlFzmOg==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@config-plugins/react-native-webrtc": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@config-plugins/react-native-webrtc/-/react-native-webrtc-10.0.0.tgz", + "integrity": "sha512-q6owBOwQo3HRx4/b0FteE06Ymlcx7pK5bw+Stg77wgTWyxWAJ90yfVvvdMckzxuxMwDd78o9yCLKIONTulHD4A==", + "license": "MIT", + "peerDependencies": { + "expo": "^52" + } + }, + "node_modules/@daily-co/config-plugin-rn-daily-js": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/@daily-co/config-plugin-rn-daily-js/-/config-plugin-rn-daily-js-0.0.7.tgz", + "integrity": "sha512-8j6itEb2sxkxPDOnaO0FKpGIKvbvtLho0l25CdS01aa4VEAUKHWrxyUO6OVQkt2btfifsugBD6oUpO0X1fCbKQ==", + "dependencies": { + "expo-build-properties": "~0.8.3" + }, + "peerDependencies": { + "expo": "^52.0.0" + } + }, + "node_modules/@daily-co/config-plugin-rn-daily-js/node_modules/expo-build-properties": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/expo-build-properties/-/expo-build-properties-0.8.3.tgz", + "integrity": "sha512-kEDDuAadHqJTkvCGK4fXYHVrePiJO1DjyW95AicmwuGwQvGJydYFbuoauf9ybAU+4UH4arhbce8gHI3ZpIj3Jw==", + "license": "MIT", + "dependencies": { + "ajv": "^8.11.0", + "semver": "^7.5.3" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/@daily-co/daily-js": { + "version": "0.73.0", + "resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.73.0.tgz", + "integrity": "sha512-Wz8c60hgmkx8fcEeDAi4L4J0rbafiihWKyXFyhYoFYPsw2OdChHpA4RYwIB+1enRws5IK+/HdmzFDYLQsB4A6w==", + "license": "BSD-2-Clause", + "dependencies": { + "@babel/runtime": "^7.12.5", + "@sentry/browser": "^8.33.1", + "bowser": "^2.8.1", + "dequal": "^2.0.3", + "events": "^3.1.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@daily-co/react-native-daily-js": { + "version": "0.70.0", + "resolved": "https://registry.npmjs.org/@daily-co/react-native-daily-js/-/react-native-daily-js-0.70.0.tgz", + "integrity": "sha512-arHUJSQXs756XPq6PlddBcFDYgBMMg900B7KJhazr8tKBBsL4nb0bbz+WJfAAcsPOOJEamiFvlp9Qw3LlM3GfA==", + "license": "BSD-2-Clause", + "dependencies": { + "@daily-co/daily-js": "^0.73.0", + "@types/react-native-background-timer": "^2.0.0", + "base-64": "^1.0.0", + "react-native-url-polyfill": "^1.1.2" + }, + "peerDependencies": { + "@daily-co/react-native-webrtc": "^118.0.3-daily.2", + "@react-native-async-storage/async-storage": "^1.15.7", + "react-native-background-timer": "^2.3.1", + "react-native-get-random-values": "^1.9.0" + } + }, + "node_modules/@daily-co/react-native-webrtc": { + "version": "118.0.3-daily.2", + "resolved": "https://registry.npmjs.org/@daily-co/react-native-webrtc/-/react-native-webrtc-118.0.3-daily.2.tgz", + "integrity": "sha512-Ofwvnx0WL+Q21tQBJOWNKvV1gk/5kwPerwUCD7hCREuBDVRfSNtpRhQcuISNjmn7Z2eV405hgK0c9kOUu8vDQg==", + "license": "MIT", + "dependencies": { + "@types/react": "17.0.40", + "@types/react-native": "0.67.3", + "base64-js": "1.5.1", + "debug": "4.3.4", + "event-target-shim": "6.0.2" + }, + "peerDependencies": { + "react-native": ">=0.60.0" + } + }, + "node_modules/@daily-co/react-native-webrtc/node_modules/@types/react": { + "version": "17.0.40", + "resolved": "https://registry.npmjs.org/@types/react/-/react-17.0.40.tgz", + "integrity": "sha512-UrXhD/JyLH+W70nNSufXqMZNuUD2cXHu6UjCllC6pmOQgBX4SGXOH8fjRka0O0Ee0HrFxapDD8Bwn81Kmiz6jQ==", + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "@types/scheduler": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/@daily-co/react-native-webrtc/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "license": "MIT", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@daily-co/react-native-webrtc/node_modules/event-target-shim": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-6.0.2.tgz", + "integrity": "sha512-8q3LsZjRezbFZ2PN+uP+Q7pnHUMmAOziU2vA2OwoFaKIXxlxl38IylhSSgUorWu/rf4er67w0ikBqjBFk/pomA==", + "license": "MIT", + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "node_modules/@daily-co/react-native-webrtc/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "license": "MIT" + }, + "node_modules/@expo/bunyan": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@expo/bunyan/-/bunyan-4.0.1.tgz", + "integrity": "sha512-+Lla7nYSiHZirgK+U/uYzsLv/X+HaJienbD5AKX1UQZHYfWaP+9uuQluRB4GrEVWF0GZ7vEVp/jzaOT9k/SQlg==", + "engines": [ + "node >=0.10.0" + ], + "license": "MIT", + "dependencies": { + "uuid": "^8.0.0" + } + }, + "node_modules/@expo/cli": { + "version": "0.22.11", + "resolved": "https://registry.npmjs.org/@expo/cli/-/cli-0.22.11.tgz", + "integrity": "sha512-D5Vl7IBLi53WmL57NAFYB1mIqlMQxDIZVzbi/FTpo5a3oIHELKr0ElTKeOLf1f1/Y3FA7cxgphoawdA0+O1JWQ==", + "license": "MIT", + "dependencies": { + "@0no-co/graphql.web": "^1.0.8", + "@babel/runtime": "^7.20.0", + "@expo/code-signing-certificates": "^0.0.5", + "@expo/config": "~10.0.8", + "@expo/config-plugins": "~9.0.14", + "@expo/devcert": "^1.1.2", + "@expo/env": "~0.4.1", + "@expo/image-utils": "^0.6.4", + "@expo/json-file": "^9.0.1", + "@expo/metro-config": "~0.19.9", + "@expo/osascript": "^2.1.5", + "@expo/package-manager": "^1.7.1", + "@expo/plist": "^0.2.1", + "@expo/prebuild-config": "^8.0.25", + "@expo/rudder-sdk-node": "^1.1.1", + "@expo/spawn-async": "^1.7.2", + "@expo/xcpretty": "^4.3.0", + "@react-native/dev-middleware": "0.76.6", + "@urql/core": "^5.0.6", + "@urql/exchange-retry": "^1.3.0", + "accepts": "^1.3.8", + "arg": "^5.0.2", + "better-opn": "~3.0.2", + "bplist-creator": "0.0.7", + "bplist-parser": "^0.3.1", + "cacache": "^18.0.2", + "chalk": "^4.0.0", + "ci-info": "^3.3.0", + "compression": "^1.7.4", + "connect": "^3.7.0", + "debug": "^4.3.4", + "env-editor": "^0.4.1", + "fast-glob": "^3.3.2", + "form-data": "^3.0.1", + "freeport-async": "^2.0.0", + "fs-extra": "~8.1.0", + "getenv": "^1.0.0", + "glob": "^10.4.2", + "internal-ip": "^4.3.0", + "is-docker": "^2.0.0", + "is-wsl": "^2.1.1", + "lodash.debounce": "^4.0.8", + "minimatch": "^3.0.4", + "node-forge": "^1.3.1", + "npm-package-arg": "^11.0.0", + "ora": "^3.4.0", + "picomatch": "^3.0.1", + "pretty-bytes": "^5.6.0", + "pretty-format": "^29.7.0", + "progress": "^2.0.3", + "prompts": "^2.3.2", + "qrcode-terminal": "0.11.0", + "require-from-string": "^2.0.2", + "requireg": "^0.2.2", + "resolve": "^1.22.2", + "resolve-from": "^5.0.0", + "resolve.exports": "^2.0.3", + "semver": "^7.6.0", + "send": "^0.19.0", + "slugify": "^1.3.4", + "source-map-support": "~0.5.21", + "stacktrace-parser": "^0.1.10", + "structured-headers": "^0.4.1", + "tar": "^6.2.1", + "temp-dir": "^2.0.0", + "tempy": "^0.7.1", + "terminal-link": "^2.1.1", + "undici": "^6.18.2", + "unique-string": "~2.0.0", + "wrap-ansi": "^7.0.0", + "ws": "^8.12.1" + }, + "bin": { + "expo-internal": "build/bin/cli" + } + }, + "node_modules/@expo/cli/node_modules/@react-native/debugger-frontend": { + "version": "0.76.6", + "resolved": "https://registry.npmjs.org/@react-native/debugger-frontend/-/debugger-frontend-0.76.6.tgz", + "integrity": "sha512-kP97xMQjiANi5/lmf8MakS7d8FTJl+BqYHQMqyvNiY+eeWyKnhqW2GL2v3eEUBAuyPBgJGivuuO4RvjZujduJg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=18" + } + }, + "node_modules/@expo/cli/node_modules/@react-native/dev-middleware": { + "version": "0.76.6", + "resolved": "https://registry.npmjs.org/@react-native/dev-middleware/-/dev-middleware-0.76.6.tgz", + "integrity": "sha512-1bAyd2/X48Nzb45s5l2omM75vy764odx/UnDs4sJfFCuK+cupU4nRPgl0XWIqgdM/2+fbQ3E4QsVS/WIKTFxvQ==", + "license": "MIT", + "dependencies": { + "@isaacs/ttlcache": "^1.4.1", + "@react-native/debugger-frontend": "0.76.6", + "chrome-launcher": "^0.15.2", + "chromium-edge-launcher": "^0.2.0", + "connect": "^3.6.5", + "debug": "^2.2.0", + "nullthrows": "^1.1.1", + "open": "^7.0.3", + "selfsigned": "^2.4.1", + "serve-static": "^1.13.1", + "ws": "^6.2.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@expo/cli/node_modules/@react-native/dev-middleware/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/@expo/cli/node_modules/@react-native/dev-middleware/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/@expo/cli/node_modules/@react-native/dev-middleware/node_modules/ws": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.3.tgz", + "integrity": "sha512-jmTjYU0j60B+vHey6TfR3Z7RD61z/hmxBS3VMSGIrroOWXQEneK1zNuotOUrGyBHQj0yrpsLHPWtigEFd13ndA==", + "license": "MIT", + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/@expo/cli/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@expo/cli/node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/@expo/cli/node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "license": "MIT", + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/@expo/cli/node_modules/picomatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz", + "integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/@expo/cli/node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "license": "MIT", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/@expo/cli/node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/@expo/code-signing-certificates": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@expo/code-signing-certificates/-/code-signing-certificates-0.0.5.tgz", + "integrity": "sha512-BNhXkY1bblxKZpltzAx98G2Egj9g1Q+JRcvR7E99DOj862FTCX+ZPsAUtPTr7aHxwtrL7+fL3r0JSmM9kBm+Bw==", + "license": "MIT", + "dependencies": { + "node-forge": "^1.2.1", + "nullthrows": "^1.1.1" + } + }, + "node_modules/@expo/config": { + "version": "10.0.8", + "resolved": "https://registry.npmjs.org/@expo/config/-/config-10.0.8.tgz", + "integrity": "sha512-RaKwi8e6PbkMilRexdsxObLMdQwxhY6mlgel+l/eW+IfIw8HEydSU0ERlzYUjlGJxHLHUXe4rC2vw8FEvaowyQ==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "~7.10.4", + "@expo/config-plugins": "~9.0.14", + "@expo/config-types": "^52.0.3", + "@expo/json-file": "^9.0.1", + "deepmerge": "^4.3.1", + "getenv": "^1.0.0", + "glob": "^10.4.2", + "require-from-string": "^2.0.2", + "resolve-from": "^5.0.0", + "resolve-workspace-root": "^2.0.0", + "semver": "^7.6.0", + "slugify": "^1.3.4", + "sucrase": "3.35.0" + } + }, + "node_modules/@expo/config-plugins": { + "version": "9.0.14", + "resolved": "https://registry.npmjs.org/@expo/config-plugins/-/config-plugins-9.0.14.tgz", + "integrity": "sha512-Lx1ebV95rTFKKQmbu4wMPLz65rKn7mqSpfANdCx+KwRxuLY2JQls8V4h3lQjG6dW8NWf9qV5QaEFAgNB6VMyOQ==", + "license": "MIT", + "dependencies": { + "@expo/config-types": "^52.0.3", + "@expo/json-file": "~9.0.1", + "@expo/plist": "^0.2.1", + "@expo/sdk-runtime-versions": "^1.0.0", + "chalk": "^4.1.2", + "debug": "^4.3.5", + "getenv": "^1.0.0", + "glob": "^10.4.2", + "resolve-from": "^5.0.0", + "semver": "^7.5.4", + "slash": "^3.0.0", + "slugify": "^1.6.6", + "xcode": "^3.0.1", + "xml2js": "0.6.0" + } + }, + "node_modules/@expo/config-plugins/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@expo/config-types": { + "version": "52.0.3", + "resolved": "https://registry.npmjs.org/@expo/config-types/-/config-types-52.0.3.tgz", + "integrity": "sha512-muxvuARmbysH5OGaiBRlh1Y6vfdmL56JtpXxB+y2Hfhu0ezG1U4FjZYBIacthckZPvnDCcP3xIu1R+eTo7/QFA==", + "license": "MIT" + }, + "node_modules/@expo/config/node_modules/@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "license": "MIT", + "dependencies": { + "@babel/highlight": "^7.10.4" + } + }, + "node_modules/@expo/devcert": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@expo/devcert/-/devcert-1.1.4.tgz", + "integrity": "sha512-fqBODr8c72+gBSX5Ty3SIzaY4bXainlpab78+vEYEKL3fXmsOswMLf0+KE36mUEAa36BYabX7K3EiXOXX5OPMw==", + "license": "MIT", + "dependencies": { + "application-config-path": "^0.1.0", + "command-exists": "^1.2.4", + "debug": "^3.1.0", + "eol": "^0.9.1", + "get-port": "^3.2.0", + "glob": "^10.4.2", + "lodash": "^4.17.21", + "mkdirp": "^0.5.1", + "password-prompt": "^1.0.4", + "sudo-prompt": "^8.2.0", + "tmp": "^0.0.33", + "tslib": "^2.4.0" + } + }, + "node_modules/@expo/devcert/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/@expo/devcert/node_modules/sudo-prompt": { + "version": "8.2.5", + "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-8.2.5.tgz", + "integrity": "sha512-rlBo3HU/1zAJUrkY6jNxDOC9eVYliG6nS4JA8u8KAshITd07tafMc/Br7xQwCSseXwJ2iCcHCE8SNWX3q8Z+kw==", + "license": "MIT" + }, + "node_modules/@expo/env": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@expo/env/-/env-0.4.1.tgz", + "integrity": "sha512-oDtbO3i9yXD1nx93acWiPTWGljJ3vABn35x1NAbqtQ2JL6mFOcRcArt1dwi4imZyLnG4VCcjabT9irj+LgYntw==", + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "debug": "^4.3.4", + "dotenv": "~16.4.5", + "dotenv-expand": "~11.0.6", + "getenv": "^1.0.0" + } + }, + "node_modules/@expo/env/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@expo/fingerprint": { + "version": "0.11.7", + "resolved": "https://registry.npmjs.org/@expo/fingerprint/-/fingerprint-0.11.7.tgz", + "integrity": "sha512-2rfYVS4nqWmOPQk+AL5GPfPSawbqqmI5mL++bxAhWADt+d+fjoQYfIrGtjZxQ30f9o/a1PrRPVSuh2j09+diVg==", + "license": "MIT", + "dependencies": { + "@expo/spawn-async": "^1.7.2", + "arg": "^5.0.2", + "chalk": "^4.1.2", + "debug": "^4.3.4", + "find-up": "^5.0.0", + "getenv": "^1.0.0", + "minimatch": "^3.0.4", + "p-limit": "^3.1.0", + "resolve-from": "^5.0.0", + "semver": "^7.6.0" + }, + "bin": { + "fingerprint": "bin/cli.js" + } + }, + "node_modules/@expo/fingerprint/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@expo/fingerprint/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@expo/image-utils": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/@expo/image-utils/-/image-utils-0.6.4.tgz", + "integrity": "sha512-L++1PBzSvf5iYc6UHJ8Db8GcYNkfLDw+a+zqEFBQ3xqRXP/muxb/O7wuiMFlXrj/cfkx4e0U+z1a4ceV0A7S7Q==", + "license": "MIT", + "dependencies": { + "@expo/spawn-async": "^1.7.2", + "chalk": "^4.0.0", + "fs-extra": "9.0.0", + "getenv": "^1.0.0", + "jimp-compact": "0.16.1", + "parse-png": "^2.1.0", + "resolve-from": "^5.0.0", + "semver": "^7.6.0", + "temp-dir": "~2.0.0", + "unique-string": "~2.0.0" + } + }, + "node_modules/@expo/image-utils/node_modules/fs-extra": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.0.tgz", + "integrity": "sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g==", + "license": "MIT", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^1.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@expo/image-utils/node_modules/universalify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz", + "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@expo/json-file": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@expo/json-file/-/json-file-9.0.1.tgz", + "integrity": "sha512-ZVPhbbEBEwafPCJ0+kI25O2Iivt3XKHEKAADCml1q2cmOIbQnKgLyn8DpOJXqWEyRQr/VWS+hflBh8DU2YFSqg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "~7.10.4", + "json5": "^2.2.3", + "write-file-atomic": "^2.3.0" + } + }, + "node_modules/@expo/json-file/node_modules/@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "license": "MIT", + "dependencies": { + "@babel/highlight": "^7.10.4" + } + }, + "node_modules/@expo/metro-config": { + "version": "0.19.9", + "resolved": "https://registry.npmjs.org/@expo/metro-config/-/metro-config-0.19.9.tgz", + "integrity": "sha512-JAsLWhFQqwLH0KsI4OMbPXsKFji5KJEmsi+/02Sz1GCT17YrjRmv1fZ91regUS/FUH2Y/PDAE/+2ulrTgMeG7A==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.20.0", + "@babel/generator": "^7.20.5", + "@babel/parser": "^7.20.0", + "@babel/types": "^7.20.0", + "@expo/config": "~10.0.8", + "@expo/env": "~0.4.1", + "@expo/json-file": "~9.0.1", + "@expo/spawn-async": "^1.7.2", + "chalk": "^4.1.0", + "debug": "^4.3.2", + "fs-extra": "^9.1.0", + "getenv": "^1.0.0", + "glob": "^10.4.2", + "jsc-safe-url": "^0.2.4", + "lightningcss": "~1.27.0", + "minimatch": "^3.0.4", + "postcss": "~8.4.32", + "resolve-from": "^5.0.0" + } + }, + "node_modules/@expo/metro-config/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@expo/osascript": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@expo/osascript/-/osascript-2.1.5.tgz", + "integrity": "sha512-Cp7YF7msGiTAIbFdzNovwHBfecdMLVL5XzSqq4xQz72ALFCQ3uSIUXRph1QV2r61ugH7Yem0gY8yi7RcDlI4qg==", + "license": "MIT", + "dependencies": { + "@expo/spawn-async": "^1.7.2", + "exec-async": "^2.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@expo/package-manager": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/@expo/package-manager/-/package-manager-1.7.1.tgz", + "integrity": "sha512-DKbELrTOdl7U3KT0C07Aka9P+sUP3LL+1UTKf1KmLx2x2gPH1IC+c68N7iQlwNt+yA37qIw6/vKoqyTGu5EL9g==", + "license": "MIT", + "dependencies": { + "@expo/json-file": "^9.0.1", + "@expo/spawn-async": "^1.7.2", + "ansi-regex": "^5.0.0", + "chalk": "^4.0.0", + "find-up": "^5.0.0", + "js-yaml": "^3.13.1", + "micromatch": "^4.0.8", + "npm-package-arg": "^11.0.0", + "ora": "^3.4.0", + "resolve-workspace-root": "^2.0.0", + "split": "^1.0.1", + "sudo-prompt": "9.1.1" + } + }, + "node_modules/@expo/plist": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@expo/plist/-/plist-0.2.1.tgz", + "integrity": "sha512-9TaXGuNxa0LQwHQn4rYiU6YaERv6dPnQgsdKWq2rKKTr6LWOtGNQCi/yOk/HBLeZSxBm59APT5/6x60uRvr0Mg==", + "license": "MIT", + "dependencies": { + "@xmldom/xmldom": "~0.7.7", + "base64-js": "^1.2.3", + "xmlbuilder": "^14.0.0" + } + }, + "node_modules/@expo/prebuild-config": { + "version": "8.0.25", + "resolved": "https://registry.npmjs.org/@expo/prebuild-config/-/prebuild-config-8.0.25.tgz", + "integrity": "sha512-xYHV8eiydZEDedf2AGaOFRFwcGlaSzrqQH94dwX42urNCU03FO0RUb7yPp4nkb7WNFg5Ov6PDsV7ES+YwzNgYQ==", + "license": "MIT", + "dependencies": { + "@expo/config": "~10.0.8", + "@expo/config-plugins": "~9.0.14", + "@expo/config-types": "^52.0.3", + "@expo/image-utils": "^0.6.4", + "@expo/json-file": "^9.0.1", + "@react-native/normalize-colors": "0.76.6", + "debug": "^4.3.1", + "fs-extra": "^9.0.0", + "resolve-from": "^5.0.0", + "semver": "^7.6.0", + "xml2js": "0.6.0" + } + }, + "node_modules/@expo/prebuild-config/node_modules/@react-native/normalize-colors": { + "version": "0.76.6", + "resolved": "https://registry.npmjs.org/@react-native/normalize-colors/-/normalize-colors-0.76.6.tgz", + "integrity": "sha512-1n4udXH2Cla31iA/8eLRdhFHpYUYK1NKWCn4m1Sr9L4SarWKAYuRFliK1fcLvPPALCFoFlWvn8I0ekdUOHMzDQ==", + "license": "MIT" + }, + "node_modules/@expo/prebuild-config/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@expo/rudder-sdk-node": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@expo/rudder-sdk-node/-/rudder-sdk-node-1.1.1.tgz", + "integrity": "sha512-uy/hS/awclDJ1S88w9UGpc6Nm9XnNUjzOAAib1A3PVAnGQIwebg8DpFqOthFBTlZxeuV/BKbZ5jmTbtNZkp1WQ==", + "license": "MIT", + "dependencies": { + "@expo/bunyan": "^4.0.0", + "@segment/loosely-validate-event": "^2.0.0", + "fetch-retry": "^4.1.1", + "md5": "^2.2.1", + "node-fetch": "^2.6.1", + "remove-trailing-slash": "^0.1.0", + "uuid": "^8.3.2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@expo/sdk-runtime-versions": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@expo/sdk-runtime-versions/-/sdk-runtime-versions-1.0.0.tgz", + "integrity": "sha512-Doz2bfiPndXYFPMRwPyGa1k5QaKDVpY806UJj570epIiMzWaYyCtobasyfC++qfIXVb5Ocy7r3tP9d62hAQ7IQ==", + "license": "MIT" + }, + "node_modules/@expo/spawn-async": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@expo/spawn-async/-/spawn-async-1.7.2.tgz", + "integrity": "sha512-QdWi16+CHB9JYP7gma19OVVg0BFkvU8zNj9GjWorYI8Iv8FUxjOCcYRuAmX4s/h91e4e7BPsskc8cSrZYho9Ew==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@expo/vector-icons": { + "version": "14.0.4", + "resolved": "https://registry.npmjs.org/@expo/vector-icons/-/vector-icons-14.0.4.tgz", + "integrity": "sha512-+yKshcbpDfbV4zoXOgHxCwh7lkE9VVTT5T03OUlBsqfze1PLy6Hi4jp1vSb1GVbY6eskvMIivGVc9SKzIv0oEQ==", + "license": "MIT", + "dependencies": { + "prop-types": "^15.8.1" + } + }, + "node_modules/@expo/xcpretty": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@expo/xcpretty/-/xcpretty-4.3.2.tgz", + "integrity": "sha512-ReZxZ8pdnoI3tP/dNnJdnmAk7uLT4FjsKDGW7YeDdvdOMz2XCQSmSCM9IWlrXuWtMF9zeSB6WJtEhCQ41gQOfw==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/code-frame": "7.10.4", + "chalk": "^4.1.0", + "find-up": "^5.0.0", + "js-yaml": "^4.1.0" + }, + "bin": { + "excpretty": "build/cli.js" + } + }, + "node_modules/@expo/xcpretty/node_modules/@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "license": "MIT", + "dependencies": { + "@babel/highlight": "^7.10.4" + } + }, + "node_modules/@expo/xcpretty/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/@expo/xcpretty/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/ttlcache": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@isaacs/ttlcache/-/ttlcache-1.4.1.tgz", + "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/create-cache-key-function": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/create-cache-key-function/-/create-cache-key-function-29.7.0.tgz", + "integrity": "sha512-4QqS3LY5PBmTRHj9sAg1HLoPzqAI0uOX6wI/TRqHIcOxlFidy6YEmCQJk6FSZjNLGCeubDMfmkWL+qaLKhSGQA==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform/node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@npmcli/fs": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.1.tgz", + "integrity": "sha512-q9CRWjpHCMIh5sVyefoD1cA7PkvILqCZsnSOEUUivORLjxCO/Irmue2DprETiNgEqktDBZaM1Bi+jrarx1XdCg==", + "license": "ISC", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@react-native-async-storage/async-storage": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@react-native-async-storage/async-storage/-/async-storage-1.23.1.tgz", + "integrity": "sha512-Qd2kQ3yi6Y3+AcUlrHxSLlnBvpdCEMVGFlVBneVOjaFaPU61g1huc38g339ysXspwY1QZA2aNhrk/KlHGO+ewA==", + "license": "MIT", + "dependencies": { + "merge-options": "^3.0.4" + }, + "peerDependencies": { + "react-native": "^0.0.0-0 || >=0.60 <1.0" + } + }, + "node_modules/@react-native/assets-registry": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/assets-registry/-/assets-registry-0.76.3.tgz", + "integrity": "sha512-7Fnc3lzCFFpnoyL1egua6d/qUp0KiIpeSLbfOMln4nI2g2BMzyFHdPjJnpLV2NehmS0omOOkrfRqK5u1F/MXzA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/babel-plugin-codegen": { + "version": "0.76.6", + "resolved": "https://registry.npmjs.org/@react-native/babel-plugin-codegen/-/babel-plugin-codegen-0.76.6.tgz", + "integrity": "sha512-yFC9I/aDBOBz3ZMlqKn2NY/mDUtCksUNZ7AQmBiTAeVTUP0ujEjE0hTOx5Qd+kok7A7hwZEX87HdSgjiJZfr5g==", + "license": "MIT", + "dependencies": { + "@react-native/codegen": "0.76.6" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/babel-plugin-codegen/node_modules/@react-native/codegen": { + "version": "0.76.6", + "resolved": "https://registry.npmjs.org/@react-native/codegen/-/codegen-0.76.6.tgz", + "integrity": "sha512-BABb3e5G/+hyQYEYi0AODWh2km2d8ERoASZr6Hv90pVXdUHRYR+yxCatX7vSd9rnDUYndqRTzD0hZWAucPNAKg==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.25.3", + "glob": "^7.1.1", + "hermes-parser": "0.23.1", + "invariant": "^2.2.4", + "jscodeshift": "^0.14.0", + "mkdirp": "^0.5.1", + "nullthrows": "^1.1.1", + "yargs": "^17.6.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@babel/preset-env": "^7.1.6" + } + }, + "node_modules/@react-native/babel-plugin-codegen/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@react-native/babel-preset": { + "version": "0.76.6", + "resolved": "https://registry.npmjs.org/@react-native/babel-preset/-/babel-preset-0.76.6.tgz", + "integrity": "sha512-ojlVWY6S/VE/nb9hIRetPMTsW9ZmGb2R3dnToEXAtQQDz41eHMHXbkw/k2h0THp6qhas25ruNvn3N5n2o+lBzg==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.2", + "@babel/plugin-proposal-export-default-from": "^7.24.7", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-default-from": "^7.24.7", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-transform-arrow-functions": "^7.24.7", + "@babel/plugin-transform-async-generator-functions": "^7.25.4", + "@babel/plugin-transform-async-to-generator": "^7.24.7", + "@babel/plugin-transform-block-scoping": "^7.25.0", + "@babel/plugin-transform-class-properties": "^7.25.4", + "@babel/plugin-transform-classes": "^7.25.4", + "@babel/plugin-transform-computed-properties": "^7.24.7", + "@babel/plugin-transform-destructuring": "^7.24.8", + "@babel/plugin-transform-flow-strip-types": "^7.25.2", + "@babel/plugin-transform-for-of": "^7.24.7", + "@babel/plugin-transform-function-name": "^7.25.1", + "@babel/plugin-transform-literals": "^7.25.2", + "@babel/plugin-transform-logical-assignment-operators": "^7.24.7", + "@babel/plugin-transform-modules-commonjs": "^7.24.8", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.24.7", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.24.7", + "@babel/plugin-transform-numeric-separator": "^7.24.7", + "@babel/plugin-transform-object-rest-spread": "^7.24.7", + "@babel/plugin-transform-optional-catch-binding": "^7.24.7", + "@babel/plugin-transform-optional-chaining": "^7.24.8", + "@babel/plugin-transform-parameters": "^7.24.7", + "@babel/plugin-transform-private-methods": "^7.24.7", + "@babel/plugin-transform-private-property-in-object": "^7.24.7", + "@babel/plugin-transform-react-display-name": "^7.24.7", + "@babel/plugin-transform-react-jsx": "^7.25.2", + "@babel/plugin-transform-react-jsx-self": "^7.24.7", + "@babel/plugin-transform-react-jsx-source": "^7.24.7", + "@babel/plugin-transform-regenerator": "^7.24.7", + "@babel/plugin-transform-runtime": "^7.24.7", + "@babel/plugin-transform-shorthand-properties": "^7.24.7", + "@babel/plugin-transform-spread": "^7.24.7", + "@babel/plugin-transform-sticky-regex": "^7.24.7", + "@babel/plugin-transform-typescript": "^7.25.2", + "@babel/plugin-transform-unicode-regex": "^7.24.7", + "@babel/template": "^7.25.0", + "@react-native/babel-plugin-codegen": "0.76.6", + "babel-plugin-syntax-hermes-parser": "^0.25.1", + "babel-plugin-transform-flow-enums": "^0.0.2", + "react-refresh": "^0.14.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@babel/core": "*" + } + }, + "node_modules/@react-native/codegen": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/codegen/-/codegen-0.76.3.tgz", + "integrity": "sha512-oJCH/jbYeGmFJql8/y76gqWCCd74pyug41yzYAjREso1Z7xL88JhDyKMvxEnfhSdMOZYVl479N80xFiXPy3ZYA==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.25.3", + "glob": "^7.1.1", + "hermes-parser": "0.23.1", + "invariant": "^2.2.4", + "jscodeshift": "^0.14.0", + "mkdirp": "^0.5.1", + "nullthrows": "^1.1.1", + "yargs": "^17.6.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@babel/preset-env": "^7.1.6" + } + }, + "node_modules/@react-native/codegen/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@react-native/community-cli-plugin": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/community-cli-plugin/-/community-cli-plugin-0.76.3.tgz", + "integrity": "sha512-vgsLixHS24jR0d0QqPykBWFaC+V8x9cM3cs4oYXw3W199jgBNGP9MWcUJLazD2vzrT/lUTVBVg0rBeB+4XR6fg==", + "license": "MIT", + "dependencies": { + "@react-native/dev-middleware": "0.76.3", + "@react-native/metro-babel-transformer": "0.76.3", + "chalk": "^4.0.0", + "execa": "^5.1.1", + "invariant": "^2.2.4", + "metro": "^0.81.0", + "metro-config": "^0.81.0", + "metro-core": "^0.81.0", + "node-fetch": "^2.2.0", + "readline": "^1.3.0", + "semver": "^7.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@react-native-community/cli-server-api": "*" + }, + "peerDependenciesMeta": { + "@react-native-community/cli-server-api": { + "optional": true + } + } + }, + "node_modules/@react-native/debugger-frontend": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/debugger-frontend/-/debugger-frontend-0.76.3.tgz", + "integrity": "sha512-pMHQ3NpPB28RxXciSvm2yD+uDx3pkhzfuWkc7VFgOduyzPSIr0zotUiOJzsAtrj8++bPbOsAraCeQhCqoOTWQw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/dev-middleware": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/dev-middleware/-/dev-middleware-0.76.3.tgz", + "integrity": "sha512-b+2IpW40z1/S5Jo5JKrWPmucYU/PzeGyGBZZ/SJvmRnBDaP3txb9yIqNZAII1EWsKNhedh8vyRO5PSuJ9Juqzw==", + "license": "MIT", + "dependencies": { + "@isaacs/ttlcache": "^1.4.1", + "@react-native/debugger-frontend": "0.76.3", + "chrome-launcher": "^0.15.2", + "chromium-edge-launcher": "^0.2.0", + "connect": "^3.6.5", + "debug": "^2.2.0", + "nullthrows": "^1.1.1", + "open": "^7.0.3", + "selfsigned": "^2.4.1", + "serve-static": "^1.13.1", + "ws": "^6.2.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/gradle-plugin": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/gradle-plugin/-/gradle-plugin-0.76.3.tgz", + "integrity": "sha512-t0aYZ8ND7+yc+yIm6Yp52bInneYpki6RSIFZ9/LMUzgMKvEB62ptt/7sfho9QkKHCNxE1DJSWIqLIGi/iHHkyg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/js-polyfills": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/js-polyfills/-/js-polyfills-0.76.3.tgz", + "integrity": "sha512-pubJFArMMrdZiytH+W95KngcSQs+LsxOBsVHkwgMnpBfRUxXPMK4fudtBwWvhnwN76Oe+WhxSq7vOS5XgoPhmw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/metro-babel-transformer": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/metro-babel-transformer/-/metro-babel-transformer-0.76.3.tgz", + "integrity": "sha512-b2zQPXmW7avw/7zewc9nzMULPIAjsTwN03hskhxHUJH5pzUf7pIklB3FrgYPZrRhJgzHiNl3tOPu7vqiKzBYPg==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.2", + "@react-native/babel-preset": "0.76.3", + "hermes-parser": "0.23.1", + "nullthrows": "^1.1.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@babel/core": "*" + } + }, + "node_modules/@react-native/metro-babel-transformer/node_modules/@react-native/babel-plugin-codegen": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/babel-plugin-codegen/-/babel-plugin-codegen-0.76.3.tgz", + "integrity": "sha512-mZ7jmIIg4bUnxCqY3yTOkoHvvzsDyrZgfnIKiTGm5QACrsIGa5eT3pMFpMm2OpxGXRDrTMsYdPXE2rCyDX52VQ==", + "license": "MIT", + "dependencies": { + "@react-native/codegen": "0.76.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@react-native/metro-babel-transformer/node_modules/@react-native/babel-preset": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/babel-preset/-/babel-preset-0.76.3.tgz", + "integrity": "sha512-zi2nPlQf9q2fmfPyzwWEj6DU96v8ziWtEfG7CTAX2PG/Vjfsr94vn/wWrCdhBVvLRQ6Kvd/MFAuDYpxmQwIiVQ==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.2", + "@babel/plugin-proposal-export-default-from": "^7.24.7", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-syntax-export-default-from": "^7.24.7", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-transform-arrow-functions": "^7.24.7", + "@babel/plugin-transform-async-generator-functions": "^7.25.4", + "@babel/plugin-transform-async-to-generator": "^7.24.7", + "@babel/plugin-transform-block-scoping": "^7.25.0", + "@babel/plugin-transform-class-properties": "^7.25.4", + "@babel/plugin-transform-classes": "^7.25.4", + "@babel/plugin-transform-computed-properties": "^7.24.7", + "@babel/plugin-transform-destructuring": "^7.24.8", + "@babel/plugin-transform-flow-strip-types": "^7.25.2", + "@babel/plugin-transform-for-of": "^7.24.7", + "@babel/plugin-transform-function-name": "^7.25.1", + "@babel/plugin-transform-literals": "^7.25.2", + "@babel/plugin-transform-logical-assignment-operators": "^7.24.7", + "@babel/plugin-transform-modules-commonjs": "^7.24.8", + "@babel/plugin-transform-named-capturing-groups-regex": "^7.24.7", + "@babel/plugin-transform-nullish-coalescing-operator": "^7.24.7", + "@babel/plugin-transform-numeric-separator": "^7.24.7", + "@babel/plugin-transform-object-rest-spread": "^7.24.7", + "@babel/plugin-transform-optional-catch-binding": "^7.24.7", + "@babel/plugin-transform-optional-chaining": "^7.24.8", + "@babel/plugin-transform-parameters": "^7.24.7", + "@babel/plugin-transform-private-methods": "^7.24.7", + "@babel/plugin-transform-private-property-in-object": "^7.24.7", + "@babel/plugin-transform-react-display-name": "^7.24.7", + "@babel/plugin-transform-react-jsx": "^7.25.2", + "@babel/plugin-transform-react-jsx-self": "^7.24.7", + "@babel/plugin-transform-react-jsx-source": "^7.24.7", + "@babel/plugin-transform-regenerator": "^7.24.7", + "@babel/plugin-transform-runtime": "^7.24.7", + "@babel/plugin-transform-shorthand-properties": "^7.24.7", + "@babel/plugin-transform-spread": "^7.24.7", + "@babel/plugin-transform-sticky-regex": "^7.24.7", + "@babel/plugin-transform-typescript": "^7.25.2", + "@babel/plugin-transform-unicode-regex": "^7.24.7", + "@babel/template": "^7.25.0", + "@react-native/babel-plugin-codegen": "0.76.3", + "babel-plugin-syntax-hermes-parser": "^0.25.1", + "babel-plugin-transform-flow-enums": "^0.0.2", + "react-refresh": "^0.14.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@babel/core": "*" + } + }, + "node_modules/@react-native/normalize-colors": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/normalize-colors/-/normalize-colors-0.76.3.tgz", + "integrity": "sha512-Yrpmrh4IDEupUUM/dqVxhAN8QW1VEUR3Qrk2lzJC1jB2s46hDe0hrMP2vs12YJqlzshteOthjwXQlY0TgIzgbg==", + "license": "MIT" + }, + "node_modules/@react-native/virtualized-lists": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/@react-native/virtualized-lists/-/virtualized-lists-0.76.3.tgz", + "integrity": "sha512-wTGv9pVh3vAOWb29xFm+J9VRe9dUcUcb9FyaMLT/Hxa88W4wqa5ZMe1V9UvrrBiA1G5DKjv8/1ZcDsJhyugVKA==", + "license": "MIT", + "dependencies": { + "invariant": "^2.2.4", + "nullthrows": "^1.1.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/react": "^18.2.6", + "react": "*", + "react-native": "*" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@segment/loosely-validate-event": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@segment/loosely-validate-event/-/loosely-validate-event-2.0.0.tgz", + "integrity": "sha512-ZMCSfztDBqwotkl848ODgVcAmN4OItEWDCkshcKz0/W6gGSQayuuCtWV/MlodFivAZD793d6UgANd6wCXUfrIw==", + "dependencies": { + "component-type": "^1.2.1", + "join-component": "^1.1.0" + } + }, + "node_modules/@sentry-internal/browser-utils": { + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.52.0.tgz", + "integrity": "sha512-ojFldpRpGrgacIQMbbMZeqLYetNJJ61n+Pz29FpggaIRrbkq84ocoy4FCy+9BuLo6ywgxtUFrjOXD9pPRcZtUA==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.52.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/feedback": { + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.52.0.tgz", + "integrity": "sha512-r62Ufg4uGlvQsQ+nRSiq9y0ieVFRlZvUaoT/zMjmPuMg29O9rRAMdPJuiCpBH4++x8KJoJ9c2HBRizn6/3uc5Q==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.52.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay": { + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.52.0.tgz", + "integrity": "sha512-b4hQPni1G2tcV5XuAPSV4RTX3vqPXO9RfUXLuTBzOTNzBHDoj8nQv0yVvcysGy5tBAuVRo5ya5A+PG/iC6FA9A==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.52.0", + "@sentry/core": "8.52.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay-canvas": { + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.52.0.tgz", + "integrity": "sha512-4ES2uCUb9yEO1cbg15UBqiYU/syQYj5GviI+TvYvnPX3I8K2mK941ZRqfHh2HpFMhMxLgfX4jDqDGizNhXWdqg==", + "license": "MIT", + "dependencies": { + "@sentry-internal/replay": "8.52.0", + "@sentry/core": "8.52.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/browser": { + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.52.0.tgz", + "integrity": "sha512-7JpJ9zpInozBzy61eJf/6RPHoKUCFcoFuKd9rttkN1gyY9xkU1cQK+x1f0deiIHnF9ydftmDtXW+kGFI/+xqtw==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.52.0", + "@sentry-internal/feedback": "8.52.0", + "@sentry-internal/replay": "8.52.0", + "@sentry-internal/replay-canvas": "8.52.0", + "@sentry/core": "8.52.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/core": { + "version": "8.52.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.52.0.tgz", + "integrity": "sha512-2j3B7IKmseTKFm6AyheJ+RSgXqIsx+3blFSuxpkdvsEt60Lbzva2uDkCENfBDOclioo1kvHgsyuXLfWW4A+wwA==", + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "license": "MIT" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", + "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/node": { + "version": "22.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.12.0.tgz", + "integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@types/node-forge": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@types/node-forge/-/node-forge-1.3.11.tgz", + "integrity": "sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/prop-types": { + "version": "15.7.14", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz", + "integrity": "sha512-gNMvNH49DJ7OJYv+KAKn0Xp45p8PLl6zo2YnvDIbTd4J6MER2BmWN49TG7n9LvkyihINxeKW8+3bfS2yDC9dzQ==", + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.18.tgz", + "integrity": "sha512-t4yC+vtgnkYjNSKlFx1jkAhH8LgTo2N/7Qvi83kdEaUtMDiwpbLAktKDaAMlRcJ5eSxZkH74eEGt1ky31d7kfQ==", + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-native": { + "version": "0.67.3", + "resolved": "https://registry.npmjs.org/@types/react-native/-/react-native-0.67.3.tgz", + "integrity": "sha512-hF4uOZFl2PPQtGWOtLoafrlCJeU815X3PgfVePM+7EhOIZhYXKH7+p3R3cZSnwVnrU5Ep/JfiHimMDliY3o8oQ==", + "license": "MIT", + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/react-native-background-timer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/react-native-background-timer/-/react-native-background-timer-2.0.2.tgz", + "integrity": "sha512-cMAep0M5yqUHjiiRPvGiviqiJYdI45KSjbI5ufsIFSQGFwHwrHJC/8yawNhy0G3Gix6fufWLsEj6jC5niUNHiQ==", + "license": "MIT" + }, + "node_modules/@types/react-native/node_modules/@types/react": { + "version": "19.0.8", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.8.tgz", + "integrity": "sha512-9P/o1IGdfmQxrujGbIMDyYaaCykhLKc0NGCtYcECNUr9UAaDe4gwvV9bR6tvd5Br1SG0j+PBpbKr2UYY8CwqSw==", + "license": "MIT", + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/scheduler": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.23.0.tgz", + "integrity": "sha512-YIoDCTH3Af6XM5VuwGG/QL/CJqga1Zm3NkU3HZ4ZHK2fRMPYP1VczsTUqtsf43PH/iJNVlPHAo2oWX7BSdB2Hw==", + "license": "MIT" + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.33", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", + "integrity": "sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==", + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "license": "MIT" + }, + "node_modules/@urql/core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@urql/core/-/core-5.1.0.tgz", + "integrity": "sha512-yC3sw8yqjbX45GbXxfiBY8GLYCiyW/hLBbQF9l3TJrv4ro00Y0ChkKaD9I2KntRxAVm9IYBqh0awX8fwWAe/Yw==", + "license": "MIT", + "dependencies": { + "@0no-co/graphql.web": "^1.0.5", + "wonka": "^6.3.2" + } + }, + "node_modules/@urql/exchange-retry": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@urql/exchange-retry/-/exchange-retry-1.3.0.tgz", + "integrity": "sha512-FLt+d81gP4oiHah4hWFDApimc+/xABWMU1AMYsZ1PVB0L0YPtrMCjbOp9WMM7hBzy4gbTDrG24sio0dCfSh/HQ==", + "license": "MIT", + "dependencies": { + "@urql/core": "^5.0.0", + "wonka": "^6.3.2" + }, + "peerDependencies": { + "@urql/core": "^5.0.0" + } + }, + "node_modules/@xmldom/xmldom": { + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.7.13.tgz", + "integrity": "sha512-lm2GW5PkosIzccsaZIz7tp8cPADSIlIHWDFTR1N0SzfinhhYgeIQjFMz4rYzanCScr3DqQLeomUDArp6MWKm+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "license": "MIT", + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/anser": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/anser/-/anser-1.4.10.tgz", + "integrity": "sha512-hCv9AqTQ8ycjpSd3upOJd7vFwW1JaoYQ7tpham03GJ1ca8/65rqn0RpaWpItOAd6ylW9wAw6luXYPJIyPFVOww==", + "license": "MIT" + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "license": "MIT" + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/application-config-path": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/application-config-path/-/application-config-path-0.1.1.tgz", + "integrity": "sha512-zy9cHePtMP0YhwG+CfHm0bgwdnga2X3gZexpdCwEj//dpb+TKajtiC8REEUJUSq6Ab4f9cgNy2l8ObXzCXFkEw==", + "license": "MIT" + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "license": "MIT" + }, + "node_modules/ast-types": { + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.15.2.tgz", + "integrity": "sha512-c27loCv9QkZinsa5ProX751khO9DJl/AcB5c2KNtA6NRvHKS0PgLfcftz72KVq504vB0Gku5s2kUZzDBvQWvHg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", + "license": "ISC", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/babel-core": { + "version": "7.0.0-bridge.0", + "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", + "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", + "license": "MIT", + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2": { + "version": "0.4.12", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.4.12.tgz", + "integrity": "sha512-CPWT6BwvhrTO2d8QVorhTCQw9Y43zOu7G9HigcfxvepOU6b8o3tcWad6oVgZIsZCTt42FFv97aA7ZJsbM4+8og==", + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.22.6", + "@babel/helper-define-polyfill-provider": "^0.6.3", + "semver": "^6.3.1" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/babel-plugin-polyfill-corejs2/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/babel-plugin-polyfill-corejs3": { + "version": "0.10.6", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.10.6.tgz", + "integrity": "sha512-b37+KR2i/khY5sKmWNVQAnitvquQbNdWy6lJdsr0kmquCKEEUgMKK4SboVM3HtfnZilfjr4MMQ7vY58FVWDtIA==", + "license": "MIT", + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.6.2", + "core-js-compat": "^3.38.0" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/babel-plugin-polyfill-regenerator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.6.3.tgz", + "integrity": "sha512-LiWSbl4CRSIa5x/JAU6jZiG9eit9w6mz+yVMFwDE83LAWvt0AfGBoZ7HS/mkhrKuh2ZlzfVZYKoLjXdqw6Yt7Q==", + "license": "MIT", + "dependencies": { + "@babel/helper-define-polyfill-provider": "^0.6.3" + }, + "peerDependencies": { + "@babel/core": "^7.4.0 || ^8.0.0-0 <8.0.0" + } + }, + "node_modules/babel-plugin-react-native-web": { + "version": "0.19.13", + "resolved": "https://registry.npmjs.org/babel-plugin-react-native-web/-/babel-plugin-react-native-web-0.19.13.tgz", + "integrity": "sha512-4hHoto6xaN23LCyZgL9LJZc3olmAxd7b6jDzlZnKXAh4rRAbZRKNBJoOOdp46OBqgy+K0t0guTj5/mhA8inymQ==", + "license": "MIT" + }, + "node_modules/babel-plugin-syntax-hermes-parser": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/babel-plugin-syntax-hermes-parser/-/babel-plugin-syntax-hermes-parser-0.25.1.tgz", + "integrity": "sha512-IVNpGzboFLfXZUAwkLFcI/bnqVbwky0jP3eBno4HKtqvQJAHBLdgxiG6lQ4to0+Q/YCN3PO0od5NZwIKyY4REQ==", + "license": "MIT", + "dependencies": { + "hermes-parser": "0.25.1" + } + }, + "node_modules/babel-plugin-syntax-hermes-parser/node_modules/hermes-estree": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", + "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", + "license": "MIT" + }, + "node_modules/babel-plugin-syntax-hermes-parser/node_modules/hermes-parser": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", + "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", + "license": "MIT", + "dependencies": { + "hermes-estree": "0.25.1" + } + }, + "node_modules/babel-plugin-transform-flow-enums": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-flow-enums/-/babel-plugin-transform-flow-enums-0.0.2.tgz", + "integrity": "sha512-g4aaCrDDOsWjbm0PUUeVnkcVd6AKJsVc/MbnPhEotEpkeJQP6b8nzewohQi7+QS8UyPehOhGWn0nOwjvWpmMvQ==", + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-flow": "^7.12.1" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", + "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/babel-preset-expo": { + "version": "12.0.6", + "resolved": "https://registry.npmjs.org/babel-preset-expo/-/babel-preset-expo-12.0.6.tgz", + "integrity": "sha512-az3H7gDVo0wxNBAFES8h5vLLWE8NPGkD9g5P962hDEOqZUdyPacb9MOzicypeLmcq9zQWr6E3iVtEHoNagCTTQ==", + "license": "MIT", + "dependencies": { + "@babel/plugin-proposal-decorators": "^7.12.9", + "@babel/plugin-transform-export-namespace-from": "^7.22.11", + "@babel/plugin-transform-object-rest-spread": "^7.12.13", + "@babel/plugin-transform-parameters": "^7.22.15", + "@babel/preset-react": "^7.22.15", + "@babel/preset-typescript": "^7.23.0", + "@react-native/babel-preset": "0.76.6", + "babel-plugin-react-native-web": "~0.19.13", + "react-refresh": "^0.14.2" + }, + "peerDependencies": { + "babel-plugin-react-compiler": "^19.0.0-beta-9ee70a1-20241017", + "react-compiler-runtime": "^19.0.0-beta-8a03594-20241020" + }, + "peerDependenciesMeta": { + "babel-plugin-react-compiler": { + "optional": true + }, + "react-compiler-runtime": { + "optional": true + } + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base-64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-1.0.0.tgz", + "integrity": "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-opn": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/better-opn/-/better-opn-3.0.2.tgz", + "integrity": "sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ==", + "license": "MIT", + "dependencies": { + "open": "^8.0.4" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/better-opn/node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "license": "MIT", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", + "license": "MIT" + }, + "node_modules/bplist-creator": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/bplist-creator/-/bplist-creator-0.0.7.tgz", + "integrity": "sha512-xp/tcaV3T5PCiaY04mXga7o/TE+t95gqeLmADeBI1CvZtdWTbgBt3uLpvh4UWtenKeBhCV6oVxGk38yZr2uYEA==", + "license": "MIT", + "dependencies": { + "stream-buffers": "~2.2.0" + } + }, + "node_modules/bplist-parser": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.3.2.tgz", + "integrity": "sha512-apC2+fspHGI3mMKj+dGevkGo/tCqVB8jMb6i+OX+E29p0Iposz07fABkRIfVUPNd5A5VbuOz1bZbnmkKLYF+wQ==", + "license": "MIT", + "dependencies": { + "big-integer": "1.6.x" + }, + "engines": { + "node": ">= 5.10.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.4.tgz", + "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "license": "MIT", + "dependencies": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "node_modules/buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "license": "MIT" + }, + "node_modules/buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", + "license": "MIT" + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cacache": { + "version": "18.0.4", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.4.tgz", + "integrity": "sha512-B+L5iIa9mgcjLbliir2th36yEwPftrzteHYujzsx3dFP/31GCHcIeS8f5MGd80odLOjaOvSpU3EEAmRQptkxLQ==", + "license": "ISC", + "dependencies": { + "@npmcli/fs": "^3.1.0", + "fs-minipass": "^3.0.0", + "glob": "^10.2.2", + "lru-cache": "^10.0.1", + "minipass": "^7.0.3", + "minipass-collect": "^2.0.1", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "p-map": "^4.0.0", + "ssri": "^10.0.0", + "tar": "^6.1.11", + "unique-filename": "^3.0.0" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/cacache/node_modules/fs-minipass": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", + "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/caller-callsite": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/caller-callsite/-/caller-callsite-2.0.0.tgz", + "integrity": "sha512-JuG3qI4QOftFsZyOn1qq87fq5grLIyk1JYd5lJmdA+fG7aQ9pA/i3JIJGcO3q0MrRcHlOt1U+ZeHW8Dq9axALQ==", + "license": "MIT", + "dependencies": { + "callsites": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/caller-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-2.0.0.tgz", + "integrity": "sha512-MCL3sf6nCSXOwCTzvPKhN18TU7AHTvdtam8DAogxcrJ8Rjfbbg7Lgng64H9Iy+vUV6VGFClN/TyxBkAebLRR4A==", + "license": "MIT", + "dependencies": { + "caller-callsite": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/callsites": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-2.0.0.tgz", + "integrity": "sha512-ksWePWBloaWPxJYQ8TL0JHvtci6G5QTKwQ95RcWAa/lzoAKuAOflGdAK92hpHXjkwb8zLxoLNUoNYZgVsaJzvQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001696", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001696.tgz", + "integrity": "sha512-pDCPkvzfa39ehJtJ+OwGT/2yvT2SbjfHhiIW2LWOAcMQ7BzwxT/XuyUp4OTOd0XFWA6BKw0JalnBHgSi5DGJBQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/chrome-launcher": { + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-0.15.2.tgz", + "integrity": "sha512-zdLEwNo3aUVzIhKhTtXfxhdvZhUghrnmkvcAq2NoDd+LeOHKf03H5jwZ8T/STsAlzyALkBVK552iaG1fGf1xVQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^4.0.0", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^1.0.0" + }, + "bin": { + "print-chrome-path": "bin/print-chrome-path.js" + }, + "engines": { + "node": ">=12.13.0" + } + }, + "node_modules/chromium-edge-launcher": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/chromium-edge-launcher/-/chromium-edge-launcher-0.2.0.tgz", + "integrity": "sha512-JfJjUnq25y9yg4FABRRVPmBGWPZZi+AQXT4mxupb67766/0UlhG8PAZCz6xzEMXTbW3CsSoE8PcCWA49n35mKg==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^4.0.0", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^1.0.0", + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + } + }, + "node_modules/chromium-edge-launcher/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/command-exists": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", + "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==", + "license": "MIT" + }, + "node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "license": "MIT" + }, + "node_modules/component-type": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/component-type/-/component-type-1.2.2.tgz", + "integrity": "sha512-99VUHREHiN5cLeHm3YLq312p6v+HUEcwtLCAtelvUDI6+SH5g5Cr85oNR2S1o6ywzL0ykMbuwLzM2ANocjEOIA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compressible/node_modules/mime-db": { + "version": "1.53.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.53.0.tgz", + "integrity": "sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.5.tgz", + "integrity": "sha512-bQJ0YRck5ak3LgtnpKkiabX5pNF7tMUh1BSy2ZBOTh0Dim0BUu6aPPwByIns6/A5Prh8PufSPerMDUklpzes2Q==", + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.0.2", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/connect": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz", + "integrity": "sha512-ZqRXc+tZukToSNmh5C2iWMSoV3X1YUcPbqEM4DkEG5tNQXrQUZCNVGGv3IuicnkMtPfGf3Xtp8WCXs295iQ1pQ==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "finalhandler": "1.1.2", + "parseurl": "~1.3.3", + "utils-merge": "1.0.1" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "license": "MIT" + }, + "node_modules/core-js-compat": { + "version": "3.40.0", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.40.0.tgz", + "integrity": "sha512-0XEDpr5y5mijvw8Lbc6E5AkjrHfp7eEoPlu36SWeAbcL8fn1G1ANe8DBlo2XoNN89oVpxWwOjYIPVzR4ZvsKCQ==", + "license": "MIT", + "dependencies": { + "browserslist": "^4.24.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.2.1.tgz", + "integrity": "sha512-H65gsXo1SKjf8zmrJ67eJk8aIRKV5ff2D4uKZIBZShbhGSpEmsQOPW/SKMKYhSTrqR7ufy6RP69rPogdaPh/kA==", + "license": "MIT", + "dependencies": { + "import-fresh": "^2.0.0", + "is-directory": "^0.3.1", + "js-yaml": "^3.13.1", + "parse-json": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cross-fetch": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.2.0.tgz", + "integrity": "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==", + "license": "MIT", + "dependencies": { + "node-fetch": "^2.7.0" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cross-spawn/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "license": "BSD-3-Clause", + "engines": { + "node": "*" + } + }, + "node_modules/crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-gateway": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-4.2.0.tgz", + "integrity": "sha512-h6sMrVB1VMWVrW13mSc6ia/DwYYw5MN6+exNu1OaJeFac5aSAvwM7lZ0NVfTABuSkQelr4h5oebg3KB1XPdjgA==", + "license": "BSD-2-Clause", + "dependencies": { + "execa": "^1.0.0", + "ip-regex": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/default-gateway/node_modules/cross-spawn": { + "version": "6.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.6.tgz", + "integrity": "sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==", + "license": "MIT", + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/default-gateway/node_modules/execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/default-gateway/node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/default-gateway/node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-gateway/node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", + "license": "MIT", + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/default-gateway/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "license": "ISC", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/default-gateway/node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-gateway/node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/default-gateway/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/del": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", + "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", + "license": "MIT", + "dependencies": { + "globby": "^11.0.1", + "graceful-fs": "^4.2.4", + "is-glob": "^4.0.1", + "is-path-cwd": "^2.2.0", + "is-path-inside": "^3.0.2", + "p-map": "^4.0.0", + "rimraf": "^3.0.2", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/denodeify": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/denodeify/-/denodeify-1.2.1.tgz", + "integrity": "sha512-KNTihKNmQENUZeKu5fzfpzRqR5S2VMp4gl9RFHiWzj9DfvYQPMJ6XHKNaQxaGCXwPk6y9yme3aUoaiAe+KX+vg==", + "license": "MIT" + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", + "license": "Apache-2.0", + "bin": { + "detect-libc": "bin/detect-libc.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "license": "MIT", + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dotenv-expand": { + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.7.tgz", + "integrity": "sha512-zIHwmZPRshsCdpMDyVsqGmgyP0yT8GAgXUnkdAoJisxvf33k7yO6OuoKmcTGuXPWSsm8Oh88nZicRLA9Y0rUeA==", + "license": "BSD-2-Clause", + "dependencies": { + "dotenv": "^16.4.5" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.88", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.88.tgz", + "integrity": "sha512-K3C2qf1o+bGzbilTDCTBhTQcMS9KW60yTAaTeeXsfvQuTDDwlokLam/AdqlqcSy9u4UainDgsHV23ksXAOgamw==", + "license": "ISC" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-editor": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/env-editor/-/env-editor-0.4.2.tgz", + "integrity": "sha512-ObFo8v4rQJAE59M69QzwloxPZtd33TpYEIjtKD1rrFDcM1Gd7IkDxEBU+HriziN6HSHQnBJi8Dmy+JWkav5HKA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/eol": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/eol/-/eol-0.9.1.tgz", + "integrity": "sha512-Ds/TEoZjwggRoz/Q2O7SE3i4Jm66mqTDfmdHdq/7DKVk3bro9Q8h6WdXKdPqFLMoqxrDK5SVRzHVPOS6uuGtrg==", + "license": "MIT" + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/error-stack-parser": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", + "integrity": "sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==", + "license": "MIT", + "dependencies": { + "stackframe": "^1.3.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/exec-async": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/exec-async/-/exec-async-2.2.0.tgz", + "integrity": "sha512-87OpwcEiMia/DeiKFzaQNBNFeN3XkkpYIh9FyOqq5mS2oKv3CBE67PXoEKcr6nodWdXNogTiQ0jE2NGuoffXPw==", + "license": "MIT" + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/expo": { + "version": "52.0.28", + "resolved": "https://registry.npmjs.org/expo/-/expo-52.0.28.tgz", + "integrity": "sha512-0O/JEYYCFszJ85frislm79YmlrQA5ghAQXV4dqcQcsy9FqftdicD4p/ehT36yiuGIhaKC6fn25LEaJ9JR2ei7g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.20.0", + "@expo/cli": "0.22.11", + "@expo/config": "~10.0.8", + "@expo/config-plugins": "~9.0.14", + "@expo/fingerprint": "0.11.7", + "@expo/metro-config": "0.19.9", + "@expo/vector-icons": "^14.0.0", + "babel-preset-expo": "~12.0.6", + "expo-asset": "~11.0.2", + "expo-constants": "~17.0.5", + "expo-file-system": "~18.0.7", + "expo-font": "~13.0.3", + "expo-keep-awake": "~14.0.2", + "expo-modules-autolinking": "2.0.7", + "expo-modules-core": "2.2.0", + "fbemitter": "^3.0.0", + "web-streams-polyfill": "^3.3.2", + "whatwg-url-without-unicode": "8.0.0-3" + }, + "bin": { + "expo": "bin/cli" + }, + "peerDependencies": { + "@expo/dom-webview": "*", + "@expo/metro-runtime": "*", + "react": "*", + "react-native": "*", + "react-native-webview": "*" + }, + "peerDependenciesMeta": { + "@expo/dom-webview": { + "optional": true + }, + "@expo/metro-runtime": { + "optional": true + }, + "react-native-webview": { + "optional": true + } + } + }, + "node_modules/expo-asset": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/expo-asset/-/expo-asset-11.0.2.tgz", + "integrity": "sha512-We3Td5WsNsNQyXoheLnuwic6JCOt/pqXqIIyWaZ3z/PeHrA+SwoQdI18MjDhkudLK08tbIVyDSUW8IJHXa04eg==", + "license": "MIT", + "dependencies": { + "@expo/image-utils": "^0.6.4", + "expo-constants": "~17.0.4", + "invariant": "^2.2.4", + "md5-file": "^3.2.3" + }, + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*" + } + }, + "node_modules/expo-build-properties": { + "version": "0.13.2", + "resolved": "https://registry.npmjs.org/expo-build-properties/-/expo-build-properties-0.13.2.tgz", + "integrity": "sha512-ML2GwBgn0Bo4yPgnSGb7h3XVxCigS/KFdid3xPC2HldEioTP3UewB/2Qa4WBsam9Fb7lAuRyVHAfRoA3swpDzg==", + "license": "MIT", + "dependencies": { + "ajv": "^8.11.0", + "semver": "^7.6.0" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-constants": { + "version": "17.0.5", + "resolved": "https://registry.npmjs.org/expo-constants/-/expo-constants-17.0.5.tgz", + "integrity": "sha512-6SHXh32jCB+vrp2TRDNkoGoM421eOBPZIXX9ixI0hKKz71tIjD+LMr/P+rGUd/ks312MP3WK3j5vcYYPkCD8tQ==", + "license": "MIT", + "dependencies": { + "@expo/config": "~10.0.8", + "@expo/env": "~0.4.1" + }, + "peerDependencies": { + "expo": "*", + "react-native": "*" + } + }, + "node_modules/expo-dev-client": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/expo-dev-client/-/expo-dev-client-5.0.10.tgz", + "integrity": "sha512-iCrpt4XOQjTWbsqlZQSG3wOHsAyboJNg9xpHWBKJy3JFC2uCPH36cX2NvkmEtWqWKXKUjrx0t4B/X9blcDnvSQ==", + "license": "MIT", + "dependencies": { + "expo-dev-launcher": "5.0.25", + "expo-dev-menu": "6.0.18", + "expo-dev-menu-interface": "1.9.3", + "expo-manifests": "~0.15.5", + "expo-updates-interface": "~1.0.0" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-dev-launcher": { + "version": "5.0.25", + "resolved": "https://registry.npmjs.org/expo-dev-launcher/-/expo-dev-launcher-5.0.25.tgz", + "integrity": "sha512-5iH89otFs2lFEXMFRXg5E+YMC1wxoZCp2FuemzLPuNtNC8HX64hUy+PeU8F4H8Xc17K6Hd6zAp9QJqgX4l4eMQ==", + "license": "MIT", + "dependencies": { + "ajv": "8.11.0", + "expo-dev-menu": "6.0.18", + "expo-manifests": "~0.15.5", + "resolve-from": "^5.0.0" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-dev-launcher/node_modules/ajv": { + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/expo-dev-menu": { + "version": "6.0.18", + "resolved": "https://registry.npmjs.org/expo-dev-menu/-/expo-dev-menu-6.0.18.tgz", + "integrity": "sha512-QexBMNbZR/J3nNF7UaUs7PcY77bTjLSXWHFTuRM17bGlNCBJWfmoSdKSJ0YQtOTx560bJpCdtWJAn0DR2rj3TA==", + "license": "MIT", + "dependencies": { + "expo-dev-menu-interface": "1.9.3" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-dev-menu-interface": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/expo-dev-menu-interface/-/expo-dev-menu-interface-1.9.3.tgz", + "integrity": "sha512-KY/dWTBE1l47i9V366JN5rC6YIdOc9hz8yAmZzkl5DrPia5l3M2WIjtnpHC9zUkNjiSiG2urYoOAq4H/uLdmyg==", + "license": "MIT", + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-file-system": { + "version": "18.0.7", + "resolved": "https://registry.npmjs.org/expo-file-system/-/expo-file-system-18.0.7.tgz", + "integrity": "sha512-6PpbQfogMXdzOsJzlJayy5qf40IfIHhudtAOzr32RlRYL4Hkmk3YcR9jG0PWQ0rklJfAhbAdP63yOcN+wDgzaA==", + "license": "MIT", + "dependencies": { + "web-streams-polyfill": "^3.3.2" + }, + "peerDependencies": { + "expo": "*", + "react-native": "*" + } + }, + "node_modules/expo-font": { + "version": "13.0.3", + "resolved": "https://registry.npmjs.org/expo-font/-/expo-font-13.0.3.tgz", + "integrity": "sha512-9IdYz+A+b3KvuCYP7DUUXF4VMZjPU+IsvAnLSVJ2TfP6zUD2JjZFx3jeo/cxWRkYk/aLj5+53Te7elTAScNl4Q==", + "license": "MIT", + "dependencies": { + "fontfaceobserver": "^2.1.0" + }, + "peerDependencies": { + "expo": "*", + "react": "*" + } + }, + "node_modules/expo-json-utils": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/expo-json-utils/-/expo-json-utils-0.14.0.tgz", + "integrity": "sha512-xjGfK9dL0B1wLnOqNkX0jM9p48Y0I5xEPzHude28LY67UmamUyAACkqhZGaPClyPNfdzczk7Ej6WaRMT3HfXvw==", + "license": "MIT" + }, + "node_modules/expo-keep-awake": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/expo-keep-awake/-/expo-keep-awake-14.0.2.tgz", + "integrity": "sha512-71XAMnoWjKZrN8J7Q3+u0l9Ytp4OfhNAYz8BCWF1/9aFUw09J3I7Z5DuI3MUsVMa/KWi+XhG+eDUFP8cVA19Uw==", + "license": "MIT", + "peerDependencies": { + "expo": "*", + "react": "*" + } + }, + "node_modules/expo-manifests": { + "version": "0.15.5", + "resolved": "https://registry.npmjs.org/expo-manifests/-/expo-manifests-0.15.5.tgz", + "integrity": "sha512-3X3eQomnTa4G0Y9GoJeyewHPTscuzWMrTB3x4CknqOyXpGOJjOuCKjhzvccHxXZAt0XswqBI94iTbqIofo9Uqw==", + "license": "MIT", + "dependencies": { + "@expo/config": "~10.0.8", + "expo-json-utils": "~0.14.0" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-modules-autolinking": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/expo-modules-autolinking/-/expo-modules-autolinking-2.0.7.tgz", + "integrity": "sha512-rkGc6a/90AC3q8wSy4V+iIpq6Fd0KXmQICKrvfmSWwrMgJmLfwP4QTrvLYPYOOMjFwNJcTaohcH8vzW/wYKrMg==", + "license": "MIT", + "dependencies": { + "@expo/spawn-async": "^1.7.2", + "chalk": "^4.1.0", + "commander": "^7.2.0", + "fast-glob": "^3.2.5", + "find-up": "^5.0.0", + "fs-extra": "^9.1.0", + "require-from-string": "^2.0.2", + "resolve-from": "^5.0.0" + }, + "bin": { + "expo-modules-autolinking": "bin/expo-modules-autolinking.js" + } + }, + "node_modules/expo-modules-autolinking/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/expo-modules-core": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/expo-modules-core/-/expo-modules-core-2.2.0.tgz", + "integrity": "sha512-mOFEHIe6jZ7G5pYUVSQ2Ghs3CUr9Uz6DOh4JI+4PsTf0gmEvMmMEOrxirS89jRWQjXPJ7QaGBK0CJrZlj/Sdeg==", + "license": "MIT", + "dependencies": { + "invariant": "^2.2.4" + } + }, + "node_modules/expo-splash-screen": { + "version": "0.29.21", + "resolved": "https://registry.npmjs.org/expo-splash-screen/-/expo-splash-screen-0.29.21.tgz", + "integrity": "sha512-7uZ+qvIuNcvrvrLIklW+Wbt6llPuCj6LKYjrMu+GOX8s///laldS4TGiMAbqcE7fmfCzQ8ffgfY7xhxRourhcA==", + "license": "MIT", + "dependencies": { + "@expo/prebuild-config": "^8.0.25" + }, + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/expo-status-bar": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/expo-status-bar/-/expo-status-bar-2.0.1.tgz", + "integrity": "sha512-AkIPX7jWHRPp83UBZ1iXtVvyr0g+DgBVvIXTtlmPtmUsm8Vq9Bb5IGj86PW8osuFlgoTVAg7HI/+Ok7yEYwiRg==", + "license": "MIT", + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, + "node_modules/expo-updates-interface": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/expo-updates-interface/-/expo-updates-interface-1.0.0.tgz", + "integrity": "sha512-93oWtvULJOj+Pp+N/lpTcFfuREX1wNeHtp7Lwn8EbzYYmdn37MvZU3TPW2tYYCZuhzmKEXnUblYcruYoDu7IrQ==", + "license": "MIT", + "peerDependencies": { + "expo": "*" + } + }, + "node_modules/exponential-backoff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", + "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==", + "license": "Apache-2.0" + }, + "node_modules/fast-base64-decode": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fast-base64-decode/-/fast-base64-decode-1.0.0.tgz", + "integrity": "sha512-qwaScUgUGBYeDNRnbc/KyllVU88Jk1pRHPStuF/lO7B0/RTRLj7U0lkdTAutlBblY08rwZDff6tNU9cjv6j//Q==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fastq": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.18.0.tgz", + "integrity": "sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/fbemitter": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/fbemitter/-/fbemitter-3.0.0.tgz", + "integrity": "sha512-KWKaceCwKQU0+HPoop6gn4eOHk50bBv/VxjJtGMfwmJt3D29JpN4H4eisCtIPA+a8GVBam+ldMMpMjJUvpDyHw==", + "license": "BSD-3-Clause", + "dependencies": { + "fbjs": "^3.0.0" + } + }, + "node_modules/fbjs": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-3.0.5.tgz", + "integrity": "sha512-ztsSx77JBtkuMrEypfhgc3cI0+0h+svqeie7xHbh1k/IKdcydnvadp/mUaGgjAOXQmQSxsqgaRhS3q9fy+1kxg==", + "license": "MIT", + "dependencies": { + "cross-fetch": "^3.1.5", + "fbjs-css-vars": "^1.0.0", + "loose-envify": "^1.0.0", + "object-assign": "^4.1.0", + "promise": "^7.1.1", + "setimmediate": "^1.0.5", + "ua-parser-js": "^1.0.35" + } + }, + "node_modules/fbjs-css-vars": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/fbjs-css-vars/-/fbjs-css-vars-1.0.2.tgz", + "integrity": "sha512-b2XGFAFdWZWg0phtAWLHCk836A1Xann+I+Dgd3Gk64MHKZO44FfoD1KxyvbSh0qZsIoXQGGlVztIY+oitJPpRQ==", + "license": "MIT" + }, + "node_modules/fbjs/node_modules/promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "license": "MIT", + "dependencies": { + "asap": "~2.0.3" + } + }, + "node_modules/fetch-retry": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-4.1.1.tgz", + "integrity": "sha512-e6eB7zN6UBSwGVwrbWVH+gdLnkW9WwHhmq2YDK1Sh30pzx1onRVGBvogTlUeWxwTa+L86NYdo4hFkh7O8ZjSnA==", + "license": "MIT" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz", + "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.3", + "statuses": "~1.5.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "license": "MIT", + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flow-enums-runtime": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/flow-enums-runtime/-/flow-enums-runtime-0.0.6.tgz", + "integrity": "sha512-3PYnM29RFXwvAN6Pc/scUfkI7RwhQ/xqyLUyPNlXUp9S40zI8nup9tUSrTLSVnWGBN38FNiGWbwZOB6uR4OGdw==", + "license": "MIT" + }, + "node_modules/flow-parser": { + "version": "0.259.1", + "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.259.1.tgz", + "integrity": "sha512-xiXLmMH2Z7OmdE9Q+MjljUMr/rbemFqZIRxaeZieVScG4HzQrKKhNcCYZbWTGpoN7ZPi7z8ClQbeVPq6t5AszQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/fontfaceobserver": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/fontfaceobserver/-/fontfaceobserver-2.3.0.tgz", + "integrity": "sha512-6FPvD/IVyT4ZlNe7Wcn5Fb/4ChigpucKYSvD6a+0iMoLn2inpo711eyIcKjmDtE5XNcgAkSH9uN/nfAeZzHEfg==", + "license": "BSD-2-Clause" + }, + "node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.2.tgz", + "integrity": "sha512-sJe+TQb2vIaIyO783qN6BlMYWMw3WBOHA1Ay2qxsnjuafEOQFJ2JakedOQirT6D5XPRxDvS7AHYyem9fTpb4LQ==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/freeport-async": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/freeport-async/-/freeport-async-2.0.0.tgz", + "integrity": "sha512-K7od3Uw45AJg00XUmy15+Hae2hOcgKcmN3/EF6Y7i01O0gaqiRx8sUSpsb9+BRNL8RPBrhzPsVfy8q9ADlJuWQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "license": "MIT", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-port": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", + "integrity": "sha512-x5UJKlgeUiNT8nyo/AcnwLnZuZNcSjSw0kogRB+Whd1fjjFq4B1hySFxSFWWSn4mIBzg3sRNUDFYc4g5gjPoLg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/getenv": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/getenv/-/getenv-1.0.0.tgz", + "integrity": "sha512-7yetJWqbS9sbn0vIfliPsFgoXMKn/YMF+Wuiog97x+urnSRRRZ7xB+uVkwGKzRgq9CDFfMQnE9ruL5DHv9c6Xg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/globby": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", + "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "license": "MIT", + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.9", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hermes-estree": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.23.1.tgz", + "integrity": "sha512-eT5MU3f5aVhTqsfIReZ6n41X5sYn4IdQL0nvz6yO+MMlPxw49aSARHLg/MSehQftyjnrE8X6bYregzSumqc6cg==", + "license": "MIT" + }, + "node_modules/hermes-parser": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.23.1.tgz", + "integrity": "sha512-oxl5h2DkFW83hT4DAUJorpah8ou4yvmweUzLJmmr6YV2cezduCdlil1AvU/a/xSsAFo4WUcNA4GoV5Bvq6JffA==", + "license": "MIT", + "dependencies": { + "hermes-estree": "0.23.1" + } + }, + "node_modules/hosted-git-info": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", + "integrity": "sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==", + "license": "ISC", + "dependencies": { + "lru-cache": "^10.0.1" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/image-size": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.2.0.tgz", + "integrity": "sha512-4S8fwbO6w3GeCVN6OPtA9I5IGKkcDMPcKndtUlpJuCwu7JLjtj7JZpwqLuyY2nrmQT3AWsCJLSKPsc2mPBSl3w==", + "license": "MIT", + "dependencies": { + "queue": "6.0.2" + }, + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=16.x" + } + }, + "node_modules/import-fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-2.0.0.tgz", + "integrity": "sha512-eZ5H8rcgYazHbKC3PG4ClHNykCSxtAhxSSEM+2mb+7evD2CKF5V7c0dNum7AdpDh0ZdICwZY9sRSn8f+KH96sg==", + "license": "MIT", + "dependencies": { + "caller-path": "^2.0.0", + "resolve-from": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/import-fresh/node_modules/resolve-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz", + "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/internal-ip": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/internal-ip/-/internal-ip-4.3.0.tgz", + "integrity": "sha512-S1zBo1D6zcsyuC6PMmY5+55YMILQ9av8lotMx447Bq6SAgo/sDK6y6uUKmuYhW7eacnIhFfsPmCNYdDzsnnDCg==", + "license": "MIT", + "dependencies": { + "default-gateway": "^4.2.0", + "ipaddr.js": "^1.9.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.0.0" + } + }, + "node_modules/ip-regex": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", + "integrity": "sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-directory": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz", + "integrity": "sha512-yVChGzahRFvbkscn2MlwGismPO12i9+znNruC5gVEntG3qu0xQMzsGg/JFbrsqDOHtHFPci+V5aP5T9I+yeKqw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/jimp-compact": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/jimp-compact/-/jimp-compact-0.16.1.tgz", + "integrity": "sha512-dZ6Ra7u1G8c4Letq/B5EzAxj4tLFHL+cGtdpR+PVm4yzPDj+lCk+AbivWt1eOM+ikzkowtyV7qSqX6qr3t71Ww==", + "license": "MIT" + }, + "node_modules/join-component": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/join-component/-/join-component-1.1.0.tgz", + "integrity": "sha512-bF7vcQxbODoGK1imE2P9GS9aw4zD0Sd+Hni68IMZLj7zRnquH7dXUmMw9hDI5S/Jzt7q+IyTXN0rSg2GI0IKhQ==", + "license": "MIT" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsc-android": { + "version": "250231.0.0", + "resolved": "https://registry.npmjs.org/jsc-android/-/jsc-android-250231.0.0.tgz", + "integrity": "sha512-rS46PvsjYmdmuz1OAWXY/1kCYG7pnf1TBqeTiOJr1iDz7s5DLxxC9n/ZMknLDxzYzNVfI7R95MH10emSSG1Wuw==", + "license": "BSD-2-Clause" + }, + "node_modules/jsc-safe-url": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/jsc-safe-url/-/jsc-safe-url-0.2.4.tgz", + "integrity": "sha512-0wM3YBWtYePOjfyXQH5MWQ8H7sdk5EXSwZvmSLKk2RboVQ2Bu239jycHDz5J/8Blf3K0Qnoy2b6xD+z10MFB+Q==", + "license": "0BSD" + }, + "node_modules/jscodeshift": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.14.0.tgz", + "integrity": "sha512-7eCC1knD7bLUPuSCwXsMZUH51O8jIcoVyKtI6P0XM0IVzlGjckPy3FIwQlorzbN0Sg79oK+RlohN32Mqf/lrYA==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.13.16", + "@babel/parser": "^7.13.16", + "@babel/plugin-proposal-class-properties": "^7.13.0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", + "@babel/plugin-proposal-optional-chaining": "^7.13.12", + "@babel/plugin-transform-modules-commonjs": "^7.13.8", + "@babel/preset-flow": "^7.13.13", + "@babel/preset-typescript": "^7.13.0", + "@babel/register": "^7.13.16", + "babel-core": "^7.0.0-bridge.0", + "chalk": "^4.1.2", + "flow-parser": "0.*", + "graceful-fs": "^4.2.4", + "micromatch": "^4.0.4", + "neo-async": "^2.5.0", + "node-dir": "^0.1.17", + "recast": "^0.21.0", + "temp": "^0.8.4", + "write-file-atomic": "^2.3.0" + }, + "bin": { + "jscodeshift": "bin/jscodeshift.js" + }, + "peerDependencies": { + "@babel/preset-env": "^7.1.6" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lighthouse-logger": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-1.4.2.tgz", + "integrity": "sha512-gPWxznF6TKmUHrOQjlVo2UbaL2EJ71mb2CCeRs/2qBpi4L/g4LUVc9+3lKQ6DTUZwJswfM7ainGrLO1+fOqa2g==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^2.6.9", + "marky": "^1.2.2" + } + }, + "node_modules/lightningcss": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.27.0.tgz", + "integrity": "sha512-8f7aNmS1+etYSLHht0fQApPc2kNO8qGRutifN5rVIc6Xo6ABsEbqOr758UwI7ALVbTt4x1fllKt0PYgzD9S3yQ==", + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^1.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-darwin-arm64": "1.27.0", + "lightningcss-darwin-x64": "1.27.0", + "lightningcss-freebsd-x64": "1.27.0", + "lightningcss-linux-arm-gnueabihf": "1.27.0", + "lightningcss-linux-arm64-gnu": "1.27.0", + "lightningcss-linux-arm64-musl": "1.27.0", + "lightningcss-linux-x64-gnu": "1.27.0", + "lightningcss-linux-x64-musl": "1.27.0", + "lightningcss-win32-arm64-msvc": "1.27.0", + "lightningcss-win32-x64-msvc": "1.27.0" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.27.0.tgz", + "integrity": "sha512-Gl/lqIXY+d+ySmMbgDf0pgaWSqrWYxVHoc88q+Vhf2YNzZ8DwoRzGt5NZDVqqIW5ScpSnmmjcgXP87Dn2ylSSQ==", + "cpu": [ + "arm64" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "license": "MIT" + }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", + "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", + "license": "MIT", + "dependencies": { + "chalk": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/log-symbols/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "license": "MIT" + }, + "node_modules/log-symbols/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/log-symbols/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "license": "MIT", + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "license": "ISC", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/marky": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.5.tgz", + "integrity": "sha512-q9JtQJKjpsVxCRVgQ+WapguSbKC3SQ5HEzFGPAJMStgh3QjCawp00UKv3MTTAArTmGmmPUvllHZoNbZ3gs0I+Q==", + "license": "Apache-2.0" + }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, + "node_modules/md5-file": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-3.2.3.tgz", + "integrity": "sha512-3Tkp1piAHaworfcCgH0jKbTvj1jWWFgbvh2cXaNCgHwyTCBxxvD1Y04rmfpvdPm1P4oXMOpm6+2H7sr7v9v8Fw==", + "license": "MIT", + "dependencies": { + "buffer-alloc": "^1.1.0" + }, + "bin": { + "md5-file": "cli.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/memoize-one": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", + "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==", + "license": "MIT" + }, + "node_modules/merge-options": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/merge-options/-/merge-options-3.0.4.tgz", + "integrity": "sha512-2Sug1+knBjkaMsMgf1ctR1Ujx+Ayku4EdJN4Z+C2+JzoeF7A3OZ9KM2GY0CpQS51NR61LTurMJrRKPhSs3ZRTQ==", + "license": "MIT", + "dependencies": { + "is-plain-obj": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/metro": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro/-/metro-0.81.0.tgz", + "integrity": "sha512-kzdzmpL0gKhEthZ9aOV7sTqvg6NuTxDV8SIm9pf9sO8VVEbKrQk5DNcwupOUjgPPFAuKUc2NkT0suyT62hm2xg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.24.7", + "@babel/core": "^7.25.2", + "@babel/generator": "^7.25.0", + "@babel/parser": "^7.25.3", + "@babel/template": "^7.25.0", + "@babel/traverse": "^7.25.3", + "@babel/types": "^7.25.2", + "accepts": "^1.3.7", + "chalk": "^4.0.0", + "ci-info": "^2.0.0", + "connect": "^3.6.5", + "debug": "^2.2.0", + "denodeify": "^1.2.1", + "error-stack-parser": "^2.0.6", + "flow-enums-runtime": "^0.0.6", + "graceful-fs": "^4.2.4", + "hermes-parser": "0.24.0", + "image-size": "^1.0.2", + "invariant": "^2.2.4", + "jest-worker": "^29.6.3", + "jsc-safe-url": "^0.2.2", + "lodash.throttle": "^4.1.1", + "metro-babel-transformer": "0.81.0", + "metro-cache": "0.81.0", + "metro-cache-key": "0.81.0", + "metro-config": "0.81.0", + "metro-core": "0.81.0", + "metro-file-map": "0.81.0", + "metro-resolver": "0.81.0", + "metro-runtime": "0.81.0", + "metro-source-map": "0.81.0", + "metro-symbolicate": "0.81.0", + "metro-transform-plugins": "0.81.0", + "metro-transform-worker": "0.81.0", + "mime-types": "^2.1.27", + "nullthrows": "^1.1.1", + "serialize-error": "^2.1.0", + "source-map": "^0.5.6", + "strip-ansi": "^6.0.0", + "throat": "^5.0.0", + "ws": "^7.5.10", + "yargs": "^17.6.2" + }, + "bin": { + "metro": "src/cli.js" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-babel-transformer": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-babel-transformer/-/metro-babel-transformer-0.81.0.tgz", + "integrity": "sha512-Dc0QWK4wZIeHnyZ3sevWGTnnSkIDDn/SWyfrn99zbKbDOCoCYy71PAn9uCRrP/hduKLJQOy+tebd63Rr9D8tXg==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.2", + "flow-enums-runtime": "^0.0.6", + "hermes-parser": "0.24.0", + "nullthrows": "^1.1.1" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-babel-transformer/node_modules/hermes-estree": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.24.0.tgz", + "integrity": "sha512-LyoXLB7IFzeZW0EvAbGZacbxBN7t6KKSDqFJPo3Ydow7wDlrDjXwsdiAHV6XOdvEN9MEuWXsSIFN4tzpyrXIHw==", + "license": "MIT" + }, + "node_modules/metro-babel-transformer/node_modules/hermes-parser": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.24.0.tgz", + "integrity": "sha512-IJooSvvu2qNRe7oo9Rb04sUT4omtZqZqf9uq9WM25Tb6v3usmvA93UqfnnoWs5V0uYjEl9Al6MNU10MCGKLwpg==", + "license": "MIT", + "dependencies": { + "hermes-estree": "0.24.0" + } + }, + "node_modules/metro-cache": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-cache/-/metro-cache-0.81.0.tgz", + "integrity": "sha512-DyuqySicHXkHUDZFVJmh0ygxBSx6pCKUrTcSgb884oiscV/ROt1Vhye+x+OIHcsodyA10gzZtrVtxIFV4l9I4g==", + "license": "MIT", + "dependencies": { + "exponential-backoff": "^3.1.1", + "flow-enums-runtime": "^0.0.6", + "metro-core": "0.81.0" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-cache-key": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-cache-key/-/metro-cache-key-0.81.0.tgz", + "integrity": "sha512-qX/IwtknP9bQZL78OK9xeSvLM/xlGfrs6SlUGgHvrxtmGTRSsxcyqxR+c+7ch1xr05n62Gin/O44QKg5V70rNQ==", + "license": "MIT", + "dependencies": { + "flow-enums-runtime": "^0.0.6" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-config": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-config/-/metro-config-0.81.0.tgz", + "integrity": "sha512-6CinEaBe3WLpRlKlYXXu8r1UblJhbwD6Gtnoib5U8j6Pjp7XxMG9h/DGMeNp9aGLDu1OieUqiXpFo7O0/rR5Kg==", + "license": "MIT", + "dependencies": { + "connect": "^3.6.5", + "cosmiconfig": "^5.0.5", + "flow-enums-runtime": "^0.0.6", + "jest-validate": "^29.6.3", + "metro": "0.81.0", + "metro-cache": "0.81.0", + "metro-core": "0.81.0", + "metro-runtime": "0.81.0" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-core": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-core/-/metro-core-0.81.0.tgz", + "integrity": "sha512-CVkM5YCOAFkNMvJai6KzA0RpztzfEKRX62/PFMOJ9J7K0uq/UkOFLxcgpcncMIrfy0PbfEj811b69tjULUQe1Q==", + "license": "MIT", + "dependencies": { + "flow-enums-runtime": "^0.0.6", + "lodash.throttle": "^4.1.1", + "metro-resolver": "0.81.0" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-file-map": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-file-map/-/metro-file-map-0.81.0.tgz", + "integrity": "sha512-zMDI5uYhQCyxbye/AuFx/pAbsz9K+vKL7h1ShUXdN2fz4VUPiyQYRsRqOoVG1DsiCgzd5B6LW0YW77NFpjDQeg==", + "license": "MIT", + "dependencies": { + "anymatch": "^3.0.3", + "debug": "^2.2.0", + "fb-watchman": "^2.0.0", + "flow-enums-runtime": "^0.0.6", + "graceful-fs": "^4.2.4", + "invariant": "^2.2.4", + "jest-worker": "^29.6.3", + "micromatch": "^4.0.4", + "node-abort-controller": "^3.1.1", + "nullthrows": "^1.1.1", + "walker": "^1.0.7" + }, + "engines": { + "node": ">=18.18" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/metro-minify-terser": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-minify-terser/-/metro-minify-terser-0.81.0.tgz", + "integrity": "sha512-U2ramh3W822ZR1nfXgIk+emxsf5eZSg10GbQrT0ZizImK8IZ5BmJY+BHRIkQgHzWFpExOVxC7kWbGL1bZALswA==", + "license": "MIT", + "dependencies": { + "flow-enums-runtime": "^0.0.6", + "terser": "^5.15.0" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-resolver": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-resolver/-/metro-resolver-0.81.0.tgz", + "integrity": "sha512-Uu2Q+buHhm571cEwpPek8egMbdSTqmwT/5U7ZVNpK6Z2ElQBBCxd7HmFAslKXa7wgpTO2FAn6MqGeERbAtVDUA==", + "license": "MIT", + "dependencies": { + "flow-enums-runtime": "^0.0.6" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-runtime": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-runtime/-/metro-runtime-0.81.0.tgz", + "integrity": "sha512-6oYB5HOt37RuGz2eV4A6yhcl+PUTwJYLDlY9vhT+aVjbUWI6MdBCf69vc4f5K5Vpt+yOkjy+2LDwLS0ykWFwYw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.25.0", + "flow-enums-runtime": "^0.0.6" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-source-map": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-source-map/-/metro-source-map-0.81.0.tgz", + "integrity": "sha512-TzsVxhH83dyxg4A4+L1nzNO12I7ps5IHLjKGZH3Hrf549eiZivkdjYiq/S5lOB+p2HiQ+Ykcwtmcja95LIC62g==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.3", + "@babel/traverse--for-generate-function-map": "npm:@babel/traverse@^7.25.3", + "@babel/types": "^7.25.2", + "flow-enums-runtime": "^0.0.6", + "invariant": "^2.2.4", + "metro-symbolicate": "0.81.0", + "nullthrows": "^1.1.1", + "ob1": "0.81.0", + "source-map": "^0.5.6", + "vlq": "^1.0.0" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-symbolicate": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-symbolicate/-/metro-symbolicate-0.81.0.tgz", + "integrity": "sha512-C/1rWbNTPYp6yzID8IPuQPpVGzJ2rbWYBATxlvQ9dfK5lVNoxcwz77hjcY8ISLsRRR15hyd/zbjCNKPKeNgE1Q==", + "license": "MIT", + "dependencies": { + "flow-enums-runtime": "^0.0.6", + "invariant": "^2.2.4", + "metro-source-map": "0.81.0", + "nullthrows": "^1.1.1", + "source-map": "^0.5.6", + "through2": "^2.0.1", + "vlq": "^1.0.0" + }, + "bin": { + "metro-symbolicate": "src/index.js" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-transform-plugins": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-transform-plugins/-/metro-transform-plugins-0.81.0.tgz", + "integrity": "sha512-uErLAPBvttGCrmGSCa0dNHlOTk3uJFVEVWa5WDg6tQ79PRmuYRwzUgLhVzn/9/kyr75eUX3QWXN79Jvu4txt6Q==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.2", + "@babel/generator": "^7.25.0", + "@babel/template": "^7.25.0", + "@babel/traverse": "^7.25.3", + "flow-enums-runtime": "^0.0.6", + "nullthrows": "^1.1.1" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro-transform-worker": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/metro-transform-worker/-/metro-transform-worker-0.81.0.tgz", + "integrity": "sha512-HrQ0twiruhKy0yA+9nK5bIe3WQXZcC66PXTvRIos61/EASLAP2DzEmW7IxN/MGsfZegN2UzqL2CG38+mOB45vg==", + "license": "MIT", + "dependencies": { + "@babel/core": "^7.25.2", + "@babel/generator": "^7.25.0", + "@babel/parser": "^7.25.3", + "@babel/types": "^7.25.2", + "flow-enums-runtime": "^0.0.6", + "metro": "0.81.0", + "metro-babel-transformer": "0.81.0", + "metro-cache": "0.81.0", + "metro-cache-key": "0.81.0", + "metro-minify-terser": "0.81.0", + "metro-source-map": "0.81.0", + "metro-transform-plugins": "0.81.0", + "nullthrows": "^1.1.1" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/metro/node_modules/ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "license": "MIT" + }, + "node_modules/metro/node_modules/hermes-estree": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.24.0.tgz", + "integrity": "sha512-LyoXLB7IFzeZW0EvAbGZacbxBN7t6KKSDqFJPo3Ydow7wDlrDjXwsdiAHV6XOdvEN9MEuWXsSIFN4tzpyrXIHw==", + "license": "MIT" + }, + "node_modules/metro/node_modules/hermes-parser": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.24.0.tgz", + "integrity": "sha512-IJooSvvu2qNRe7oo9Rb04sUT4omtZqZqf9uq9WM25Tb6v3usmvA93UqfnnoWs5V0uYjEl9Al6MNU10MCGKLwpg==", + "license": "MIT", + "dependencies": { + "hermes-estree": "0.24.0" + } + }, + "node_modules/metro/node_modules/ws": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-collect": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-2.0.1.tgz", + "integrity": "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "license": "MIT", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT" + }, + "node_modules/nested-error-stacks": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.0.1.tgz", + "integrity": "sha512-SrQrok4CATudVzBS7coSz26QRSmlK9TzzoFbeKfcPBUFPjcQM9Rqvr/DlJkOrwI/0KcgvMub1n1g5Jt9EgRn4A==", + "license": "MIT" + }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "license": "MIT" + }, + "node_modules/node-abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", + "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", + "license": "MIT" + }, + "node_modules/node-dir": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", + "integrity": "sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==", + "license": "MIT", + "dependencies": { + "minimatch": "^3.0.2" + }, + "engines": { + "node": ">= 0.10.5" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-package-arg": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-11.0.3.tgz", + "integrity": "sha512-sHGJy8sOC1YraBywpzQlIKBE4pBbGbiF95U6Auspzyem956E0+FtDtsx1ZxlOJkQCZ1AFXAY/yuvtFYrOxF+Bw==", + "license": "ISC", + "dependencies": { + "hosted-git-info": "^7.0.0", + "proc-log": "^4.0.0", + "semver": "^7.3.5", + "validate-npm-package-name": "^5.0.0" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/nullthrows": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz", + "integrity": "sha512-2vPPEi+Z7WqML2jZYddDIfy5Dqb0r2fze2zTxNNknZaFpVHU3mFB3R+DWeJWGVx0ecvttSGlJTI+WG+8Z4cDWw==", + "license": "MIT" + }, + "node_modules/ob1": { + "version": "0.81.0", + "resolved": "https://registry.npmjs.org/ob1/-/ob1-0.81.0.tgz", + "integrity": "sha512-6Cvrkxt1tqaRdWqTAMcVYEiO5i1xcF9y7t06nFdjFqkfPsEloCf8WwhXdwBpNUkVYSQlSGS7cDgVQR86miBfBQ==", + "license": "MIT", + "dependencies": { + "flow-enums-runtime": "^0.0.6" + }, + "engines": { + "node": ">=18.18" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", + "integrity": "sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==", + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-3.4.0.tgz", + "integrity": "sha512-eNwHudNbO1folBP3JsZ19v9azXWtQZjICdr3Q0TDPIaeBQ3mXLrh54wM+er0+hSp+dWKf+Z8KM58CYzEyIYxYg==", + "license": "MIT", + "dependencies": { + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-spinners": "^2.0.0", + "log-symbols": "^2.2.0", + "strip-ansi": "^5.2.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/ora/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ora/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ora/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ora/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/ora/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "license": "MIT" + }, + "node_modules/ora/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/ora/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ora/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/ora/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "license": "MIT", + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", + "license": "MIT", + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/parse-png": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-png/-/parse-png-2.1.0.tgz", + "integrity": "sha512-Nt/a5SfCLiTnQAjx3fHlqp8hRgTL3z7kTQZzvIMS9uCAepnCyjpdEc6M/sz69WqMBdaDBw9sF1F1UaHROYzGkQ==", + "license": "MIT", + "dependencies": { + "pngjs": "^3.3.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/password-prompt": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/password-prompt/-/password-prompt-1.1.3.tgz", + "integrity": "sha512-HkrjG2aJlvF0t2BMH0e2LB/EHf3Lcq3fNMzy4GYHcQblAvOl+QQji1Lx7WRBMqpVK8p+KR7bCg7oqAMXtdgqyw==", + "license": "0BSD", + "dependencies": { + "ansi-escapes": "^4.3.2", + "cross-spawn": "^7.0.3" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pirates": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", + "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "license": "MIT", + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "license": "MIT", + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "license": "MIT", + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "license": "MIT", + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/plist": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.0.tgz", + "integrity": "sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==", + "license": "MIT", + "dependencies": { + "@xmldom/xmldom": "^0.8.8", + "base64-js": "^1.5.1", + "xmlbuilder": "^15.1.1" + }, + "engines": { + "node": ">=10.4.0" + } + }, + "node_modules/plist/node_modules/@xmldom/xmldom": { + "version": "0.8.10", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.10.tgz", + "integrity": "sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/plist/node_modules/xmlbuilder": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", + "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==", + "license": "MIT", + "engines": { + "node": ">=8.0" + } + }, + "node_modules/pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/postcss": { + "version": "8.4.49", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", + "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/pretty-bytes": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-5.6.0.tgz", + "integrity": "sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/proc-log": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-4.2.0.tgz", + "integrity": "sha512-g8+OnU/L2v+wyiVK+D5fA34J7EH8jZ8DDlvwhRCMxmMj7UCBvxiO1mGeN+36JXIKF4zevU4kRBd8lVgG9vLelA==", + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/promise": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/promise/-/promise-8.3.0.tgz", + "integrity": "sha512-rZPNPKTOYVNEEKFaq1HqTgOwZD+4/YHS5ukLzQCypkj+OkYx7iv0mA91lJlpPPZ8vMau3IIGj5Qlwrx+8iiSmg==", + "license": "MIT", + "dependencies": { + "asap": "~2.0.6" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/prop-types/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qrcode-terminal": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/qrcode-terminal/-/qrcode-terminal-0.11.0.tgz", + "integrity": "sha512-Uu7ii+FQy4Qf82G4xu7ShHhjhGahEpCWc3x8UavY3CTcWV+ufmmCtwkr7ZKsX42jdL0kr1B5FKUeqJvAn51jzQ==", + "bin": { + "qrcode-terminal": "bin/qrcode-terminal.js" + } + }, + "node_modules/queue": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz", + "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==", + "license": "MIT", + "dependencies": { + "inherits": "~2.0.3" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-devtools-core": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/react-devtools-core/-/react-devtools-core-5.3.2.tgz", + "integrity": "sha512-crr9HkVrDiJ0A4zot89oS0Cgv0Oa4OG1Em4jit3P3ZxZSKPMYyMjfwMqgcJna9o625g8oN87rBm8SWWrSTBZxg==", + "license": "MIT", + "dependencies": { + "shell-quote": "^1.6.1", + "ws": "^7" + } + }, + "node_modules/react-devtools-core/node_modules/ws": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz", + "integrity": "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/react-native": { + "version": "0.76.3", + "resolved": "https://registry.npmjs.org/react-native/-/react-native-0.76.3.tgz", + "integrity": "sha512-0TUhgmlouRNf6yuDIIAdbQl0g1VsONgCMsLs7Et64hjj5VLMCA7np+4dMrZvGZ3wRNqzgeyT9oWJsUm49AcwSQ==", + "license": "MIT", + "dependencies": { + "@jest/create-cache-key-function": "^29.6.3", + "@react-native/assets-registry": "0.76.3", + "@react-native/codegen": "0.76.3", + "@react-native/community-cli-plugin": "0.76.3", + "@react-native/gradle-plugin": "0.76.3", + "@react-native/js-polyfills": "0.76.3", + "@react-native/normalize-colors": "0.76.3", + "@react-native/virtualized-lists": "0.76.3", + "abort-controller": "^3.0.0", + "anser": "^1.4.9", + "ansi-regex": "^5.0.0", + "babel-jest": "^29.7.0", + "babel-plugin-syntax-hermes-parser": "^0.23.1", + "base64-js": "^1.5.1", + "chalk": "^4.0.0", + "commander": "^12.0.0", + "event-target-shim": "^5.0.1", + "flow-enums-runtime": "^0.0.6", + "glob": "^7.1.1", + "invariant": "^2.2.4", + "jest-environment-node": "^29.6.3", + "jsc-android": "^250231.0.0", + "memoize-one": "^5.0.0", + "metro-runtime": "^0.81.0", + "metro-source-map": "^0.81.0", + "mkdirp": "^0.5.1", + "nullthrows": "^1.1.1", + "pretty-format": "^29.7.0", + "promise": "^8.3.0", + "react-devtools-core": "^5.3.1", + "react-refresh": "^0.14.0", + "regenerator-runtime": "^0.13.2", + "scheduler": "0.24.0-canary-efb381bbf-20230505", + "semver": "^7.1.3", + "stacktrace-parser": "^0.1.10", + "whatwg-fetch": "^3.0.0", + "ws": "^6.2.3", + "yargs": "^17.6.2" + }, + "bin": { + "react-native": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/react": "^18.2.6", + "react": "^18.2.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-native-background-timer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/react-native-background-timer/-/react-native-background-timer-2.4.1.tgz", + "integrity": "sha512-TE4Kiy7jUyv+hugxDxitzu38sW1NqjCk4uE5IgU2WevLv7sZacaBc6PZKOShNRPGirLl1NWkaG3LDEkdb9Um5g==", + "license": "MIT", + "peerDependencies": { + "react-native": ">=0.47.0" + } + }, + "node_modules/react-native-get-random-values": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/react-native-get-random-values/-/react-native-get-random-values-1.11.0.tgz", + "integrity": "sha512-4BTbDbRmS7iPdhYLRcz3PGFIpFJBwNZg9g42iwa2P6FOv9vZj/xJc678RZXnLNZzd0qd7Q3CCF6Yd+CU2eoXKQ==", + "license": "MIT", + "dependencies": { + "fast-base64-decode": "^1.0.0" + }, + "peerDependencies": { + "react-native": ">=0.56" + } + }, + "node_modules/react-native-url-polyfill": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/react-native-url-polyfill/-/react-native-url-polyfill-1.3.0.tgz", + "integrity": "sha512-w9JfSkvpqqlix9UjDvJjm1EjSt652zVQ6iwCIj1cVVkwXf4jQhQgTNXY6EVTwuAmUjg6BC6k9RHCBynoLFo3IQ==", + "license": "MIT", + "dependencies": { + "whatwg-url-without-unicode": "8.0.0-3" + }, + "peerDependencies": { + "react-native": "*" + } + }, + "node_modules/react-native/node_modules/babel-plugin-syntax-hermes-parser": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/babel-plugin-syntax-hermes-parser/-/babel-plugin-syntax-hermes-parser-0.23.1.tgz", + "integrity": "sha512-uNLD0tk2tLUjGFdmCk+u/3FEw2o+BAwW4g+z2QVlxJrzZYOOPADroEcNtTPt5lNiScctaUmnsTkVEnOwZUOLhA==", + "license": "MIT", + "dependencies": { + "hermes-parser": "0.23.1" + } + }, + "node_modules/react-native/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/react-refresh": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.2.tgz", + "integrity": "sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readline": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz", + "integrity": "sha512-k2d6ACCkiNYz222Fs/iNze30rRJ1iIicW7JuX/7/cozvih6YCkFZH+J6mAFDVgv0dRBaAyr4jDqC95R2y4IADg==", + "license": "BSD" + }, + "node_modules/recast": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.21.5.tgz", + "integrity": "sha512-hjMmLaUXAm1hIuTqOdeYObMslq/q+Xff6QE3Y2P+uoHAg2nmVlLBps2hzh1UJDdMtDTMXOFewK6ky51JQIeECg==", + "license": "MIT", + "dependencies": { + "ast-types": "0.15.2", + "esprima": "~4.0.0", + "source-map": "~0.6.1", + "tslib": "^2.0.1" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/recast/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/regenerate": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", + "integrity": "sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==", + "license": "MIT" + }, + "node_modules/regenerate-unicode-properties": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/regenerate-unicode-properties/-/regenerate-unicode-properties-10.2.0.tgz", + "integrity": "sha512-DqHn3DwbmmPVzeKj9woBadqmXxLvQoQIwu7nopMc72ztvxVmVk2SBhSnx67zuye5TP+lJsb/TBQsjLKhnDf3MA==", + "license": "MIT", + "dependencies": { + "regenerate": "^1.4.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, + "node_modules/regenerator-transform": { + "version": "0.15.2", + "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.15.2.tgz", + "integrity": "sha512-hfMp2BoF0qOk3uc5V20ALGDS2ddjQaLrdl7xrGXvAIow7qeWRM2VA2HuCHkUKk9slq3VwEwLNK3DFBqDfPGYtg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.8.4" + } + }, + "node_modules/regexpu-core": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-6.2.0.tgz", + "integrity": "sha512-H66BPQMrv+V16t8xtmq+UC0CBpiTBA60V8ibS1QVReIp8T1z8hwFxqcGzm9K6lgsN7sB5edVH8a+ze6Fqm4weA==", + "license": "MIT", + "dependencies": { + "regenerate": "^1.4.2", + "regenerate-unicode-properties": "^10.2.0", + "regjsgen": "^0.8.0", + "regjsparser": "^0.12.0", + "unicode-match-property-ecmascript": "^2.0.0", + "unicode-match-property-value-ecmascript": "^2.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/regjsgen": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.8.0.tgz", + "integrity": "sha512-RvwtGe3d7LvWiDQXeQw8p5asZUmfU1G/l6WbUXeHta7Y2PEIvBTwH6E2EfmYUK8pxcxEdEmaomqyp0vZZ7C+3Q==", + "license": "MIT" + }, + "node_modules/regjsparser": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.12.0.tgz", + "integrity": "sha512-cnE+y8bz4NhMjISKbgeVJtqNbtf5QpjZP+Bslo+UqkIt9QPnX9q095eiRRASJG1/tz6dlNr6Z5NsBiWYokp6EQ==", + "license": "BSD-2-Clause", + "dependencies": { + "jsesc": "~3.0.2" + }, + "bin": { + "regjsparser": "bin/parser" + } + }, + "node_modules/regjsparser/node_modules/jsesc": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/remove-trailing-slash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz", + "integrity": "sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA==", + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requireg": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/requireg/-/requireg-0.2.2.tgz", + "integrity": "sha512-nYzyjnFcPNGR3lx9lwPPPnuQxv6JWEZd2Ci0u9opN7N5zUEPIhY/GbL3vMGOr2UXwEg9WwSyV9X9Y/kLFgPsOg==", + "dependencies": { + "nested-error-stacks": "~2.0.1", + "rc": "~1.2.7", + "resolve": "~1.7.1" + }, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/requireg/node_modules/resolve": { + "version": "1.7.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.7.1.tgz", + "integrity": "sha512-c7rwLofp8g1U+h1KNyHL/jicrKg1Ek4q+Lr33AL65uZTinUZHe30D5HlyN5V9NW0JX1D5dXQ4jqW5l7Sy/kGfw==", + "license": "MIT", + "dependencies": { + "path-parse": "^1.0.5" + } + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-workspace-root": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-workspace-root/-/resolve-workspace-root-2.0.0.tgz", + "integrity": "sha512-IsaBUZETJD5WsI11Wt8PKHwaIe45or6pwNc8yflvLJ4DWtImK9kuLoH5kUva/2Mmx/RdIyr4aONNSa2v9LTJsw==", + "license": "MIT" + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", + "license": "MIT", + "dependencies": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/restore-cursor/node_modules/mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/restore-cursor/node_modules/onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "license": "ISC" + }, + "node_modules/scheduler": { + "version": "0.24.0-canary-efb381bbf-20230505", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.24.0-canary-efb381bbf-20230505.tgz", + "integrity": "sha512-ABvovCDe/k9IluqSh4/ISoq8tIJnW8euVAWYt5j/bg6dRnqwQwiGO1F/V4AyK96NGF/FB04FhOUDuWj8IKfABA==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/selfsigned": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-2.4.1.tgz", + "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "license": "MIT", + "dependencies": { + "@types/node-forge": "^1.3.0", + "node-forge": "^1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.0.tgz", + "integrity": "sha512-DrfFnPzblFmNrIZzg5RzHegbiRWg7KMR7btwi2yjHwx06zsUbO5g613sVwEV7FTwmzJu+Io0lJe2GJ3LxqpvBQ==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.1.tgz", + "integrity": "sha512-p4rRk4f23ynFEfcD9LA0xRYngj+IyGiEYyqqOak8kaN0TvNmuxC2dcVeBn62GpCeR2CpWqyHCNScTP91QbAVFg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serialize-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-2.1.0.tgz", + "integrity": "sha512-ghgmKt5o4Tly5yEG/UJp8qTd0AN7Xalw4XBtDEKP655B699qMEtra1WlXeE6WIvdEG481JvRxULKsInq/iNysw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-static/node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static/node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-static/node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.2.tgz", + "integrity": "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/simple-plist": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/simple-plist/-/simple-plist-1.3.1.tgz", + "integrity": "sha512-iMSw5i0XseMnrhtIzRb7XpQEXepa9xhWxGUojHBL43SIpQuDQkh3Wpy67ZbDzZVr6EKxvwVChnVpdl8hEVLDiw==", + "license": "MIT", + "dependencies": { + "bplist-creator": "0.1.0", + "bplist-parser": "0.3.1", + "plist": "^3.0.5" + } + }, + "node_modules/simple-plist/node_modules/bplist-creator": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bplist-creator/-/bplist-creator-0.1.0.tgz", + "integrity": "sha512-sXaHZicyEEmY86WyueLTQesbeoH/mquvarJaQNbjuOQO+7gbFcDEWqKmcWA4cOTLzFlfgvkiVxolk1k5bBIpmg==", + "license": "MIT", + "dependencies": { + "stream-buffers": "2.2.x" + } + }, + "node_modules/simple-plist/node_modules/bplist-parser": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.3.1.tgz", + "integrity": "sha512-PyJxiNtA5T2PlLIeBot4lbp7rj4OadzjnMZD/G5zuBNt8ei/yCU7+wW0h2bag9vr8c+/WuRWmSxbqAl9hL1rBA==", + "license": "MIT", + "dependencies": { + "big-integer": "1.6.x" + }, + "engines": { + "node": ">= 5.10.0" + } + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "license": "MIT", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/ssri": { + "version": "10.0.6", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-10.0.6.tgz", + "integrity": "sha512-MGrFH9Z4NP9Iyhqn16sDtBpRRNJ0Y2hNa6D65h736fVSaPCHr4DM4sWUNvVaSuC+0OBGhwsrydQwmgfg5LncqQ==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/stackframe": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", + "integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==", + "license": "MIT" + }, + "node_modules/stacktrace-parser": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/stacktrace-parser/-/stacktrace-parser-0.1.10.tgz", + "integrity": "sha512-KJP1OCML99+8fhOHxwwzyWrlUuVX5GQ0ZpJTd1DFXhdkrvg1szxfHhawXUZ3g9TkXORQd4/WG68jMlQZ2p8wlg==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.7.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/stream-buffers": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-2.2.0.tgz", + "integrity": "sha512-uyQK/mx5QjHun80FLJTfaWE7JtwfRMKBLkMne6udYOmvH0CawotVa7TfgYHzAnpphn4+TweIx1QKMnRIbipmUg==", + "license": "Unlicense", + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/structured-headers": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/structured-headers/-/structured-headers-0.4.1.tgz", + "integrity": "sha512-0MP/Cxx5SzeeZ10p/bZI0S6MpgD+yxAhi1BOQ34jgnMXsCq3j1t6tQnZu+KdlL7dvJTLT3g9xN8tl10TqgFMcg==", + "license": "MIT" + }, + "node_modules/sucrase": { + "version": "3.35.0", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", + "integrity": "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "glob": "^10.3.10", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/sucrase/node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/sudo-prompt": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-9.1.1.tgz", + "integrity": "sha512-es33J1g2HjMpyAhz8lOR+ICmXXAqTuKbuXuUWLhOLew20oN9oUCgCJx615U/v7aioZg7IX5lIh9x34vwneu4pA==", + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-hyperlinks": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tar": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "license": "ISC", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^5.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", + "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "license": "ISC", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/temp": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", + "integrity": "sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==", + "license": "MIT", + "dependencies": { + "rimraf": "~2.6.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/temp-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", + "integrity": "sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/temp/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/temp/node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/tempy": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/tempy/-/tempy-0.7.1.tgz", + "integrity": "sha512-vXPxwOyaNVi9nyczO16mxmHGpl6ASC5/TVhRRHpqeYHvKQm58EaWNvZXxAhR0lYYnBOQFjXjhzeLsaXdjxLjRg==", + "license": "MIT", + "dependencies": { + "del": "^6.0.0", + "is-stream": "^2.0.0", + "temp-dir": "^2.0.0", + "type-fest": "^0.16.0", + "unique-string": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tempy/node_modules/type-fest": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.16.0.tgz", + "integrity": "sha512-eaBzG6MxNzEn9kiwvtre90cXaNLkmadMWa1zQMs3XORCXNbsH/OewwbxC5ia9dCxIxnTAsSxXJaa/p5y8DlvJg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/terminal-link": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", + "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^4.2.1", + "supports-hyperlinks": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/terser": { + "version": "5.37.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.37.0.tgz", + "integrity": "sha512-B8wRRkmre4ERucLM/uXx4MOV5cbnOlVAqUst+1+iLKPI0dOgFO28f84ptoQt9HEI537PMzfYa/d+GEPKTRXmYA==", + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "license": "MIT", + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "license": "MIT", + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/throat": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/throat/-/throat-5.0.0.tgz", + "integrity": "sha512-fcwX4mndzpLQKBS1DVYhGAcYaYt7vsHNIvQV+WXMvnow5cgjPphq5CaayLaGsjRdSCKZFNGt7/GYAuXaNOiYCA==", + "license": "MIT" + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "license": "MIT", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "license": "MIT", + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "license": "Apache-2.0" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.7.1.tgz", + "integrity": "sha512-Ne2YiiGN8bmrmJJEuTWTLJR32nh/JdL1+PSicowtNb0WFpn59GK8/lfD61bVtzguz7b3PBt74nxpv/Pw5po5Rg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=8" + } + }, + "node_modules/ua-parser-js": { + "version": "1.0.40", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.40.tgz", + "integrity": "sha512-z6PJ8Lml+v3ichVojCiB8toQJBuwR42ySM4ezjXIqXK3M0HczmKQ3LF4rhU55PfD99KEEXQG6yb7iOMyvYuHew==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + } + ], + "license": "MIT", + "bin": { + "ua-parser-js": "script/cli.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/undici": { + "version": "6.21.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.1.tgz", + "integrity": "sha512-q/1rj5D0/zayJB2FraXdaWxbhWiNKDvu8naDT2dl1yTlvJp4BLtOcp2a5BvgGNQpYYJzau7tf1WgKv3b+7mqpQ==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "license": "MIT" + }, + "node_modules/unicode-canonical-property-names-ecmascript": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz", + "integrity": "sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-ecmascript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-ecmascript/-/unicode-match-property-ecmascript-2.0.0.tgz", + "integrity": "sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==", + "license": "MIT", + "dependencies": { + "unicode-canonical-property-names-ecmascript": "^2.0.0", + "unicode-property-aliases-ecmascript": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-match-property-value-ecmascript": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.2.0.tgz", + "integrity": "sha512-4IehN3V/+kkr5YeSSDDQG8QLqO26XpL2XP3GQtqwlT/QYSECAwFztxVHjlbh0+gjJ3XmNLS0zDsbgs9jWKExLg==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unicode-property-aliases-ecmascript": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz", + "integrity": "sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/unique-filename": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz", + "integrity": "sha512-afXhuC55wkAmZ0P18QsVE6kp8JaxrEokN2HGIoIVv2ijHQd419H0+6EigAFcIzXeMIkcIkNBpB3L/DXB3cTS/g==", + "license": "ISC", + "dependencies": { + "unique-slug": "^4.0.0" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/unique-slug": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-4.0.0.tgz", + "integrity": "sha512-WrcA6AyEfqDX5bWige/4NQfPZMtASNVxdmWR76WESYQVAACSgWcR6e9i0mofqqBxYFtL4oAxPIptY73/0YE1DQ==", + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "license": "MIT", + "dependencies": { + "crypto-random-string": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.2.tgz", + "integrity": "sha512-PPypAm5qvlD7XMZC3BujecnaOxwhrtoFR+Dqkk5Aa/6DssiH0ibKoketaj9w8LP7Bont1rYeoV5plxD7RTEPRg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/validate-npm-package-name": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-5.0.1.tgz", + "integrity": "sha512-OljLrQ9SQdOUqTaQxqL5dEfZWrXExyyWsozYlAWFawPVNuD83igl7uJD2RTkNMbniIYgt8l81eCJGIdQF7avLQ==", + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vlq": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/vlq/-/vlq-1.0.1.tgz", + "integrity": "sha512-gQpnTgkubC6hQgdIcRdYGDSDc+SaujOdyesZQMv6JlfQee/9Mp0Qhnys6WxDWvQnL5WZdT7o2Ul187aSt0Rq+w==", + "license": "MIT" + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", + "license": "MIT" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/whatwg-url-without-unicode": { + "version": "8.0.0-3", + "resolved": "https://registry.npmjs.org/whatwg-url-without-unicode/-/whatwg-url-without-unicode-8.0.0-3.tgz", + "integrity": "sha512-HoKuzZrUlgpz35YO27XgD28uh/WJH4B0+3ttFqRo//lmq+9T/mIOJ6kqmINI9HpUpz1imRC/nR/lxKpJiv0uig==", + "license": "MIT", + "dependencies": { + "buffer": "^5.4.3", + "punycode": "^2.1.1", + "webidl-conversions": "^5.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/whatwg-url/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wonka": { + "version": "6.3.4", + "resolved": "https://registry.npmjs.org/wonka/-/wonka-6.3.4.tgz", + "integrity": "sha512-CjpbqNtBGNAeyNS/9W6q3kSkKE52+FjIj7AkFlLr11s/VWGUu6a2CdYSdGxocIhIVjaW/zchesBQUKPVU69Cqg==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "node_modules/ws": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.3.tgz", + "integrity": "sha512-jmTjYU0j60B+vHey6TfR3Z7RD61z/hmxBS3VMSGIrroOWXQEneK1zNuotOUrGyBHQj0yrpsLHPWtigEFd13ndA==", + "license": "MIT", + "dependencies": { + "async-limiter": "~1.0.0" + } + }, + "node_modules/xcode": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/xcode/-/xcode-3.0.1.tgz", + "integrity": "sha512-kCz5k7J7XbJtjABOvkc5lJmkiDh8VhjVCGNiqdKCscmVpdVUpEAyXv1xmCLkQJ5dsHqx3IPO4XW+NTDhU/fatA==", + "license": "Apache-2.0", + "dependencies": { + "simple-plist": "^1.1.0", + "uuid": "^7.0.3" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/xcode/node_modules/uuid": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz", + "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/xml2js": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.0.tgz", + "integrity": "sha512-eLTh0kA8uHceqesPqSE+VvO1CDDJWMwlQfB6LuN6T8w6MaDJ8Txm8P7s5cHD0miF0V+GGTZrDQfxPZQVsur33w==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xml2js/node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/xmlbuilder": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-14.0.0.tgz", + "integrity": "sha512-ts+B2rSe4fIckR6iquDjsKbQFK2NlUk6iG5nf14mDEyldgoc2nEKZ3jZWMPTxGQwVgToSjt6VGIho1H8/fNFTg==", + "license": "MIT", + "engines": { + "node": ">=8.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/examples/bot-ready-signalling/client/react-native/package.json b/examples/bot-ready-signalling/client/react-native/package.json new file mode 100644 index 000000000..c35307874 --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/package.json @@ -0,0 +1,30 @@ +{ + "name": "bot-ready-rn", + "version": "1.0.0", + "scripts": { + "start": "expo start --dev-client", + "android": "expo run:android --device", + "ios": "expo run:ios --device", + "web": "expo start --web" + }, + "dependencies": { + "@config-plugins/react-native-webrtc": "^10.0.0", + "@daily-co/config-plugin-rn-daily-js": "0.0.7", + "@daily-co/react-native-daily-js": "^0.70.0", + "@daily-co/react-native-webrtc": "^118.0.3-daily.2", + "@react-native-async-storage/async-storage": "1.23.1", + "expo": "^52.0.0", + "expo-build-properties": "~0.13.1", + "expo-dev-client": "~5.0.5", + "expo-splash-screen": "~0.29.16", + "expo-status-bar": "~2.0.0", + "react": "18.3.1", + "react-native": "0.76.3", + "react-native-background-timer": "^2.4.1", + "react-native-get-random-values": "^1.11.0" + }, + "devDependencies": { + "@babel/core": "^7.12.9" + }, + "private": true +} diff --git a/examples/bot-ready-signalling/client/react-native/src/App.js b/examples/bot-ready-signalling/client/react-native/src/App.js new file mode 100644 index 000000000..6c17f09b6 --- /dev/null +++ b/examples/bot-ready-signalling/client/react-native/src/App.js @@ -0,0 +1,153 @@ +import { + View, + SafeAreaView, + StyleSheet, + Text, + Button, + TextInput, +} from "react-native"; +import React, { useEffect, useState, useCallback } from "react"; +import Daily, { + DailyMediaView, + DailyEventObjectParticipant, +} from "@daily-co/react-native-daily-js"; + +const styles = StyleSheet.create({ + safeArea: { + flex: 1, + backgroundColor: "#f7f9fa", + width: "100%", + }, + outCallContainer: { + flex: 1, + justifyContent: "center", + alignItems: "center", + }, + inCallContainer: { + position: "absolute", + width: "100%", + height: "100%", + }, + dailyMediaView: { + flex: 1, + aspectRatio: 9 / 16, + }, + roomUrlInput: { + borderRadius: 8, + marginVertical: 8, + padding: 12, + fontStyle: "normal", + fontWeight: "normal", + borderWidth: 1, + width: "100%", + }, + infoView: { + flex: 1, + alignItems: "center", + justifyContent: "center", + }, + controlButton: { + flex: 1, + }, +}); + +const ROOM_URL_TEMPLATE = "https://filipi.daily.co/public"; + +export default function App() { + const [videoTrack, setVideoTrack] = useState(); + const [callObject, setCallObject] = useState(); + const [inCall, setInCall] = useState(false); + const [roomUrl, setRoomUrl] = useState(ROOM_URL_TEMPLATE); + const [remoteParticipantCount, setRemoteParticipantCount] = useState(0); + + const handleNewParticipantsState = (event: DailyEventObjectParticipant) => { + const participant = event.participant; + // Early out as needed to avoid display the local participant's video + if (participant.local) { + return; + } + const videoTrack = participant.tracks.video; + setVideoTrack(videoTrack.persistentTrack); + // Set participant count minus the local participant + setRemoteParticipantCount(callObject.participantCounts().present - 1); + }; + + const joinRoom = () => { + console.log("Joining room"); + callObject.join({ + url: roomUrl, + }); + }; + + const leaveRoom = async () => { + console.log("Leaving the room"); + await callObject.leave(); + }; + + // Create the callObject and join the meeting + useEffect(() => { + const callObject = Daily.createCallObject(); + setCallObject(callObject); + return () => {}; + }, []); + + //Add the listeners + useEffect(() => { + if (!callObject) { + return; + } + callObject + .on("joined-meeting", () => setInCall(true)) + .on("left-meeting", () => setInCall(false)) + .on("participant-joined", handleNewParticipantsState) + .on("participant-updated", handleNewParticipantsState) + .on("participant-left", handleNewParticipantsState); + return () => {}; + }, [callObject]); + + return ( + + {inCall ? ( + + {remoteParticipantCount > 0 ? ( + + ) : ( + + No one else is in the call yet! + Invite others to join the call using this link: + {roomUrl} + + )} + + + ) : ( + + + Not in a call yet + { + setRoomUrl(newRoomURL); + }} + /> + + + + )} + + ); +} From 24cf106ed2c9efe0179e4bee9880d0461c8cc1cb Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 30 Jan 2025 09:14:18 -0300 Subject: [PATCH 003/427] Refactoring the code to ask for the room that it should connect. --- .../client/react-native/src/App.js | 230 +++++++----------- 1 file changed, 88 insertions(+), 142 deletions(-) diff --git a/examples/bot-ready-signalling/client/react-native/src/App.js b/examples/bot-ready-signalling/client/react-native/src/App.js index 6c17f09b6..85b10bec4 100644 --- a/examples/bot-ready-signalling/client/react-native/src/App.js +++ b/examples/bot-ready-signalling/client/react-native/src/App.js @@ -1,153 +1,99 @@ -import { - View, - SafeAreaView, - StyleSheet, - Text, - Button, - TextInput, -} from "react-native"; -import React, { useEffect, useState, useCallback } from "react"; -import Daily, { - DailyMediaView, - DailyEventObjectParticipant, -} from "@daily-co/react-native-daily-js"; +import React, { useState, useEffect } from 'react'; +import {SafeAreaView, View, Text, Button, StyleSheet, ScrollView} from 'react-native'; +import Daily from "@daily-co/react-native-daily-js"; -const styles = StyleSheet.create({ - safeArea: { - flex: 1, - backgroundColor: "#f7f9fa", - width: "100%", - }, - outCallContainer: { - flex: 1, - justifyContent: "center", - alignItems: "center", - }, - inCallContainer: { - position: "absolute", - width: "100%", - height: "100%", - }, - dailyMediaView: { - flex: 1, - aspectRatio: 9 / 16, - }, - roomUrlInput: { - borderRadius: 8, - marginVertical: 8, - padding: 12, - fontStyle: "normal", - fontWeight: "normal", - borderWidth: 1, - width: "100%", - }, - infoView: { - flex: 1, - alignItems: "center", - justifyContent: "center", - }, - controlButton: { - flex: 1, - }, -}); +const CallScreen = () => { + const [connectionStatus, setConnectionStatus] = useState('Disconnected'); + const [isConnected, setIsConnected] = useState(false); + const [callObject, setCallObject] = useState(null); + const [logs, setLogs] = useState([]); -const ROOM_URL_TEMPLATE = "https://filipi.daily.co/public"; - -export default function App() { - const [videoTrack, setVideoTrack] = useState(); - const [callObject, setCallObject] = useState(); - const [inCall, setInCall] = useState(false); - const [roomUrl, setRoomUrl] = useState(ROOM_URL_TEMPLATE); - const [remoteParticipantCount, setRemoteParticipantCount] = useState(0); - - const handleNewParticipantsState = (event: DailyEventObjectParticipant) => { - const participant = event.participant; - // Early out as needed to avoid display the local participant's video - if (participant.local) { - return; - } - const videoTrack = participant.tracks.video; - setVideoTrack(videoTrack.persistentTrack); - // Set participant count minus the local participant - setRemoteParticipantCount(callObject.participantCounts().present - 1); - }; - - const joinRoom = () => { - console.log("Joining room"); - callObject.join({ - url: roomUrl, - }); - }; - - const leaveRoom = async () => { - console.log("Leaving the room"); - await callObject.leave(); - }; - - // Create the callObject and join the meeting useEffect(() => { - const callObject = Daily.createCallObject(); - setCallObject(callObject); - return () => {}; - }, []); - - //Add the listeners - useEffect(() => { - if (!callObject) { - return; + if (callObject) { + setupTrackListeners(callObject); } - callObject - .on("joined-meeting", () => setInCall(true)) - .on("left-meeting", () => setInCall(false)) - .on("participant-joined", handleNewParticipantsState) - .on("participant-updated", handleNewParticipantsState) - .on("participant-left", handleNewParticipantsState); - return () => {}; }, [callObject]); + const log = (message) => { + setLogs((prevLogs) => [...prevLogs, `${new Date().toISOString()} - ${message}`]); + console.log(message); + }; + + const setupTrackListeners = (callObject) => { + callObject.on("joined-meeting", () => { + setConnectionStatus('Connected'); + setIsConnected(true); + log('Client connected'); + }); + callObject.on("left-meeting", () => { + setConnectionStatus('Disconnected'); + setIsConnected(false); + log('Client disconnected'); + }); + callObject.on("error", (evt) => log(`Error: ${evt.error}`)); + }; + + const connect = async () => { + try { + const callObject = Daily.createCallObject({ subscribeToTracksAutomatically: true }); + setCallObject(callObject); + const connectionUrl = 'http://192.168.1.16:7860/connect' + const res = await fetch(connectionUrl, { method: "POST", headers: { "Content-Type": "application/json" } }); + const roomInfo = await res.json(); + await callObject.join({ url: roomInfo.room_url }); + } catch (error) { + log(`Error connecting: ${error.message}`); + } + }; + + const disconnect = async () => { + if (callObject) { + try { + await callObject.leave(); + await callObject.destroy(); + setCallObject(null); + } catch (error) { + log(`Error disconnecting: ${error.message}`); + } + } + }; + return ( - - {inCall ? ( - - {remoteParticipantCount > 0 ? ( - - ) : ( - - No one else is in the call yet! - Invite others to join the call using this link: - {roomUrl} + + + + Status: {connectionStatus} + + - - ) : ( - - - Not in a call yet - { - setRoomUrl(newRoomURL); - }} - /> - - )} - + ); -} +}; + +const styles = StyleSheet.create({ + safeArea: { flex: 1, backgroundColor: '#f0f0f0', padding: 20 }, + container: { flex: 1, maxWidth: 1200, margin: 'auto' }, + statusBar: { flexDirection: 'row', justifyContent: 'space-between', alignItems: 'center', padding: 10, backgroundColor: '#fff', borderRadius: 8, marginBottom: 20 }, + status: { fontWeight: 'bold' }, + controls: { flexDirection: 'row', gap: 10 }, + debugPanel: { backgroundColor: '#fff', borderRadius: 8, padding: 20 }, + debugTitle: { fontSize: 16, fontWeight: 'bold' }, + debugLog: { height: 200, overflow: 'scroll', backgroundColor: '#f8f8f8', padding: 10, borderRadius: 4, fontFamily: 'monospace', fontSize: 12, lineHeight: 1.4 }, +}); + +export default CallScreen; From 2b6a9922078d8de966f0b184d8c8ac92fe0e9aff Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 30 Jan 2025 09:37:33 -0300 Subject: [PATCH 004/427] Sending the app-message to start playing audio once the track has started. --- .../client/react-native/src/App.js | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/examples/bot-ready-signalling/client/react-native/src/App.js b/examples/bot-ready-signalling/client/react-native/src/App.js index 85b10bec4..5b9e439f9 100644 --- a/examples/bot-ready-signalling/client/react-native/src/App.js +++ b/examples/bot-ready-signalling/client/react-native/src/App.js @@ -30,7 +30,26 @@ const CallScreen = () => { setIsConnected(false); log('Client disconnected'); }); + callObject.on("participant-left", () => { + // When the bot leaves, we are also disconnecting from the call + disconnect().catch((err) => { + log(`Failed to disconnect ${err}`); + }) + }); callObject.on("error", (evt) => log(`Error: ${evt.error}`)); + // Other events just for awareness + callObject.on("track-started", (evt) => { + handleEventToConsole(evt) + log("Will send the audio message to play the audio at the next tick") + callObject.sendAppMessage("playable") + }); + callObject.on("track-stopped", handleEventToConsole); + callObject.on("participant-joined", handleEventToConsole); + callObject.on("participant-updated", handleEventToConsole); + }; + + const handleEventToConsole = (evt) => { + log(`Received event: ${evt.action}`); }; const connect = async () => { @@ -72,7 +91,6 @@ const CallScreen = () => { Debug Info - Debug logs will appear here... {logs.map((logEntry, index) => ( {logEntry} From a48e5d07142f27207c081afb10e89a603c183540 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 30 Jan 2025 10:14:37 -0300 Subject: [PATCH 005/427] Only sending the message when it is a remote audio track. --- .../client/react-native/src/App.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/bot-ready-signalling/client/react-native/src/App.js b/examples/bot-ready-signalling/client/react-native/src/App.js index 5b9e439f9..2d07eca1e 100644 --- a/examples/bot-ready-signalling/client/react-native/src/App.js +++ b/examples/bot-ready-signalling/client/react-native/src/App.js @@ -36,13 +36,16 @@ const CallScreen = () => { log(`Failed to disconnect ${err}`); }) }); + // Trigger so the bot can start sending audio + callObject.on("track-started", (evt) => { + if (evt.track.kind === "audio" && evt.participant.local === false) { + handleEventToConsole(evt) + log("Sending the message that will trigger the bot to play the audio.") + callObject.sendAppMessage("playable") + } + }); callObject.on("error", (evt) => log(`Error: ${evt.error}`)); // Other events just for awareness - callObject.on("track-started", (evt) => { - handleEventToConsole(evt) - log("Will send the audio message to play the audio at the next tick") - callObject.sendAppMessage("playable") - }); callObject.on("track-stopped", handleEventToConsole); callObject.on("participant-joined", handleEventToConsole); callObject.on("participant-updated", handleEventToConsole); From 75ca0571bb374aefb9b385836145e52198e153f7 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 30 Jan 2025 10:31:04 -0300 Subject: [PATCH 006/427] Improving the layout from the bot ready react native demo. --- .../client/react-native/src/App.js | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/bot-ready-signalling/client/react-native/src/App.js b/examples/bot-ready-signalling/client/react-native/src/App.js index 2d07eca1e..3f56904dc 100644 --- a/examples/bot-ready-signalling/client/react-native/src/App.js +++ b/examples/bot-ready-signalling/client/react-native/src/App.js @@ -86,20 +86,20 @@ const CallScreen = () => { Status: {connectionStatus} - + + + + + + +

+

Debug Info

+
+
+ + + + + + + diff --git a/examples/instant-voice/client/javascript/package-lock.json b/examples/instant-voice/client/javascript/package-lock.json new file mode 100644 index 000000000..54ed9131c --- /dev/null +++ b/examples/instant-voice/client/javascript/package-lock.json @@ -0,0 +1,1449 @@ +{ + "name": "client", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "client", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@pipecat-ai/client-js": "^0.3.2", + "@pipecat-ai/daily-transport": "^0.3.5" + }, + "devDependencies": { + "@types/node": "^22.13.1", + "@vitejs/plugin-react-swc": "^3.7.2", + "typescript": "^5.7.3", + "vite": "^6.0.2" + } + }, + "node_modules/@babel/runtime": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@daily-co/daily-js": { + "version": "0.73.0", + "resolved": "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.73.0.tgz", + "integrity": "sha512-Wz8c60hgmkx8fcEeDAi4L4J0rbafiihWKyXFyhYoFYPsw2OdChHpA4RYwIB+1enRws5IK+/HdmzFDYLQsB4A6w==", + "license": "BSD-2-Clause", + "dependencies": { + "@babel/runtime": "^7.12.5", + "@sentry/browser": "^8.33.1", + "bowser": "^2.8.1", + "dequal": "^2.0.3", + "events": "^3.1.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.0.tgz", + "integrity": "sha512-WtKdFM7ls47zkKHFVzMz8opM7LkcsIp9amDUBIAWirg70RM71WRSjdILPsY5Uv1D42ZpUfaPILDlfactHgsRkw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.0.tgz", + "integrity": "sha512-arAtTPo76fJ/ICkXWetLCc9EwEHKaeya4vMrReVlEIUCAUncH7M4bhMQ+M9Vf+FFOZJdTNMXNBrWwW+OXWpSew==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.0.tgz", + "integrity": "sha512-Vsm497xFM7tTIPYK9bNTYJyF/lsP590Qc1WxJdlB6ljCbdZKU9SY8i7+Iin4kyhV/KV5J2rOKsBQbB77Ab7L/w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.0.tgz", + "integrity": "sha512-t8GrvnFkiIY7pa7mMgJd7p8p8qqYIz1NYiAoKc75Zyv73L3DZW++oYMSHPRarcotTKuSs6m3hTOa5CKHaS02TQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz", + "integrity": "sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.0.tgz", + "integrity": "sha512-rgtz6flkVkh58od4PwTRqxbKH9cOjaXCMZgWD905JOzjFKW+7EiUObfd/Kav+A6Gyud6WZk9w+xu6QLytdi2OA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.0.tgz", + "integrity": "sha512-6Mtdq5nHggwfDNLAHkPlyLBpE5L6hwsuXZX8XNmHno9JuL2+bg2BX5tRkwjyfn6sKbxZTq68suOjgWqCicvPXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.0.tgz", + "integrity": "sha512-D3H+xh3/zphoX8ck4S2RxKR6gHlHDXXzOf6f/9dbFt/NRBDIE33+cVa49Kil4WUjxMGW0ZIYBYtaGCa2+OsQwQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.0.tgz", + "integrity": "sha512-gJKIi2IjRo5G6Glxb8d3DzYXlxdEj2NlkixPsqePSZMhLudqPhtZ4BUrpIuTjJYXxvF9njql+vRjB2oaC9XpBw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.0.tgz", + "integrity": "sha512-TDijPXTOeE3eaMkRYpcy3LarIg13dS9wWHRdwYRnzlwlA370rNdZqbcp0WTyyV/k2zSxfko52+C7jU5F9Tfj1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.0.tgz", + "integrity": "sha512-K40ip1LAcA0byL05TbCQ4yJ4swvnbzHscRmUilrmP9Am7//0UjPreh4lpYzvThT2Quw66MhjG//20mrufm40mA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.0.tgz", + "integrity": "sha512-0mswrYP/9ai+CU0BzBfPMZ8RVm3RGAN/lmOMgW4aFUSOQBjA31UP8Mr6DDhWSuMwj7jaWOT0p0WoZ6jeHhrD7g==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.0.tgz", + "integrity": "sha512-hIKvXm0/3w/5+RDtCJeXqMZGkI2s4oMUGj3/jM0QzhgIASWrGO5/RlzAzm5nNh/awHE0A19h/CvHQe6FaBNrRA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.0.tgz", + "integrity": "sha512-HcZh5BNq0aC52UoocJxaKORfFODWXZxtBaaZNuN3PUX3MoDsChsZqopzi5UupRhPHSEHotoiptqikjN/B77mYQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.0.tgz", + "integrity": "sha512-bEh7dMn/h3QxeR2KTy1DUszQjUrIHPZKyO6aN1X4BCnhfYhuQqedHaa5MxSQA/06j3GpiIlFGSsy1c7Gf9padw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.0.tgz", + "integrity": "sha512-ZcQ6+qRkw1UcZGPyrCiHHkmBaj9SiCD8Oqd556HldP+QlpUIe2Wgn3ehQGVoPOvZvtHm8HPx+bH20c9pvbkX3g==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.0.tgz", + "integrity": "sha512-vbutsFqQ+foy3wSSbmjBXXIJ6PL3scghJoM8zCL142cGaZKAdCZHyf+Bpu/MmX9zT9Q0zFBVKb36Ma5Fzfa8xA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.0.tgz", + "integrity": "sha512-hjQ0R/ulkO8fCYFsG0FZoH+pWgTTDreqpqY7UnQntnaKv95uP5iW3+dChxnx7C3trQQU40S+OgWhUVwCjVFLvg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.0.tgz", + "integrity": "sha512-MD9uzzkPQbYehwcN583yx3Tu5M8EIoTD+tUgKF982WYL9Pf5rKy9ltgD0eUgs8pvKnmizxjXZyLt0z6DC3rRXg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.0.tgz", + "integrity": "sha512-4ir0aY1NGUhIC1hdoCzr1+5b43mw99uNwVzhIq1OY3QcEwPDO3B7WNXBzaKY5Nsf1+N11i1eOfFcq+D/gOS15Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.0.tgz", + "integrity": "sha512-jVzdzsbM5xrotH+W5f1s+JtUy1UWgjU0Cf4wMvffTB8m6wP5/kx0KiaLHlbJO+dMgtxKV8RQ/JvtlFcdZ1zCPA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.0.tgz", + "integrity": "sha512-iKc8GAslzRpBytO2/aN3d2yb2z8XTVfNV0PjGlCxKo5SgWmNXx82I/Q3aG1tFfS+A2igVCY97TJ8tnYwpUWLCA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.0.tgz", + "integrity": "sha512-vQW36KZolfIudCcTnaTpmLQ24Ha1RjygBo39/aLkM2kmjkWmZGEJ5Gn9l5/7tzXA42QGIoWbICfg6KLLkIw6yw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.0.tgz", + "integrity": "sha512-7IAFPrjSQIJrGsK6flwg7NFmwBoSTyF3rl7If0hNUFQU4ilTsEPL6GuMuU9BfIWVVGuRnuIidkSMC+c0Otu8IA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@pipecat-ai/client-js": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.2.tgz", + "integrity": "sha512-psunOVrJjPka2SWlq53vxVWCA0Vt8pSXsXtn8pOLC0YTKFsUx+b7Z6quYUJcDZjCe1aAg9cKETek3Xal3Co8Tg==", + "license": "BSD-2-Clause", + "dependencies": { + "@types/events": "^3.0.3", + "clone-deep": "^4.0.1", + "events": "^3.3.0", + "typed-emitter": "^2.1.0", + "uuid": "^10.0.0" + } + }, + "node_modules/@pipecat-ai/daily-transport": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@pipecat-ai/daily-transport/-/daily-transport-0.3.5.tgz", + "integrity": "sha512-nJ0TvWPCqXPmU81U8cXOqk5mUEEvEuI06Mis+N0jN8KZUrNy1pP08iWbs07ObmIXdnQcoL+kQmHOerT4q/bF0w==", + "license": "BSD-2-Clause", + "dependencies": { + "@daily-co/daily-js": "^0.73.0" + }, + "peerDependencies": { + "@pipecat-ai/client-js": "~0.3.2" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.28.0.tgz", + "integrity": "sha512-wLJuPLT6grGZsy34g4N1yRfYeouklTgPhH1gWXCYspenKYD0s3cR99ZevOGw5BexMNywkbV3UkjADisozBmpPQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.28.0.tgz", + "integrity": "sha512-eiNkznlo0dLmVG/6wf+Ifi/v78G4d4QxRhuUl+s8EWZpDewgk7PX3ZyECUXU0Zq/Ca+8nU8cQpNC4Xgn2gFNDA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.28.0.tgz", + "integrity": "sha512-lmKx9yHsppblnLQZOGxdO66gT77bvdBtr/0P+TPOseowE7D9AJoBw8ZDULRasXRWf1Z86/gcOdpBrV6VDUY36Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.28.0.tgz", + "integrity": "sha512-8hxgfReVs7k9Js1uAIhS6zq3I+wKQETInnWQtgzt8JfGx51R1N6DRVy3F4o0lQwumbErRz52YqwjfvuwRxGv1w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.28.0.tgz", + "integrity": "sha512-lA1zZB3bFx5oxu9fYud4+g1mt+lYXCoch0M0V/xhqLoGatbzVse0wlSQ1UYOWKpuSu3gyN4qEc0Dxf/DII1bhQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.28.0.tgz", + "integrity": "sha512-aI2plavbUDjCQB/sRbeUZWX9qp12GfYkYSJOrdYTL/C5D53bsE2/nBPuoiJKoWp5SN78v2Vr8ZPnB+/VbQ2pFA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.28.0.tgz", + "integrity": "sha512-WXveUPKtfqtaNvpf0iOb0M6xC64GzUX/OowbqfiCSXTdi/jLlOmH0Ba94/OkiY2yTGTwteo4/dsHRfh5bDCZ+w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.28.0.tgz", + "integrity": "sha512-yLc3O2NtOQR67lI79zsSc7lk31xjwcaocvdD1twL64PK1yNaIqCeWI9L5B4MFPAVGEVjH5k1oWSGuYX1Wutxpg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.28.0.tgz", + "integrity": "sha512-+P9G9hjEpHucHRXqesY+3X9hD2wh0iNnJXX/QhS/J5vTdG6VhNYMxJ2rJkQOxRUd17u5mbMLHM7yWGZdAASfcg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.28.0.tgz", + "integrity": "sha512-1xsm2rCKSTpKzi5/ypT5wfc+4bOGa/9yI/eaOLW0oMs7qpC542APWhl4A37AENGZ6St6GBMWhCCMM6tXgTIplw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.28.0.tgz", + "integrity": "sha512-zgWxMq8neVQeXL+ouSf6S7DoNeo6EPgi1eeqHXVKQxqPy1B2NvTbaOUWPn/7CfMKL7xvhV0/+fq/Z/J69g1WAQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.28.0.tgz", + "integrity": "sha512-VEdVYacLniRxbRJLNtzwGt5vwS0ycYshofI7cWAfj7Vg5asqj+pt+Q6x4n+AONSZW/kVm+5nklde0qs2EUwU2g==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.28.0.tgz", + "integrity": "sha512-LQlP5t2hcDJh8HV8RELD9/xlYtEzJkm/aWGsauvdO2ulfl3QYRjqrKW+mGAIWP5kdNCBheqqqYIGElSRCaXfpw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.28.0.tgz", + "integrity": "sha512-Nl4KIzteVEKE9BdAvYoTkW19pa7LR/RBrT6F1dJCV/3pbjwDcaOq+edkP0LXuJ9kflW/xOK414X78r+K84+msw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.28.0.tgz", + "integrity": "sha512-eKpJr4vBDOi4goT75MvW+0dXcNUqisK4jvibY9vDdlgLx+yekxSm55StsHbxUsRxSTt3JEQvlr3cGDkzcSP8bw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.28.0.tgz", + "integrity": "sha512-Vi+WR62xWGsE/Oj+mD0FNAPY2MEox3cfyG0zLpotZdehPFXwz6lypkGs5y38Jd/NVSbOD02aVad6q6QYF7i8Bg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.28.0.tgz", + "integrity": "sha512-kN/Vpip8emMLn/eOza+4JwqDZBL6MPNpkdaEsgUtW1NYN3DZvZqSQrbKzJcTL6hd8YNmFTn7XGWMwccOcJBL0A==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.28.0.tgz", + "integrity": "sha512-Bvno2/aZT6usSa7lRDL2+hMjVAGjuqaymF1ApZm31JXzniR/hvr14jpU+/z4X6Gt5BPlzosscyJZGUvguXIqeQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@sentry-internal/browser-utils": { + "version": "8.49.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.49.0.tgz", + "integrity": "sha512-XkPHHdFqsN7EPaB+QGUOEmpFqXiqP67t2rRZ1HG1UwJoe0PhJEKNy7b4+WRwmT7ODSt+PvFk1gNBlJBpThwH7Q==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.49.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/feedback": { + "version": "8.49.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.49.0.tgz", + "integrity": "sha512-v/wf7WvPxEvZUB7xrCnecI3fhevVo84hw8WlxgZIz6mLUHXEIX8xYWc9H8Yet/KKJ2uEB8GQ8aDsY6S1hVEIUA==", + "license": "MIT", + "dependencies": { + "@sentry/core": "8.49.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay": { + "version": "8.49.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.49.0.tgz", + "integrity": "sha512-BDiiCBxskkktTd6FNplBc9V8l14R4T/AwRIZj2itX4xnuHewTTDjVbeyvGol4roA4r+V0Mzoi31hLEGI6yFQ5Q==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.49.0", + "@sentry/core": "8.49.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry-internal/replay-canvas": { + "version": "8.49.0", + "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.49.0.tgz", + "integrity": "sha512-/yXxI7f+Wu24FIYoRE7A0AidNxORuhAyPzb5ey1wFqMXP72nG8dXhOpcl0w+bi554FkqkLjdeUDhSOBWYZXH9g==", + "license": "MIT", + "dependencies": { + "@sentry-internal/replay": "8.49.0", + "@sentry/core": "8.49.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/browser": { + "version": "8.49.0", + "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.49.0.tgz", + "integrity": "sha512-dS4Sw2h8EixHeXOIR++XEVMTen6xCGcIQ/XhJbsjqvddXeIijW0WkxSeTfPkfs0dsqFHSisWmlmo0xhHbXvEsQ==", + "license": "MIT", + "dependencies": { + "@sentry-internal/browser-utils": "8.49.0", + "@sentry-internal/feedback": "8.49.0", + "@sentry-internal/replay": "8.49.0", + "@sentry-internal/replay-canvas": "8.49.0", + "@sentry/core": "8.49.0" + }, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@sentry/core": { + "version": "8.49.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.49.0.tgz", + "integrity": "sha512-/OAm6LdHhh8TvfDAucWfSJV7M03IOHrJm5LVjrrKr4gwQ1HKd4CDbARsBbPwHIzSRAle0IgG3sbJxEvv52JUIw==", + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@swc/core": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.10.14.tgz", + "integrity": "sha512-WSrnE6JRnH20ZYjOOgSS4aOaPv9gxlkI2KRkN24kagbZnPZMnN8bZZyzw1rrLvwgpuRGv17Uz+hflosbR+SP6w==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3", + "@swc/types": "^0.1.17" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/swc" + }, + "optionalDependencies": { + "@swc/core-darwin-arm64": "1.10.14", + "@swc/core-darwin-x64": "1.10.14", + "@swc/core-linux-arm-gnueabihf": "1.10.14", + "@swc/core-linux-arm64-gnu": "1.10.14", + "@swc/core-linux-arm64-musl": "1.10.14", + "@swc/core-linux-x64-gnu": "1.10.14", + "@swc/core-linux-x64-musl": "1.10.14", + "@swc/core-win32-arm64-msvc": "1.10.14", + "@swc/core-win32-ia32-msvc": "1.10.14", + "@swc/core-win32-x64-msvc": "1.10.14" + }, + "peerDependencies": { + "@swc/helpers": "*" + }, + "peerDependenciesMeta": { + "@swc/helpers": { + "optional": true + } + } + }, + "node_modules/@swc/core-darwin-arm64": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.10.14.tgz", + "integrity": "sha512-Dh4VyrhDDb05tdRmqJ/MucOPMTnrB4pRJol18HVyLlqu1HOT5EzonUniNTCdQbUXjgdv5UVJSTE1lYTzrp+myA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-darwin-x64": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.10.14.tgz", + "integrity": "sha512-KpzotL/I0O12RE3tF8NmQErINv0cQe/0mnN/Q50ESFzB5kU6bLgp2HMnnwDTm/XEZZRJCNe0oc9WJ5rKbAJFRQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm-gnueabihf": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.10.14.tgz", + "integrity": "sha512-20yRXZjMJVz1wp1TcscKiGTVXistG+saIaxOmxSNQia1Qun3hSWLL+u6+5kXbfYGr7R2N6kqSwtZbIfJI25r9Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-gnu": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.10.14.tgz", + "integrity": "sha512-Gy7cGrNkiMfPxQyLGxdgXPwyWzNzbHuWycJFcoKBihxZKZIW8hkPBttkGivuLC+0qOgsV2/U+S7tlvAju7FtmQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-musl": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.10.14.tgz", + "integrity": "sha512-+oYVqJvFw62InZ8PIy1rBACJPC2WTe4vbVb9kM1jJj2D7dKLm9acnnYIVIDsM5Wo7Uab8RvPHXVbs19IBurzuw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-gnu": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.10.14.tgz", + "integrity": "sha512-OmEbVEKQFLQVHwo4EJl9osmlulURy46k232Opfpn/1ji0t2KcNCci3POsnfMuoZjLkGJv8vGNJdPQxX+CP+wSA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-musl": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.10.14.tgz", + "integrity": "sha512-OZW+Icm8DMPqHbhdxplkuG8qrNnPk5i7xJOZWYi1y5bTjgGFI4nEzrsmmeHKMdQTaWwsFrm3uK1rlyQ48MmXmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-arm64-msvc": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.10.14.tgz", + "integrity": "sha512-sTvc+xrDQXy3HXZFtTEClY35Efvuc3D+busYm0+rb1+Thau4HLRY9WP+sOKeGwH9/16rzfzYEqD7Ds8A9ykrHw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-ia32-msvc": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.10.14.tgz", + "integrity": "sha512-j2iQ4y9GWTKtES5eMU0sDsFdYni7IxME7ejFej25Tv3Fq4B+U9tgtYWlJwh1858nIWDXelHiKcSh/UICAyVMdQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-x64-msvc": { + "version": "1.10.14", + "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.10.14.tgz", + "integrity": "sha512-TYtWkUSMkjs0jGPeWdtWbex4B+DlQZmN/ySVLiPI+EltYCLEXsFMkVFq6aWn48dqFHggFK0UYfvDrJUR2c3Qxg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@swc/types": { + "version": "0.1.17", + "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.17.tgz", + "integrity": "sha512-V5gRru+aD8YVyCOMAjMpWR1Ui577DD5KSJsHP8RAxopAH22jFz6GZd/qxqjO6MJHQhcsjvjOFXyDhyLQUnMveQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3" + } + }, + "node_modules/@types/estree": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/events": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz", + "integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.13.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz", + "integrity": "sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@vitejs/plugin-react-swc": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-3.7.2.tgz", + "integrity": "sha512-y0byko2b2tSVVf5Gpng1eEhX1OvPC7x8yns1Fx8jDzlJp4LS6CMkCPfLw47cjyoMrshQDoQw4qcgjsU9VvlCew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@swc/core": "^1.7.26" + }, + "peerDependencies": { + "vite": "^4 || ^5 || ^6" + } + }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", + "license": "MIT" + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/esbuild": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.0.tgz", + "integrity": "sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.24.0", + "@esbuild/android-arm": "0.24.0", + "@esbuild/android-arm64": "0.24.0", + "@esbuild/android-x64": "0.24.0", + "@esbuild/darwin-arm64": "0.24.0", + "@esbuild/darwin-x64": "0.24.0", + "@esbuild/freebsd-arm64": "0.24.0", + "@esbuild/freebsd-x64": "0.24.0", + "@esbuild/linux-arm": "0.24.0", + "@esbuild/linux-arm64": "0.24.0", + "@esbuild/linux-ia32": "0.24.0", + "@esbuild/linux-loong64": "0.24.0", + "@esbuild/linux-mips64el": "0.24.0", + "@esbuild/linux-ppc64": "0.24.0", + "@esbuild/linux-riscv64": "0.24.0", + "@esbuild/linux-s390x": "0.24.0", + "@esbuild/linux-x64": "0.24.0", + "@esbuild/netbsd-x64": "0.24.0", + "@esbuild/openbsd-arm64": "0.24.0", + "@esbuild/openbsd-x64": "0.24.0", + "@esbuild/sunos-x64": "0.24.0", + "@esbuild/win32-arm64": "0.24.0", + "@esbuild/win32-ia32": "0.24.0", + "@esbuild/win32-x64": "0.24.0" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.4.49", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", + "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.7", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, + "node_modules/rollup": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.28.0.tgz", + "integrity": "sha512-G9GOrmgWHBma4YfCcX8PjH0qhXSdH8B4HDE2o4/jaxj93S4DPCIDoLcXz99eWMji4hB29UFCEd7B2gwGJDR9cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.6" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.28.0", + "@rollup/rollup-android-arm64": "4.28.0", + "@rollup/rollup-darwin-arm64": "4.28.0", + "@rollup/rollup-darwin-x64": "4.28.0", + "@rollup/rollup-freebsd-arm64": "4.28.0", + "@rollup/rollup-freebsd-x64": "4.28.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.28.0", + "@rollup/rollup-linux-arm-musleabihf": "4.28.0", + "@rollup/rollup-linux-arm64-gnu": "4.28.0", + "@rollup/rollup-linux-arm64-musl": "4.28.0", + "@rollup/rollup-linux-powerpc64le-gnu": "4.28.0", + "@rollup/rollup-linux-riscv64-gnu": "4.28.0", + "@rollup/rollup-linux-s390x-gnu": "4.28.0", + "@rollup/rollup-linux-x64-gnu": "4.28.0", + "@rollup/rollup-linux-x64-musl": "4.28.0", + "@rollup/rollup-win32-arm64-msvc": "4.28.0", + "@rollup/rollup-win32-ia32-msvc": "4.28.0", + "@rollup/rollup-win32-x64-msvc": "4.28.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "optional": true + }, + "node_modules/typed-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "license": "MIT", + "optionalDependencies": { + "rxjs": "*" + } + }, + "node_modules/typescript": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vite": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.0.2.tgz", + "integrity": "sha512-XdQ+VsY2tJpBsKGs0wf3U/+azx8BBpYRHFAyKm5VeEZNOJZRB63q7Sc8Iup3k0TrN3KO6QgyzFf+opSbfY1y0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.24.0", + "postcss": "^8.4.49", + "rollup": "^4.23.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + } + } +} diff --git a/examples/instant-voice/client/javascript/package.json b/examples/instant-voice/client/javascript/package.json new file mode 100644 index 000000000..9c3afb89a --- /dev/null +++ b/examples/instant-voice/client/javascript/package.json @@ -0,0 +1,24 @@ +{ + "name": "client", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "devDependencies": { + "@types/node": "^22.13.1", + "@vitejs/plugin-react-swc": "^3.7.2", + "typescript": "^5.7.3", + "vite": "^6.0.2" + }, + "dependencies": { + "@pipecat-ai/client-js": "^0.3.2", + "@pipecat-ai/daily-transport": "^0.3.5" + } +} diff --git a/examples/instant-voice/client/javascript/src/app.ts b/examples/instant-voice/client/javascript/src/app.ts new file mode 100644 index 000000000..bb587307c --- /dev/null +++ b/examples/instant-voice/client/javascript/src/app.ts @@ -0,0 +1,268 @@ +/** + * Copyright (c) 2024–2025, Daily + * + * SPDX-License-Identifier: BSD 2-Clause License + */ + +/** + * RTVI Client Implementation + * + * This client connects to an RTVI-compatible bot server using WebRTC (via Daily). + * It handles audio/video streaming and manages the connection lifecycle. + * + * Requirements: + * - A running RTVI bot server (defaults to http://localhost:7860) + * - The server must implement the /connect endpoint that returns Daily.co room credentials + * - Browser with WebRTC support + */ + +import { + Participant, + RTVIClient, + RTVIClientOptions, + RTVIEvent, +} from '@pipecat-ai/client-js'; +import { DailyTransport } from '@pipecat-ai/daily-transport'; +import SoundUtils from "./util/soundUtils"; +import { InstantVoiceHelper } from "./util/instantVoiceHelper"; + +/** + * InstantVoiceClient handles the connection and media management for a real-time + * voice and video interaction with an AI bot. + */ +class InstantVoiceClient { + + private declare rtviClient: RTVIClient; + private connectBtn: HTMLButtonElement | null = null; + private disconnectBtn: HTMLButtonElement | null = null; + private statusSpan: HTMLElement | null = null; + private bufferingAudioSpan: HTMLElement | null = null; + private debugLog: HTMLElement | null = null; + private botAudio: HTMLAudioElement; + private declare startTime: number; + + constructor() { + this.botAudio = document.createElement('audio'); + this.botAudio.autoplay = true; + document.body.appendChild(this.botAudio); + this.setupDOMElements(); + this.setupEventListeners(); + this.initializeRTVIClient(); + } + + /** + * Set up references to DOM elements and create necessary media elements + */ + private setupDOMElements(): void { + this.connectBtn = document.getElementById('connect-btn') as HTMLButtonElement; + this.disconnectBtn = document.getElementById('disconnect-btn') as HTMLButtonElement; + this.statusSpan = document.getElementById('connection-status'); + this.bufferingAudioSpan = document.getElementById('buffering-status'); + this.debugLog = document.getElementById('debug-log'); + } + + /** + * Set up event listeners for connect/disconnect buttons + */ + private setupEventListeners(): void { + this.connectBtn?.addEventListener('click', () => this.connect()); + this.disconnectBtn?.addEventListener('click', () => this.disconnect()); + } + + private initializeRTVIClient(): void { + const transport = new DailyTransport({ + bufferLocalAudioUntilBotReady: true + }); + const RTVIConfig: RTVIClientOptions = { + transport, + params: { + // The baseURL and endpoint of your bot server that the client will connect to + baseUrl: 'http://localhost:7860', + endpoints: { connect: '/connect' }, + }, + enableMic: true, + enableCam: false, + callbacks: { + onConnected: () => { + this.updateStatus('Connected'); + if (this.connectBtn) this.connectBtn.disabled = true; + if (this.disconnectBtn) this.disconnectBtn.disabled = false; + }, + onDisconnected: () => { + this.updateStatus('Disconnected'); + this.updateBufferingStatus('No'); + if (this.connectBtn) this.connectBtn.disabled = false; + if (this.disconnectBtn) this.disconnectBtn.disabled = true; + this.log('Client disconnected'); + }, + onBotConnected: (participant: Participant) => { + this.log(`onBotConnected, timeTaken: ${Date.now() - this.startTime}`); + }, + onBotReady: (data) => { + this.log(`onBotReady, timeTaken: ${Date.now() - this.startTime}`); + this.log(`Bot ready: ${JSON.stringify(data)}`); + this.setupMediaTracks(); + }, + onUserTranscript: (data) => { + if (data.final) { + this.log(`User: ${data.text}`); + } + }, + onBotTranscript: (data) => this.log(`Bot: ${data.text}`), + onMessageError: (error) => console.error('Message error:', error), + onError: (error) => console.error('Error:', error), + }, + } + + this.rtviClient = new RTVIClient(RTVIConfig); + this.rtviClient.registerHelper("transport", new InstantVoiceHelper({ + callbacks: { + onAudioBufferingStarted: () => { + SoundUtils.beep() + this.updateBufferingStatus('Yes'); + this.log(`onMicCaptureStarted, timeTaken: ${Date.now() - this.startTime}`); + }, + onAudioBufferingStopped: () => { + this.updateBufferingStatus('No'); + this.log(`onMicCaptureStopped, timeTaken: ${Date.now() - this.startTime}`); + } + } + } + )); + this.setupTrackListeners(); + } + + /** + * Add a timestamped message to the debug log + */ + private log(message: string): void { + if (!this.debugLog) return; + const entry = document.createElement('div'); + entry.textContent = `${new Date().toISOString()} - ${message}`; + if (message.startsWith('User: ')) { + entry.style.color = '#2196F3'; + } else if (message.startsWith('Bot: ')) { + entry.style.color = '#4CAF50'; + } + this.debugLog.appendChild(entry); + this.debugLog.scrollTop = this.debugLog.scrollHeight; + console.log(message); + } + + /** + * Update the connection status display + */ + private updateStatus(status: string): void { + if (this.statusSpan) { + this.statusSpan.textContent = status; + } + this.log(`Status: ${status}`); + } + + /** + * Update the connection status display + */ + private updateBufferingStatus(status: string): void { + if (this.bufferingAudioSpan) { + this.bufferingAudioSpan.textContent = status; + } + this.log(`BufferingStatus: ${status}`); + } + + /** + * Check for available media tracks and set them up if present + * This is called when the bot is ready or when the transport state changes to ready + */ + setupMediaTracks() { + if (!this.rtviClient) return; + const tracks = this.rtviClient.tracks(); + if (tracks.bot?.audio) { + this.setupAudioTrack(tracks.bot.audio); + } + } + + /** + * Set up listeners for track events (start/stop) + * This handles new tracks being added during the session + */ + setupTrackListeners() { + if (!this.rtviClient) return; + + // Listen for new tracks starting + this.rtviClient.on(RTVIEvent.TrackStarted, (track, participant) => { + // Only handle non-local (bot) tracks + if (!participant?.local && track.kind === 'audio') { + this.setupAudioTrack(track); + } + }); + + // Listen for tracks stopping + this.rtviClient.on(RTVIEvent.TrackStopped, (track, participant) => { + this.log(`Track stopped: ${track.kind} from ${participant?.name || 'unknown'}`); + }); + } + + /** + * Set up an audio track for playback + * Handles both initial setup and track updates + */ + private setupAudioTrack(track: MediaStreamTrack): void { + this.log('Setting up audio track'); + if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { + const oldTrack = this.botAudio.srcObject.getAudioTracks()[0]; + if (oldTrack?.id === track.id) return; + } + this.botAudio.srcObject = new MediaStream([track]); + } + + /** + * Initialize and connect to the bot + * This sets up the RTVI client, initializes devices, and establishes the connection + */ + public async connect(): Promise { + try { + this.startTime = Date.now(); + this.log('Connecting to bot...'); + await this.rtviClient.connect(); + } catch (error) { + this.log(`Error connecting: ${(error as Error).message}`); + this.updateStatus('Error'); + this.updateBufferingStatus('No'); + + // Clean up if there's an error + if (this.rtviClient) { + try { + await this.rtviClient.disconnect(); + } catch (disconnectError) { + this.log(`Error during disconnect: ${disconnectError}`); + } + } + } + } + + /** + * Disconnect from the bot and clean up media resources + */ + public async disconnect(): Promise { + try { + await this.rtviClient.disconnect(); + if (this.botAudio.srcObject && "getAudioTracks" in this.botAudio.srcObject) { + this.botAudio.srcObject.getAudioTracks().forEach((track) => track.stop()); + this.botAudio.srcObject = null; + } + } catch (error) { + this.log(`Error disconnecting: ${(error as Error).message}`); + } + } +} + +declare global { + interface Window { + InstantVoiceClient: typeof InstantVoiceClient; + } +} + +window.addEventListener('DOMContentLoaded', () => { + window.InstantVoiceClient = InstantVoiceClient; + new InstantVoiceClient(); +}); diff --git a/examples/instant-voice/client/javascript/src/style.css b/examples/instant-voice/client/javascript/src/style.css new file mode 100644 index 000000000..9c147266e --- /dev/null +++ b/examples/instant-voice/client/javascript/src/style.css @@ -0,0 +1,98 @@ +body { + margin: 0; + padding: 20px; + font-family: Arial, sans-serif; + background-color: #f0f0f0; +} + +.container { + max-width: 1200px; + margin: 0 auto; +} + +.status-bar { + display: flex; + justify-content: space-between; + align-items: center; + padding: 10px; + background-color: #fff; + border-radius: 8px; + margin-bottom: 20px; +} + +.controls button { + padding: 8px 16px; + margin-left: 10px; + border: none; + border-radius: 4px; + cursor: pointer; +} + +#connect-btn { + background-color: #4caf50; + color: white; +} + +#disconnect-btn { + background-color: #f44336; + color: white; +} + +button:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.main-content { + background-color: #fff; + border-radius: 8px; + padding: 20px; + margin-bottom: 20px; +} + +.bot-container { + display: flex; + flex-direction: column; + align-items: center; +} + +#bot-video-container { + width: 640px; + height: 360px; + background-color: #e0e0e0; + border-radius: 8px; + margin: 20px auto; + overflow: hidden; + display: flex; + align-items: center; + justify-content: center; +} + +#bot-video-container video { + width: 100%; + height: 100%; + object-fit: cover; +} + +.debug-panel { + background-color: #fff; + border-radius: 8px; + padding: 20px; +} + +.debug-panel h3 { + margin: 0 0 10px 0; + font-size: 16px; + font-weight: bold; +} + +#debug-log { + height: 500px; + overflow-y: auto; + background-color: #f8f8f8; + padding: 10px; + border-radius: 4px; + font-family: monospace; + font-size: 12px; + line-height: 1.4; +} diff --git a/examples/instant-voice/client/javascript/src/util/instantVoiceHelper.ts b/examples/instant-voice/client/javascript/src/util/instantVoiceHelper.ts new file mode 100644 index 000000000..2ce3a15ce --- /dev/null +++ b/examples/instant-voice/client/javascript/src/util/instantVoiceHelper.ts @@ -0,0 +1,39 @@ +import {RTVIClientHelper, RTVIClientHelperOptions, RTVIMessage} from "@pipecat-ai/client-js"; +import {DailyRTVIMessageType} from '@pipecat-ai/daily-transport'; + +export type InstantVoiceHelperCallbacks = Partial<{ + onAudioBufferingStarted: () => void; + onAudioBufferingStopped: () => void; +}>; + +// --- Interface and class +export interface InstantVoiceHelperOptions extends RTVIClientHelperOptions { + callbacks?: InstantVoiceHelperCallbacks; +} +export class InstantVoiceHelper extends RTVIClientHelper { + + protected declare _options: InstantVoiceHelperOptions; + + constructor(options: InstantVoiceHelperOptions) { + super(options); + } + + handleMessage(rtviMessage: RTVIMessage): void { + switch (rtviMessage.type) { + case DailyRTVIMessageType.AUDIO_BUFFERING_STARTED: + if (this._options.callbacks?.onAudioBufferingStarted) { + this._options.callbacks?.onAudioBufferingStarted() + } + break; + case DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED: + if (this._options.callbacks?.onAudioBufferingStopped) { + this._options.callbacks?.onAudioBufferingStopped() + } + break; + } + } + + getMessageTypes(): string[] { + return [DailyRTVIMessageType.AUDIO_BUFFERING_STARTED, DailyRTVIMessageType.AUDIO_BUFFERING_STOPPED]; + } +} diff --git a/examples/instant-voice/client/javascript/src/util/soundUtils.ts b/examples/instant-voice/client/javascript/src/util/soundUtils.ts new file mode 100644 index 000000000..996185f40 --- /dev/null +++ b/examples/instant-voice/client/javascript/src/util/soundUtils.ts @@ -0,0 +1,8 @@ +class SoundUtils { + static beep() { + const snd = new Audio("data:audio/wav;base64,//uQRAAAAWMSLwUIYAAsYkXgoQwAEaYLWfkWgAI0wWs/ItAAAGDgYtAgAyN+QWaAAihwMWm4G8QQRDiMcCBcH3Cc+CDv/7xA4Tvh9Rz/y8QADBwMWgQAZG/ILNAARQ4GLTcDeIIIhxGOBAuD7hOfBB3/94gcJ3w+o5/5eIAIAAAVwWgQAVQ2ORaIQwEMAJiDg95G4nQL7mQVWI6GwRcfsZAcsKkJvxgxEjzFUgfHoSQ9Qq7KNwqHwuB13MA4a1q/DmBrHgPcmjiGoh//EwC5nGPEmS4RcfkVKOhJf+WOgoxJclFz3kgn//dBA+ya1GhurNn8zb//9NNutNuhz31f////9vt///z+IdAEAAAK4LQIAKobHItEIYCGAExBwe8jcToF9zIKrEdDYIuP2MgOWFSE34wYiR5iqQPj0JIeoVdlG4VD4XA67mAcNa1fhzA1jwHuTRxDUQ//iYBczjHiTJcIuPyKlHQkv/LHQUYkuSi57yQT//uggfZNajQ3Vmz+Zt//+mm3Wm3Q576v////+32///5/EOgAAADVghQAAAAA//uQZAUAB1WI0PZugAAAAAoQwAAAEk3nRd2qAAAAACiDgAAAAAAABCqEEQRLCgwpBGMlJkIz8jKhGvj4k6jzRnqasNKIeoh5gI7BJaC1A1AoNBjJgbyApVS4IDlZgDU5WUAxEKDNmmALHzZp0Fkz1FMTmGFl1FMEyodIavcCAUHDWrKAIA4aa2oCgILEBupZgHvAhEBcZ6joQBxS76AgccrFlczBvKLC0QI2cBoCFvfTDAo7eoOQInqDPBtvrDEZBNYN5xwNwxQRfw8ZQ5wQVLvO8OYU+mHvFLlDh05Mdg7BT6YrRPpCBznMB2r//xKJjyyOh+cImr2/4doscwD6neZjuZR4AgAABYAAAABy1xcdQtxYBYYZdifkUDgzzXaXn98Z0oi9ILU5mBjFANmRwlVJ3/6jYDAmxaiDG3/6xjQQCCKkRb/6kg/wW+kSJ5//rLobkLSiKmqP/0ikJuDaSaSf/6JiLYLEYnW/+kXg1WRVJL/9EmQ1YZIsv/6Qzwy5qk7/+tEU0nkls3/zIUMPKNX/6yZLf+kFgAfgGyLFAUwY//uQZAUABcd5UiNPVXAAAApAAAAAE0VZQKw9ISAAACgAAAAAVQIygIElVrFkBS+Jhi+EAuu+lKAkYUEIsmEAEoMeDmCETMvfSHTGkF5RWH7kz/ESHWPAq/kcCRhqBtMdokPdM7vil7RG98A2sc7zO6ZvTdM7pmOUAZTnJW+NXxqmd41dqJ6mLTXxrPpnV8avaIf5SvL7pndPvPpndJR9Kuu8fePvuiuhorgWjp7Mf/PRjxcFCPDkW31srioCExivv9lcwKEaHsf/7ow2Fl1T/9RkXgEhYElAoCLFtMArxwivDJJ+bR1HTKJdlEoTELCIqgEwVGSQ+hIm0NbK8WXcTEI0UPoa2NbG4y2K00JEWbZavJXkYaqo9CRHS55FcZTjKEk3NKoCYUnSQ0rWxrZbFKbKIhOKPZe1cJKzZSaQrIyULHDZmV5K4xySsDRKWOruanGtjLJXFEmwaIbDLX0hIPBUQPVFVkQkDoUNfSoDgQGKPekoxeGzA4DUvnn4bxzcZrtJyipKfPNy5w+9lnXwgqsiyHNeSVpemw4bWb9psYeq//uQZBoABQt4yMVxYAIAAAkQoAAAHvYpL5m6AAgAACXDAAAAD59jblTirQe9upFsmZbpMudy7Lz1X1DYsxOOSWpfPqNX2WqktK0DMvuGwlbNj44TleLPQ+Gsfb+GOWOKJoIrWb3cIMeeON6lz2umTqMXV8Mj30yWPpjoSa9ujK8SyeJP5y5mOW1D6hvLepeveEAEDo0mgCRClOEgANv3B9a6fikgUSu/DmAMATrGx7nng5p5iimPNZsfQLYB2sDLIkzRKZOHGAaUyDcpFBSLG9MCQALgAIgQs2YunOszLSAyQYPVC2YdGGeHD2dTdJk1pAHGAWDjnkcLKFymS3RQZTInzySoBwMG0QueC3gMsCEYxUqlrcxK6k1LQQcsmyYeQPdC2YfuGPASCBkcVMQQqpVJshui1tkXQJQV0OXGAZMXSOEEBRirXbVRQW7ugq7IM7rPWSZyDlM3IuNEkxzCOJ0ny2ThNkyRai1b6ev//3dzNGzNb//4uAvHT5sURcZCFcuKLhOFs8mLAAEAt4UWAAIABAAAAAB4qbHo0tIjVkUU//uQZAwABfSFz3ZqQAAAAAngwAAAE1HjMp2qAAAAACZDgAAAD5UkTE1UgZEUExqYynN1qZvqIOREEFmBcJQkwdxiFtw0qEOkGYfRDifBui9MQg4QAHAqWtAWHoCxu1Yf4VfWLPIM2mHDFsbQEVGwyqQoQcwnfHeIkNt9YnkiaS1oizycqJrx4KOQjahZxWbcZgztj2c49nKmkId44S71j0c8eV9yDK6uPRzx5X18eDvjvQ6yKo9ZSS6l//8elePK/Lf//IInrOF/FvDoADYAGBMGb7FtErm5MXMlmPAJQVgWta7Zx2go+8xJ0UiCb8LHHdftWyLJE0QIAIsI+UbXu67dZMjmgDGCGl1H+vpF4NSDckSIkk7Vd+sxEhBQMRU8j/12UIRhzSaUdQ+rQU5kGeFxm+hb1oh6pWWmv3uvmReDl0UnvtapVaIzo1jZbf/pD6ElLqSX+rUmOQNpJFa/r+sa4e/pBlAABoAAAAA3CUgShLdGIxsY7AUABPRrgCABdDuQ5GC7DqPQCgbbJUAoRSUj+NIEig0YfyWUho1VBBBA//uQZB4ABZx5zfMakeAAAAmwAAAAF5F3P0w9GtAAACfAAAAAwLhMDmAYWMgVEG1U0FIGCBgXBXAtfMH10000EEEEEECUBYln03TTTdNBDZopopYvrTTdNa325mImNg3TTPV9q3pmY0xoO6bv3r00y+IDGid/9aaaZTGMuj9mpu9Mpio1dXrr5HERTZSmqU36A3CumzN/9Robv/Xx4v9ijkSRSNLQhAWumap82WRSBUqXStV/YcS+XVLnSS+WLDroqArFkMEsAS+eWmrUzrO0oEmE40RlMZ5+ODIkAyKAGUwZ3mVKmcamcJnMW26MRPgUw6j+LkhyHGVGYjSUUKNpuJUQoOIAyDvEyG8S5yfK6dhZc0Tx1KI/gviKL6qvvFs1+bWtaz58uUNnryq6kt5RzOCkPWlVqVX2a/EEBUdU1KrXLf40GoiiFXK///qpoiDXrOgqDR38JB0bw7SoL+ZB9o1RCkQjQ2CBYZKd/+VJxZRRZlqSkKiws0WFxUyCwsKiMy7hUVFhIaCrNQsKkTIsLivwKKigsj8XYlwt/WKi2N4d//uQRCSAAjURNIHpMZBGYiaQPSYyAAABLAAAAAAAACWAAAAApUF/Mg+0aohSIRobBAsMlO//Kk4soosy1JSFRYWaLC4qZBYWFRGZdwqKiwkNBVmoWFSJkWFxX4FFRQWR+LsS4W/rFRb/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////VEFHAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAU291bmRib3kuZGUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMjAwNGh0dHA6Ly93d3cuc291bmRib3kuZGUAAAAAAAAAACU="); + void snd.play(); + } +} + +export default SoundUtils; diff --git a/examples/instant-voice/client/javascript/tsconfig.json b/examples/instant-voice/client/javascript/tsconfig.json new file mode 100644 index 000000000..c9c555d96 --- /dev/null +++ b/examples/instant-voice/client/javascript/tsconfig.json @@ -0,0 +1,111 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "rewriteRelativeImportExtensions": true, /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "noUncheckedSideEffectImports": true, /* Check side effect imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} diff --git a/examples/instant-voice/client/javascript/vite.config.js b/examples/instant-voice/client/javascript/vite.config.js new file mode 100644 index 000000000..daf85167d --- /dev/null +++ b/examples/instant-voice/client/javascript/vite.config.js @@ -0,0 +1,15 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react-swc'; + +export default defineConfig({ + plugins: [react()], + server: { + proxy: { + // Proxy /api requests to the backend server + '/connect': { + target: 'http://0.0.0.0:7860', // Replace with your backend URL + changeOrigin: true, + }, + }, + }, +}); diff --git a/examples/instant-voice/client/javascript/yarn.lock b/examples/instant-voice/client/javascript/yarn.lock new file mode 100644 index 000000000..e74aef8b7 --- /dev/null +++ b/examples/instant-voice/client/javascript/yarn.lock @@ -0,0 +1,339 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/runtime@^7.12.5": + version "7.26.0" + resolved "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz" + integrity sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw== + dependencies: + regenerator-runtime "^0.14.0" + +"@daily-co/daily-js@^0.73.0": + version "0.73.0" + resolved "https://registry.npmjs.org/@daily-co/daily-js/-/daily-js-0.73.0.tgz" + integrity sha512-Wz8c60hgmkx8fcEeDAi4L4J0rbafiihWKyXFyhYoFYPsw2OdChHpA4RYwIB+1enRws5IK+/HdmzFDYLQsB4A6w== + dependencies: + "@babel/runtime" "^7.12.5" + "@sentry/browser" "^8.33.1" + bowser "^2.8.1" + dequal "^2.0.3" + events "^3.1.0" + +"@esbuild/darwin-arm64@0.24.0": + version "0.24.0" + resolved "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.0.tgz" + integrity sha512-CKyDpRbK1hXwv79soeTJNHb5EiG6ct3efd/FTPdzOWdbZZfGhpbcqIpiD0+vwmpu0wTIL97ZRPZu8vUt46nBSw== + +"@pipecat-ai/client-js@^0.3.2", "@pipecat-ai/client-js@~0.3.2": + version "0.3.2" + resolved "https://registry.npmjs.org/@pipecat-ai/client-js/-/client-js-0.3.2.tgz" + integrity sha512-psunOVrJjPka2SWlq53vxVWCA0Vt8pSXsXtn8pOLC0YTKFsUx+b7Z6quYUJcDZjCe1aAg9cKETek3Xal3Co8Tg== + dependencies: + "@types/events" "^3.0.3" + clone-deep "^4.0.1" + events "^3.3.0" + typed-emitter "^2.1.0" + uuid "^10.0.0" + +"@pipecat-ai/daily-transport@^0.3.5": + version "0.3.5" + resolved "https://registry.npmjs.org/@pipecat-ai/daily-transport/-/daily-transport-0.3.5.tgz" + integrity sha512-nJ0TvWPCqXPmU81U8cXOqk5mUEEvEuI06Mis+N0jN8KZUrNy1pP08iWbs07ObmIXdnQcoL+kQmHOerT4q/bF0w== + dependencies: + "@daily-co/daily-js" "^0.73.0" + +"@rollup/rollup-darwin-arm64@4.28.0": + version "4.28.0" + resolved "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.28.0.tgz" + integrity sha512-lmKx9yHsppblnLQZOGxdO66gT77bvdBtr/0P+TPOseowE7D9AJoBw8ZDULRasXRWf1Z86/gcOdpBrV6VDUY36Q== + +"@sentry-internal/browser-utils@8.49.0": + version "8.49.0" + resolved "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.49.0.tgz" + integrity sha512-XkPHHdFqsN7EPaB+QGUOEmpFqXiqP67t2rRZ1HG1UwJoe0PhJEKNy7b4+WRwmT7ODSt+PvFk1gNBlJBpThwH7Q== + dependencies: + "@sentry/core" "8.49.0" + +"@sentry-internal/feedback@8.49.0": + version "8.49.0" + resolved "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.49.0.tgz" + integrity sha512-v/wf7WvPxEvZUB7xrCnecI3fhevVo84hw8WlxgZIz6mLUHXEIX8xYWc9H8Yet/KKJ2uEB8GQ8aDsY6S1hVEIUA== + dependencies: + "@sentry/core" "8.49.0" + +"@sentry-internal/replay-canvas@8.49.0": + version "8.49.0" + resolved "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.49.0.tgz" + integrity sha512-/yXxI7f+Wu24FIYoRE7A0AidNxORuhAyPzb5ey1wFqMXP72nG8dXhOpcl0w+bi554FkqkLjdeUDhSOBWYZXH9g== + dependencies: + "@sentry-internal/replay" "8.49.0" + "@sentry/core" "8.49.0" + +"@sentry-internal/replay@8.49.0": + version "8.49.0" + resolved "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.49.0.tgz" + integrity sha512-BDiiCBxskkktTd6FNplBc9V8l14R4T/AwRIZj2itX4xnuHewTTDjVbeyvGol4roA4r+V0Mzoi31hLEGI6yFQ5Q== + dependencies: + "@sentry-internal/browser-utils" "8.49.0" + "@sentry/core" "8.49.0" + +"@sentry/browser@^8.33.1": + version "8.49.0" + resolved "https://registry.npmjs.org/@sentry/browser/-/browser-8.49.0.tgz" + integrity sha512-dS4Sw2h8EixHeXOIR++XEVMTen6xCGcIQ/XhJbsjqvddXeIijW0WkxSeTfPkfs0dsqFHSisWmlmo0xhHbXvEsQ== + dependencies: + "@sentry-internal/browser-utils" "8.49.0" + "@sentry-internal/feedback" "8.49.0" + "@sentry-internal/replay" "8.49.0" + "@sentry-internal/replay-canvas" "8.49.0" + "@sentry/core" "8.49.0" + +"@sentry/core@8.49.0": + version "8.49.0" + resolved "https://registry.npmjs.org/@sentry/core/-/core-8.49.0.tgz" + integrity sha512-/OAm6LdHhh8TvfDAucWfSJV7M03IOHrJm5LVjrrKr4gwQ1HKd4CDbARsBbPwHIzSRAle0IgG3sbJxEvv52JUIw== + +"@swc/core-darwin-arm64@1.10.14": + version "1.10.14" + resolved "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.10.14.tgz" + integrity sha512-Dh4VyrhDDb05tdRmqJ/MucOPMTnrB4pRJol18HVyLlqu1HOT5EzonUniNTCdQbUXjgdv5UVJSTE1lYTzrp+myA== + +"@swc/core@^1.7.26": + version "1.10.14" + resolved "https://registry.npmjs.org/@swc/core/-/core-1.10.14.tgz" + integrity sha512-WSrnE6JRnH20ZYjOOgSS4aOaPv9gxlkI2KRkN24kagbZnPZMnN8bZZyzw1rrLvwgpuRGv17Uz+hflosbR+SP6w== + dependencies: + "@swc/counter" "^0.1.3" + "@swc/types" "^0.1.17" + optionalDependencies: + "@swc/core-darwin-arm64" "1.10.14" + "@swc/core-darwin-x64" "1.10.14" + "@swc/core-linux-arm-gnueabihf" "1.10.14" + "@swc/core-linux-arm64-gnu" "1.10.14" + "@swc/core-linux-arm64-musl" "1.10.14" + "@swc/core-linux-x64-gnu" "1.10.14" + "@swc/core-linux-x64-musl" "1.10.14" + "@swc/core-win32-arm64-msvc" "1.10.14" + "@swc/core-win32-ia32-msvc" "1.10.14" + "@swc/core-win32-x64-msvc" "1.10.14" + +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/types@^0.1.17": + version "0.1.17" + resolved "https://registry.npmjs.org/@swc/types/-/types-0.1.17.tgz" + integrity sha512-V5gRru+aD8YVyCOMAjMpWR1Ui577DD5KSJsHP8RAxopAH22jFz6GZd/qxqjO6MJHQhcsjvjOFXyDhyLQUnMveQ== + dependencies: + "@swc/counter" "^0.1.3" + +"@types/estree@1.0.6": + version "1.0.6" + resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz" + integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw== + +"@types/events@^3.0.3": + version "3.0.3" + resolved "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz" + integrity sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g== + +"@types/node@^18.0.0 || ^20.0.0 || >=22.0.0", "@types/node@^22.13.1": + version "22.13.1" + resolved "https://registry.npmjs.org/@types/node/-/node-22.13.1.tgz" + integrity sha512-jK8uzQlrvXqEU91UxiK5J7pKHyzgnI1Qnl0QDHIgVGuolJhRb9EEl28Cj9b3rGR8B2lhFCtvIm5os8lFnO/1Ew== + dependencies: + undici-types "~6.20.0" + +"@vitejs/plugin-react-swc@^3.7.2": + version "3.7.2" + resolved "https://registry.npmjs.org/@vitejs/plugin-react-swc/-/plugin-react-swc-3.7.2.tgz" + integrity sha512-y0byko2b2tSVVf5Gpng1eEhX1OvPC7x8yns1Fx8jDzlJp4LS6CMkCPfLw47cjyoMrshQDoQw4qcgjsU9VvlCew== + dependencies: + "@swc/core" "^1.7.26" + +bowser@^2.8.1: + version "2.11.0" + resolved "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz" + integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA== + +clone-deep@^4.0.1: + version "4.0.1" + resolved "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz" + integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ== + dependencies: + is-plain-object "^2.0.4" + kind-of "^6.0.2" + shallow-clone "^3.0.0" + +dequal@^2.0.3: + version "2.0.3" + resolved "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz" + integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA== + +esbuild@^0.24.0: + version "0.24.0" + resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.24.0.tgz" + integrity sha512-FuLPevChGDshgSicjisSooU0cemp/sGXR841D5LHMB7mTVOmsEHcAxaH3irL53+8YDIeVNQEySh4DaYU/iuPqQ== + optionalDependencies: + "@esbuild/aix-ppc64" "0.24.0" + "@esbuild/android-arm" "0.24.0" + "@esbuild/android-arm64" "0.24.0" + "@esbuild/android-x64" "0.24.0" + "@esbuild/darwin-arm64" "0.24.0" + "@esbuild/darwin-x64" "0.24.0" + "@esbuild/freebsd-arm64" "0.24.0" + "@esbuild/freebsd-x64" "0.24.0" + "@esbuild/linux-arm" "0.24.0" + "@esbuild/linux-arm64" "0.24.0" + "@esbuild/linux-ia32" "0.24.0" + "@esbuild/linux-loong64" "0.24.0" + "@esbuild/linux-mips64el" "0.24.0" + "@esbuild/linux-ppc64" "0.24.0" + "@esbuild/linux-riscv64" "0.24.0" + "@esbuild/linux-s390x" "0.24.0" + "@esbuild/linux-x64" "0.24.0" + "@esbuild/netbsd-x64" "0.24.0" + "@esbuild/openbsd-arm64" "0.24.0" + "@esbuild/openbsd-x64" "0.24.0" + "@esbuild/sunos-x64" "0.24.0" + "@esbuild/win32-arm64" "0.24.0" + "@esbuild/win32-ia32" "0.24.0" + "@esbuild/win32-x64" "0.24.0" + +events@^3.1.0, events@^3.3.0: + version "3.3.0" + resolved "https://registry.npmjs.org/events/-/events-3.3.0.tgz" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== + +fsevents@~2.3.2, fsevents@~2.3.3: + version "2.3.3" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + +is-plain-object@^2.0.4: + version "2.0.4" + resolved "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz" + integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== + dependencies: + isobject "^3.0.1" + +isobject@^3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz" + integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== + +kind-of@^6.0.2: + version "6.0.3" + resolved "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz" + integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== + +nanoid@^3.3.7: + version "3.3.8" + resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== + +picocolors@^1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz" + integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== + +postcss@^8.4.49: + version "8.4.49" + resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz" + integrity sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA== + dependencies: + nanoid "^3.3.7" + picocolors "^1.1.1" + source-map-js "^1.2.1" + +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + +rollup@^4.23.0: + version "4.28.0" + resolved "https://registry.npmjs.org/rollup/-/rollup-4.28.0.tgz" + integrity sha512-G9GOrmgWHBma4YfCcX8PjH0qhXSdH8B4HDE2o4/jaxj93S4DPCIDoLcXz99eWMji4hB29UFCEd7B2gwGJDR9cQ== + dependencies: + "@types/estree" "1.0.6" + optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.28.0" + "@rollup/rollup-android-arm64" "4.28.0" + "@rollup/rollup-darwin-arm64" "4.28.0" + "@rollup/rollup-darwin-x64" "4.28.0" + "@rollup/rollup-freebsd-arm64" "4.28.0" + "@rollup/rollup-freebsd-x64" "4.28.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.28.0" + "@rollup/rollup-linux-arm-musleabihf" "4.28.0" + "@rollup/rollup-linux-arm64-gnu" "4.28.0" + "@rollup/rollup-linux-arm64-musl" "4.28.0" + "@rollup/rollup-linux-powerpc64le-gnu" "4.28.0" + "@rollup/rollup-linux-riscv64-gnu" "4.28.0" + "@rollup/rollup-linux-s390x-gnu" "4.28.0" + "@rollup/rollup-linux-x64-gnu" "4.28.0" + "@rollup/rollup-linux-x64-musl" "4.28.0" + "@rollup/rollup-win32-arm64-msvc" "4.28.0" + "@rollup/rollup-win32-ia32-msvc" "4.28.0" + "@rollup/rollup-win32-x64-msvc" "4.28.0" + fsevents "~2.3.2" + +rxjs@*: + version "7.8.1" + resolved "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz" + integrity sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg== + dependencies: + tslib "^2.1.0" + +shallow-clone@^3.0.0: + version "3.0.1" + resolved "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz" + integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA== + dependencies: + kind-of "^6.0.2" + +source-map-js@^1.2.1: + version "1.2.1" + resolved "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz" + integrity sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA== + +tslib@^2.1.0: + version "2.8.1" + resolved "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz" + integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== + +typed-emitter@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz" + integrity sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA== + optionalDependencies: + rxjs "*" + +typescript@^5.7.3: + version "5.7.3" + resolved "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz" + integrity sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw== + +undici-types@~6.20.0: + version "6.20.0" + resolved "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz" + integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg== + +uuid@^10.0.0: + version "10.0.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz" + integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== + +"vite@^4 || ^5 || ^6", vite@^6.0.2: + version "6.0.2" + resolved "https://registry.npmjs.org/vite/-/vite-6.0.2.tgz" + integrity sha512-XdQ+VsY2tJpBsKGs0wf3U/+azx8BBpYRHFAyKm5VeEZNOJZRB63q7Sc8Iup3k0TrN3KO6QgyzFf+opSbfY1y0g== + dependencies: + esbuild "^0.24.0" + postcss "^8.4.49" + rollup "^4.23.0" + optionalDependencies: + fsevents "~2.3.3" diff --git a/examples/instant-voice/server/env.example b/examples/instant-voice/server/env.example new file mode 100644 index 000000000..7dd76e98c --- /dev/null +++ b/examples/instant-voice/server/env.example @@ -0,0 +1,3 @@ +DAILY_API_KEY= +GOOGLE_API_KEY= +DAILY_SAMPLE_ROOM_URL=https://yourdomain.daily.co/yourroom # (for joining the bot to the same room repeatedly for local dev) \ No newline at end of file diff --git a/examples/instant-voice/server/pyproject.toml b/examples/instant-voice/server/pyproject.toml new file mode 100644 index 000000000..2c2ce8e48 --- /dev/null +++ b/examples/instant-voice/server/pyproject.toml @@ -0,0 +1,11 @@ +[tool.ruff] +exclude = [".git", "*_pb2.py"] +line-length = 100 + +[tool.ruff.lint] +select = [ + "I", # Import rules +] + +[tool.ruff.lint.pydocstyle] +convention = "google" \ No newline at end of file diff --git a/examples/instant-voice/server/requirements.txt b/examples/instant-voice/server/requirements.txt new file mode 100644 index 000000000..292de88c2 --- /dev/null +++ b/examples/instant-voice/server/requirements.txt @@ -0,0 +1,4 @@ +python-dotenv +fastapi[all] +uvicorn +pipecat-ai[openai,silero,websocket,google,daily] diff --git a/examples/instant-voice/server/src/server.py b/examples/instant-voice/server/src/server.py new file mode 100644 index 000000000..0777a3730 --- /dev/null +++ b/examples/instant-voice/server/src/server.py @@ -0,0 +1,200 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +import asyncio +import os +from contextlib import asynccontextmanager +from typing import Any, Dict, List + +import aiohttp +import uvicorn +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware +from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams + +# Load environment variables +load_dotenv(override=True) + +NUMBER_OF_ROOMS = 1 + + +class RoomPool: + """Manages a pool of pre-created rooms for quick allocation.""" + + def __init__(self, daily_rest_helper: DailyRESTHelper): + self.daily_rest_helper = daily_rest_helper + self.pool: List[Dict[str, str]] = [] + self.lock = asyncio.Lock() + + async def fill_pool(self, count: int): + """Fills the pool with `count` new rooms.""" + for _ in range(count): + await self.add_room() + + async def add_room(self): + """Creates a new room and adds it to the pool.""" + try: + room = await self.daily_rest_helper.create_room(DailyRoomParams()) + if not room.url: + raise HTTPException(status_code=500, detail="Failed to create room") + + user_token = await self.daily_rest_helper.get_token(room.url) + if not user_token: + raise HTTPException(status_code=500, detail="Failed to get user token") + + bot_token = await self.daily_rest_helper.get_token(room.url) + if not bot_token: + raise HTTPException(status_code=500, detail="Failed to get bot token") + + async with self.lock: + self.pool.append( + {"room_url": room.url, "user_token": user_token, "bot_token": bot_token} + ) + + except Exception as e: + print(f"Error adding room to pool: {e}") + + async def get_room(self) -> Dict[str, str]: + """Retrieves a room from the pool and requests a new one to maintain the size.""" + async with self.lock: + if not self.pool: + raise HTTPException(status_code=503, detail="No available rooms") + + room = self.pool.pop(0) # Get first available room + + # Start a background task to replenish the pool + asyncio.create_task(self.add_room()) + + return room + + async def delete_room(self, room_url: str): + """Deletes a room when it is not needed anymore""" + await self.daily_rest_helper.delete_room_by_url(room_url) + + async def cleanup(self): + for rooms in self.pool: + room_url = rooms["room_url"] + await self.delete_room(room_url) + + +class BotManager: + """Manages bot subprocesses asynchronously.""" + + def __init__(self): + self.bot_procs: Dict[int, asyncio.subprocess.Process] = {} + self.room_mappings: Dict[int, str] = {} # Maps process ID to room URL + + async def start_bot(self, room_url: str, token: str) -> int: + bot_file = "single_bot" + command = f"python3 -m {bot_file} -u {room_url} -t {token}" + + try: + proc = await asyncio.create_subprocess_shell( + command, + cwd=os.path.dirname(os.path.abspath(__file__)), + ) + if proc.pid is None: + raise HTTPException(status_code=500, detail="Failed to get subprocess PID") + + self.bot_procs[proc.pid] = proc + self.room_mappings[proc.pid] = room_url + # Monitor the process and delete the room when it exits + asyncio.create_task(self._monitor_process(proc.pid)) + + return proc.pid + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}") + + async def _monitor_process(self, pid: int): + """Monitors a bot process and deletes the associated room when it exits.""" + proc = self.bot_procs.get(pid) + if proc: + await proc.wait() # Wait for the process to exit + room_url = self.room_mappings.pop(pid, None) + + if room_url: + await room_pool.delete_room(room_url) + print(f"Deleted room: {room_url}") + + del self.bot_procs[pid] + + async def cleanup(self): + """Terminates all running bot processes and deletes associated rooms.""" + for pid, proc in list(self.bot_procs.items()): + try: + proc.terminate() + await asyncio.wait_for(proc.wait(), timeout=5) + + room_url = self.room_mappings.pop(pid, None) + if room_url: + await room_pool.delete_room(room_url) # Delete room when process terminates + print(f"Deleted room: {room_url}") + + except asyncio.TimeoutError: + print(f"Process {pid} did not terminate in time.") + except Exception as e: + print(f"Error terminating process {pid}: {e}") + + # Clear remaining mappings + self.bot_procs.clear() + self.room_mappings.clear() + + +# Global instances +bot_manager = BotManager() +room_pool: RoomPool # Will be initialized in lifespan + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Handles FastAPI startup and shutdown.""" + global room_pool + aiohttp_session = aiohttp.ClientSession() + daily_rest_helper = DailyRESTHelper( + daily_api_key=os.getenv("DAILY_API_KEY", ""), + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=aiohttp_session, + ) + + room_pool = RoomPool(daily_rest_helper) + await room_pool.fill_pool(NUMBER_OF_ROOMS) # Fill pool on startup + + yield # Run app + + await bot_manager.cleanup() + await room_pool.cleanup() + await aiohttp_session.close() + + +# Initialize FastAPI app with lifespan manager +app = FastAPI(lifespan=lifespan) + +# Configure CORS to allow requests from any origin +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.post("/connect") +async def bot_connect(request: Request) -> Dict[Any, Any]: + try: + room = await room_pool.get_room() + await bot_manager.start_bot(room["room_url"], room["bot_token"]) + except HTTPException as e: + return {"error": str(e)} + + return { + "room_url": room["room_url"], + "token": room["user_token"], + } + + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=7860) diff --git a/examples/instant-voice/server/src/single_bot.py b/examples/instant-voice/server/src/single_bot.py new file mode 100644 index 000000000..e1e9df68a --- /dev/null +++ b/examples/instant-voice/server/src/single_bot.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +import argparse +import asyncio +import os +import sys + +from dotenv import load_dotenv +from loguru import logger +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIProcessor +from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +SYSTEM_INSTRUCTION = f""" +"You are Gemini Chatbot, a friendly, helpful robot. + +Your goal is to demonstrate your capabilities in a succinct way. + +Your output will be converted to audio so don't include special characters in your answers. + +Respond to what the user said in a creative and helpful way. Keep your responses brief. One or two sentences at most. +""" + + +def extract_arguments(): + parser = argparse.ArgumentParser(description="Instant Voice Example") + parser.add_argument( + "-u", "--url", type=str, required=True, help="URL of the Daily room to join" + ) + parser.add_argument( + "-t", "--token", type=str, required=False, help="Token of the Daily room to join" + ) + args, unknown = parser.parse_known_args() + url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL") + token = args.token + return url, token + + +async def main(): + room_url, token = extract_arguments() + print(f"room_url: {room_url}") + + daily_transport = DailyTransport( + room_url, + token, + "Instant voice Chatbot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Puck", # Aoede, Charon, Fenrir, Kore, Puck + transcribe_user_audio=True, + transcribe_model_audio=True, + system_instruction=SYSTEM_INSTRUCTION, + ) + + context = OpenAILLMContext() + context_aggregator = llm.create_context_aggregator(context) + + # RTVI events for Pipecat client UI + rtvi = RTVIProcessor(config=RTVIConfig(config=[]), transport=daily_transport) + + pipeline = Pipeline( + [ + daily_transport.input(), + context_aggregator.user(), + rtvi, + llm, # LLM + daily_transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + observers=[rtvi.observer()], + ), + ) + + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + await rtvi.set_bot_ready() + + @daily_transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @daily_transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + print(f"Participant left: {participant}") + await task.cancel() + + runner = PipelineRunner(handle_sigint=False) + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From e006dcf1729d8df79787ed07623719afa874bfac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 09:54:03 -0800 Subject: [PATCH 142/427] WebsocketService: handle clean server disconnection The websocket async iterator doesn't raise an exception when the server disconnects cleanly. We should handle that and raise an exception so we can reconnect. --- CHANGELOG.md | 4 ++++ src/pipecat/services/elevenlabs.py | 7 ++++++- src/pipecat/services/websocket_service.py | 11 +++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca7d08e3e..90c80da6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a websocket-based service issue (e.g. `CartesiaTTSService`) that was + preventing a reconnection after the server disconnected cleanly, which was + causing an inifite loop instead. + - Fixed a `BaseOutputTransport` issue that was causing upstream frames to no be pushed upstream. diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index e0b985411..b2047d34e 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -370,8 +370,13 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): except Exception as e: logger.error(f"{self} error closing websocket: {e}") + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + async def _receive_messages(self): - async for message in self._websocket: + async for message in self._get_websocket(): msg = json.loads(message) if msg.get("audio"): await self.stop_ttfb_metrics() diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 9c942b019..c47d6d97e 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -10,6 +10,7 @@ from typing import Awaitable, Callable, Optional import websockets from loguru import logger +from websockets.protocol import State from pipecat.frames.frames import ErrorFrame @@ -83,11 +84,13 @@ class WebsocketService(ABC): while True: try: await self._receive_messages() - logger.debug(f"{self} connection established successfully") retry_count = 0 # Reset counter on successful message receive - except websockets.ConnectionClosed as e: - logger.warning(f"{self} connection closed: {e}") - break + if self._websocket and self._websocket.state == State.CLOSED: + raise websockets.ConnectionClosedOK( + self._websocket.close_rcvd, + self._websocket.close_sent, + self._websocket.close_rcvd_then_sent, + ) except Exception as e: retry_count += 1 if retry_count >= MAX_RETRIES: From 0b91d821fb4542dd27cd9b513066372d7fb6a0b3 Mon Sep 17 00:00:00 2001 From: M1ngXU <91617361+M1ngXU@users.noreply.github.com> Date: Fri, 14 Feb 2025 20:27:08 +0100 Subject: [PATCH 143/427] Update openai.py d --- src/pipecat/services/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index b0a66eba0..3114b6920 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -110,7 +110,7 @@ class BaseOpenAILLMService(LLMService): seed: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=0) temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=2.0) # Note: top_k is currently not supported by the OpenAI client library, - # so top_k is ignore right now. + # so top_k is ignored right now. top_k: Optional[int] = Field(default=None, ge=0) top_p: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) max_tokens: Optional[int] = Field(default_factory=lambda: NOT_GIVEN, ge=1) From cacb07f4c209869314213885acbb39d48d051f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 11:18:24 -0800 Subject: [PATCH 144/427] introduce AudioContextWordTTSService --- CHANGELOG.md | 3 + src/pipecat/services/ai_services.py | 113 +++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90c80da6e..85848f22b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `AudioContextWordTTSService`. This is a TTS base class for TTS + services that handling multiple separate audio requests. + - Added new frames `EmulateUserStartedSpeakingFrame` and `EmulateUserStoppedSpeakingFrame` which can be used to emulated VAD behavior without VAD being present or not being triggered. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index ac1c4582d..0fce2482a 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -419,7 +419,7 @@ class WordTTSService(TTSService): async def start(self, frame: StartFrame): await super().start(frame) - await self._create_words_task() + self._create_words_task() async def stop(self, frame: EndFrame): await super().stop(frame) @@ -439,7 +439,7 @@ class WordTTSService(TTSService): await super()._handle_interruption(frame, direction) self.reset_word_timestamps() - async def _create_words_task(self): + def _create_words_task(self): self._words_task = self.create_task(self._words_task_handler()) async def _stop_words_task(self): @@ -469,6 +469,115 @@ class WordTTSService(TTSService): self._words_queue.task_done() +class AudioContextWordTTSService(WordTTSService): + """This services allow us to send multiple TTS request to the services. Each + request could be multiple sentences long which are grouped by context. For + this to work, the TTS service needs to support handling multiple requests at + once (i.e. multiple simultaneous contexts). + + The audio received from the TTS will be played in context order. That is, if + we requested audio for a context "A" and then audio for context "B", the + audio from context ID "A" will be played first. + + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._contexts_queue = asyncio.Queue() + self._contexts: Dict[str, asyncio.Queue] = {} + self._audio_context_task = None + + async def create_audio_context(self, context_id: str): + """Create a new audio context.""" + await self._contexts_queue.put(context_id) + self._contexts[context_id] = asyncio.Queue() + logger.trace(f"{self} created audio context {context_id}") + + async def append_to_audio_context(self, context_id: str, frame: TTSAudioRawFrame): + """Append audio to an existing context.""" + if self.audio_context_available(context_id): + logger.trace(f"{self} appending audio {frame} to audio context {context_id}") + await self._contexts[context_id].put(frame) + else: + logger.warning(f"{self} unable to append audio to context {context_id}") + + async def remove_audio_context(self, context_id: str): + """Remove an existing audio context.""" + if self.audio_context_available(context_id): + # We just mark the audio context for deletion by appending + # None. Once we reach None while handling audio we know we can + # safely remove the context. + logger.trace(f"{self} marking audio context {context_id} for deletion") + await self._contexts[context_id].put(None) + else: + logger.warning(f"{self} unable to remove context {context_id}") + + def audio_context_available(self, context_id: str) -> bool: + """Checks whether the given audio context is registered.""" + return context_id in self._contexts + + async def start(self, frame: StartFrame): + await super().start(frame) + self._create_audio_context_task() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._stop_audio_context_task() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._stop_audio_context_task() + + async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + await super()._handle_interruption(frame, direction) + await self._stop_audio_context_task() + self._create_audio_context_task() + + def _create_audio_context_task(self): + self._contexts_queue = asyncio.Queue() + self._contexts: Dict[str, asyncio.Queue] = {} + self._audio_context_task = self.create_task(self._audio_context_task_handler()) + + async def _stop_audio_context_task(self): + if self._audio_context_task: + await self.cancel_task(self._audio_context_task) + self._audio_context_task = None + + async def _audio_context_task_handler(self): + """In this task we process audio contexts in order.""" + while True: + context_id = await self._contexts_queue.get() + + # Process the audio context until the context doesn't have more + # audio available (i.e. we find None). + await self._handle_audio_context(context_id) + + # We just finished processing the context, so we can safely remove it. + del self._contexts[context_id] + self._contexts_queue.task_done() + + # Append some silence between sentences. + silence = b"\x00" * self.sample_rate + frame = TTSAudioRawFrame(audio=silence, sample_rate=self.sample_rate, num_channels=1) + await self.push_frame(frame) + + async def _handle_audio_context(self, context_id: str): + # If we don't receive any audio during this time, we consider the context finished. + AUDIO_CONTEXT_TIMEOUT = 3.0 + queue = self._contexts[context_id] + running = True + while running: + try: + frame = await asyncio.wait_for(queue.get(), timeout=AUDIO_CONTEXT_TIMEOUT) + if frame: + await self.push_frame(frame) + running = frame is not None + except asyncio.TimeoutError: + # We didn't get audio, so let's consider this context finished. + logger.trace(f"{self} time out on audio context {context_id}") + break + + class STTService(AIService): """STTService is a base class for speech-to-text services.""" From aeadb40c3f10fb698d5ce2b84f8d1c3ec61e66cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 11:18:35 -0800 Subject: [PATCH 145/427] CartesiaTTSService: use AudioContextWordTTSService By supporting multiple audio requests we fix an issue that was causing audio overlapping. --- CHANGELOG.md | 3 +++ src/pipecat/services/cartesia.py | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85848f22b..1183c1483 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `CartesiaTTSService` service issue that would cause audio overlapping + in some cases. + - Fixed a websocket-based service issue (e.g. `CartesiaTTSService`) that was preventing a reconnection after the server disconnected cleanly, which was causing an inifite loop instead. diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index d5ea46a30..49bcac2f1 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -27,7 +27,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService, WordTTSService +from pipecat.services.ai_services import AudioContextWordTTSService, TTSService from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language @@ -75,7 +75,7 @@ def language_to_cartesia_language(language: Language) -> Optional[str]: return result -class CartesiaTTSService(WordTTSService, WebsocketService): +class CartesiaTTSService(AudioContextWordTTSService, WebsocketService): class InputParams(BaseModel): language: Optional[Language] = Language.EN speed: Optional[Union[str, float]] = "" @@ -105,7 +105,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService): # if we're interrupted. Cartesia gives us word-by-word timestamps. We # can use those to generate text frames ourselves aligned with the # playout timing of the audio! - WordTTSService.__init__( + AudioContextWordTTSService.__init__( self, aggregate_sentences=True, push_text_frames=False, @@ -191,12 +191,12 @@ class CartesiaTTSService(WordTTSService, WebsocketService): self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): - await self._disconnect_websocket() - if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None + await self._disconnect_websocket() + async def _connect_websocket(self): try: logger.debug("Connecting to Cartesia") @@ -239,21 +239,19 @@ class CartesiaTTSService(WordTTSService, WebsocketService): logger.trace(f"{self}: flushing audio") msg = self._build_msg(text="", continue_transcript=False) await self._websocket.send(msg) + self._context_id = None async def _receive_messages(self): async for message in self._get_websocket(): msg = json.loads(message) - if not msg or msg["context_id"] != self._context_id: + if not msg or not self.audio_context_available(msg["context_id"]): continue if msg["type"] == "done": await self.stop_ttfb_metrics() - # Unset _context_id but not the _context_id_start_timestamp - # because we are likely still playing out audio and need the - # timestamp to set send context frames. - self._context_id = None await self.add_word_timestamps( [("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)] ) + await self.remove_audio_context(msg["context_id"]) elif msg["type"] == "timestamps": await self.add_word_timestamps( list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"])) @@ -266,12 +264,13 @@ class CartesiaTTSService(WordTTSService, WebsocketService): sample_rate=self.sample_rate, num_channels=1, ) - await self.push_frame(frame) + await self.append_to_audio_context(msg["context_id"], frame) elif msg["type"] == "error": logger.error(f"{self} error: {msg}") await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() await self.push_error(ErrorFrame(f"{self} error: {msg['error']}")) + self._context_id = None else: logger.error(f"{self} error, unknown message type: {msg}") @@ -299,6 +298,7 @@ class CartesiaTTSService(WordTTSService, WebsocketService): await self.start_ttfb_metrics() yield TTSStartedFrame() self._context_id = str(uuid.uuid4()) + await self.create_audio_context(self._context_id) msg = self._build_msg(text=text or " ") # Text must contain at least one character From f53ee79ddb0ed1bc59c2bf8e746dcba02f02c080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 11:39:05 -0800 Subject: [PATCH 146/427] RimeTTSService: use AudioContextWordTTSService --- src/pipecat/services/rime.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 0210f8de6..1634578cd 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, Optional, Union +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -28,7 +28,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService, WordTTSService +from pipecat.services.ai_services import AudioContextWordTTSService, TTSService from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language @@ -58,7 +58,7 @@ def language_to_rime_language(language: Language) -> str: return LANGUAGE_MAP.get(language, "eng") -class RimeTTSService(WordTTSService, WebsocketService): +class RimeTTSService(AudioContextWordTTSService, WebsocketService): """Text-to-Speech service using Rime's websocket API. Uses Rime's websocket JSON API to convert text to speech with word-level timing @@ -95,7 +95,7 @@ class RimeTTSService(WordTTSService, WebsocketService): params: Additional configuration parameters. """ # Initialize with parent class settings for proper frame handling - WordTTSService.__init__( + AudioContextWordTTSService.__init__( self, aggregate_sentences=True, push_text_frames=False, @@ -249,12 +249,18 @@ class RimeTTSService(WordTTSService, WebsocketService): return word_pairs + async def flush_audio(self): + if not self._context_id or not self._websocket: + return + logger.trace(f"{self}: flushing audio") + self._context_id = None + async def _receive_messages(self): """Process incoming websocket messages.""" async for message in self._get_websocket(): msg = json.loads(message) - if not msg or msg["contextId"] != self._context_id: + if not msg or not self.audio_context_available(msg["contextId"]): continue if msg["type"] == "chunk": @@ -266,7 +272,7 @@ class RimeTTSService(WordTTSService, WebsocketService): sample_rate=self.sample_rate, num_channels=1, ) - await self.push_frame(frame) + await self.append_to_audio_context(msg["contextId"], frame) elif msg["type"] == "timestamps": # Process word timing information @@ -288,6 +294,7 @@ class RimeTTSService(WordTTSService, WebsocketService): await self.push_frame(TTSStoppedFrame()) await self.stop_all_metrics() await self.push_error(ErrorFrame(f"{self} error: {msg['message']}")) + self._context_id = None async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): """Push frame and handle end-of-turn conditions.""" @@ -329,6 +336,7 @@ class RimeTTSService(WordTTSService, WebsocketService): self._started = True self._cumulative_time = 0 self._context_id = str(uuid.uuid4()) + await self.create_audio_context(self._context_id) msg = self._build_msg(text=text) await self._get_websocket().send(json.dumps(msg)) From a49d81e519e8916d3c9ec0a3666c3a618c37a666 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Feb 2025 17:06:04 -0500 Subject: [PATCH 147/427] tests: add tests for TranscriptProcessor --- tests/test_transcript_processor.py | 428 +++++++++++++++++++++++++++++ 1 file changed, 428 insertions(+) create mode 100644 tests/test_transcript_processor.py diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py new file mode 100644 index 000000000..1c6db277f --- /dev/null +++ b/tests/test_transcript_processor.py @@ -0,0 +1,428 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import unittest +from datetime import datetime, timezone +from typing import List, Tuple, cast + +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + CancelFrame, + StartInterruptionFrame, + TranscriptionFrame, + TranscriptionMessage, + TranscriptionUpdateFrame, + TTSTextFrame, +) +from pipecat.processors.transcript_processor import ( + AssistantTranscriptProcessor, + UserTranscriptProcessor, +) +from pipecat.tests.utils import SleepFrame, run_test + + +class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): + """Tests for UserTranscriptProcessor""" + + async def test_basic_transcription(self): + """Test basic transcription frame processing""" + # Create processor + processor = UserTranscriptProcessor() + + # Create test timestamp + timestamp = datetime.now(timezone.utc).isoformat() + + # Create frames to send + frames_to_send = [ + TranscriptionFrame(text="Hello, world!", user_id="test_user", timestamp=timestamp) + ] + + # Expected frames downstream - note the order: + # 1. TranscriptionUpdateFrame (processor emits the update first) + # 2. TranscriptionFrame (original frame is passed through) + expected_down_frames = [TranscriptionUpdateFrame, TranscriptionFrame] + + # Run test + received_frames, _ = await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify the content of the TranscriptionUpdateFrame + update_frame = cast( + TranscriptionUpdateFrame, received_frames[0] + ) # Note: now checking first frame + self.assertIsInstance(update_frame, TranscriptionUpdateFrame) + self.assertEqual(len(update_frame.messages), 1) + message = update_frame.messages[0] + self.assertEqual(message.role, "user") + self.assertEqual(message.content, "Hello, world!") + self.assertEqual(message.timestamp, timestamp) + + async def test_event_handler(self): + """Test that event handlers are called with transcript updates""" + # Create processor + processor = UserTranscriptProcessor() + + # Track received updates + received_updates: List[TranscriptionMessage] = [] + + # Register event handler + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.extend(frame.messages) + + # Create test data + timestamp = datetime.now(timezone.utc).isoformat() + frames_to_send = [ + TranscriptionFrame(text="First message", user_id="test_user", timestamp=timestamp), + TranscriptionFrame(text="Second message", user_id="test_user", timestamp=timestamp), + ] + + expected_down_frames = [ + TranscriptionUpdateFrame, + TranscriptionFrame, # First message + TranscriptionUpdateFrame, + TranscriptionFrame, # Second message + ] + + # Run test + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify event handler received updates + self.assertEqual(len(received_updates), 2) + + # Check first message + self.assertEqual(received_updates[0].role, "user") + self.assertEqual(received_updates[0].content, "First message") + self.assertEqual(received_updates[0].timestamp, timestamp) + + # Check second message + self.assertEqual(received_updates[1].role, "user") + self.assertEqual(received_updates[1].content, "Second message") + self.assertEqual(received_updates[1].timestamp, timestamp) + + async def test_text_aggregation(self): + """Test that TTSTextFrames are properly aggregated into a single message""" + # Create processor + processor = AssistantTranscriptProcessor() + + # Track received updates + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Create test frames simulating bot speaking multiple text chunks + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), # Wait for StartedSpeaking to process + TTSTextFrame(text="Hello"), + TTSTextFrame(text="world!"), + TTSTextFrame(text="How"), + TTSTextFrame(text="are"), + TTSTextFrame(text="you?"), + SleepFrame(sleep=0.1), # Wait for text frames to queue + BotStoppedSpeakingFrame(), + ] + + # Expected order: + # 1. BotStartedSpeakingFrame (system frame, immediate) + # 2. All queued TTSTextFrames + # 3. BotStoppedSpeakingFrame (system frame, immediate) + # 4. TranscriptionUpdateFrame (after aggregation) + expected_down_frames = [ + BotStartedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + BotStoppedSpeakingFrame, + TranscriptionUpdateFrame, + ] + + # Run test + received_frames, _ = await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify update was received + self.assertEqual(len(received_updates), 1) + + # Get the update frame + update_frame = received_updates[0] + + # Should have one aggregated message + self.assertEqual(len(update_frame.messages), 1) + + message = update_frame.messages[0] + self.assertEqual(message.role, "assistant") + self.assertEqual(message.content, "Hello world! How are you?") + + # Verify timestamp exists + self.assertIsNotNone(message.timestamp) + + # All frames should be passed through in order, with update at end + downstream_update = cast(TranscriptionUpdateFrame, received_frames[-1]) + self.assertEqual(downstream_update.messages[0].content, "Hello world! How are you?") + + async def test_empty_text_handling(self): + """Test that empty messages are not emitted""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text=""), # Empty text + TTSTextFrame(text=" "), # Just whitespace + TTSTextFrame(text="\n"), # Just newline + BotStoppedSpeakingFrame(), + # Pipeline ends here; run_test will automatically send EndFrame + ] + + # From our earlier tests, we know BotStoppedSpeakingFrame comes before TTSTextFrames + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, # empty + TTSTextFrame, # whitespace + TTSTextFrame, # newline + # No TranscriptionUpdateFrame since content is empty after stripping + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 0, "No updates should be emitted for empty content") + + async def test_interruption_handling(self): + """Test that messages are properly captured when bot is interrupted""" + processor = AssistantTranscriptProcessor() + + # Track received updates + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Simulate bot being interrupted mid-sentence + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Hello"), + TTSTextFrame(text="world"), + TTSTextFrame(text="!"), + SleepFrame(sleep=0.1), + StartInterruptionFrame(), # User interrupts here + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="New"), + TTSTextFrame(text="response"), + SleepFrame(sleep=0.1), + BotStoppedSpeakingFrame(), + ] + + # Actual order of frames: + expected_down_frames = [ + BotStartedSpeakingFrame, + TTSTextFrame, # "Hello" + TTSTextFrame, # "world" + TTSTextFrame, # "!" + TranscriptionUpdateFrame, # First message (emitted due to interruption) + StartInterruptionFrame, # Interruption frame comes after the update + BotStartedSpeakingFrame, + TTSTextFrame, # "New" + TTSTextFrame, # "response" + BotStoppedSpeakingFrame, + TranscriptionUpdateFrame, # Second message + ] + + # Run test + received_frames, _ = await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Should have received two updates + self.assertEqual(len(received_updates), 2) + + # First update should be interrupted message + first_message = received_updates[0].messages[0] + self.assertEqual(first_message.role, "assistant") + self.assertEqual(first_message.content, "Hello world !") + self.assertIsNotNone(first_message.timestamp) + + # Second update should be new response + second_message = received_updates[1].messages[0] + self.assertEqual(second_message.role, "assistant") + self.assertEqual(second_message.content, "New response") + self.assertIsNotNone(second_message.timestamp) + + # Verify timestamps are different + self.assertNotEqual(first_message.timestamp, second_message.timestamp) + + async def test_end_frame_handling(self): + """Test that final messages are captured when pipeline ends normally""" + processor = AssistantTranscriptProcessor() + + received_updates: List[TranscriptionUpdateFrame] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Hello"), + TTSTextFrame(text="world"), + # Pipeline ends here; run_test will automatically send EndFrame + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, # Final message emitted due to EndFrame + ] + + # Run test - EndFrame will be sent automatically + received_frames, _ = await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.role, "assistant") + self.assertEqual(message.content, "Hello world") + + async def test_cancel_frame_handling(self): + """Test that messages are properly captured when pipeline is cancelled""" + processor = AssistantTranscriptProcessor() + + # Track updates with timestamps to verify order + received_updates: List[Tuple[str, float]] = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + # Record message content and time received + received_updates.append((frame.messages[0].content, asyncio.get_event_loop().time())) + + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Hello"), + TTSTextFrame(text="world"), + SleepFrame(sleep=0.1), # Ensure messages are processed + CancelFrame(), + ] + + # We don't need to verify frame order, just that CancelFrame triggers message emission + expected_down_frames = [ + BotStartedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + CancelFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + send_end_frame=False, + ) + + # Verify that we received an update + self.assertEqual(len(received_updates), 1, "Should receive one update before cancellation") + content, _ = received_updates[0] + self.assertEqual(content, "Hello world") + + async def test_transcript_processor_factory(self): + """Test that factory properly manages processors and event handlers""" + from pipecat.processors.transcript_processor import TranscriptProcessor + + factory = TranscriptProcessor() + received_updates: List[TranscriptionMessage] = [] + + # Register handler with factory + @factory.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.extend(frame.messages) + + # Get processors and verify they're reused + user_proc1 = factory.user() + user_proc2 = factory.user() + self.assertIs(user_proc1, user_proc2, "User processor should be reused") + + asst_proc1 = factory.assistant() + asst_proc2 = factory.assistant() + self.assertIs(asst_proc1, asst_proc2, "Assistant processor should be reused") + + # Test user processor + timestamp = datetime.now(timezone.utc).isoformat() + frames_to_send = [ + TranscriptionFrame(text="User message", user_id="user1", timestamp=timestamp) + ] + + await run_test( + user_proc1, + frames_to_send=frames_to_send, + expected_down_frames=[TranscriptionUpdateFrame, TranscriptionFrame], + ) + + # Test assistant processor + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Assistant"), + TTSTextFrame(text="message"), + BotStoppedSpeakingFrame(), + ] + + # The actual order we see in the output: + await run_test( + asst_proc1, + frames_to_send=frames_to_send, + expected_down_frames=[ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ], + ) + + # Verify both processors triggered the same handler + self.assertEqual(len(received_updates), 2) + self.assertEqual(received_updates[0].role, "user") + self.assertEqual(received_updates[0].content, "User message") + self.assertEqual(received_updates[1].role, "assistant") + self.assertEqual(received_updates[1].content, "Assistant message") From c26fe3f277061bd6be24e3cecb21ca811dc1806d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Feb 2025 20:28:26 -0500 Subject: [PATCH 148/427] fix: ensure proper Google message format conversion in transcription filter --- examples/foundational/25-google-audio-in.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index a6108be09..af6e267dc 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -143,7 +143,14 @@ class InputTranscriptionContextFilter(FrameProcessor): return try: - message = frame.context.messages[-1] + # Make sure we're working with a GoogleLLMContext + context = GoogleLLMContext.upgrade_to_google(frame.context) + message = context.messages[-1] + + if not isinstance(message, glm.Content): + logger.error(f"Expected glm.Content, got {type(message)}") + return + last_part = message.parts[-1] if not ( message.role == "user" From 426d7ac2133d6c2076dc498abfb7b37783990d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 13:11:35 -0800 Subject: [PATCH 149/427] transports: some local audio and tk updates --- examples/foundational/01a-local-audio.py | 4 ++-- examples/foundational/13a-whisper-local.py | 4 ++-- examples/local-input-select-stt/bot.py | 4 ++-- src/pipecat/transports/local/audio.py | 18 ++++++++++-------- src/pipecat/transports/local/tk.py | 13 ++++++++++++- 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index 633697684..20aaedfae 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -16,7 +16,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia import CartesiaTTSService -from pipecat.transports.local.audio import LocalAudioTransport, LocalTransportParams +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams load_dotenv(override=True) @@ -25,7 +25,7 @@ logger.add(sys.stderr, level="DEBUG") async def main(): - transport = LocalAudioTransport(LocalTransportParams(audio_out_enabled=True)) + transport = LocalAudioTransport(LocalAudioTransportParams(audio_out_enabled=True)) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), diff --git a/examples/foundational/13a-whisper-local.py b/examples/foundational/13a-whisper-local.py index 2d3cd5a57..c9bf16782 100644 --- a/examples/foundational/13a-whisper-local.py +++ b/examples/foundational/13a-whisper-local.py @@ -16,7 +16,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper import WhisperSTTService -from pipecat.transports.local.audio import LocalAudioTransport, LocalTransportParams +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams load_dotenv(override=True) @@ -33,7 +33,7 @@ class TranscriptionLogger(FrameProcessor): async def main(): - transport = LocalAudioTransport(LocalTransportParams(audio_in_enabled=True)) + transport = LocalAudioTransport(LocalAudioTransportParams(audio_in_enabled=True)) stt = WhisperSTTService() diff --git a/examples/local-input-select-stt/bot.py b/examples/local-input-select-stt/bot.py index 1cedbf96c..f472152b9 100644 --- a/examples/local-input-select-stt/bot.py +++ b/examples/local-input-select-stt/bot.py @@ -18,7 +18,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper import Model, WhisperSTTService -from pipecat.transports.local.audio import LocalAudioTransport, LocalTransportParams +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams load_dotenv(override=True) @@ -36,7 +36,7 @@ class TranscriptionLogger(FrameProcessor): async def main(input_device: int, output_device: int): transport = LocalAudioTransport( - LocalTransportParams( + LocalAudioTransportParams( audio_in_enabled=True, audio_out_enabled=False, input_device_index=input_device, diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index 30bdaaf79..e7b803f2b 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -26,17 +26,18 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -class LocalTransportParams(TransportParams): - input_device_index: int = 0 - output_device_index: int = 0 +class LocalAudioTransportParams(TransportParams): + input_device_index: Optional[int] = None + output_device_index: Optional[int] = None class LocalAudioInputTransport(BaseInputTransport): - _params: LocalTransportParams + _params: LocalAudioTransportParams - def __init__(self, py_audio: pyaudio.PyAudio, params: LocalTransportParams): + def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams): super().__init__(params) self._py_audio = py_audio + self._in_stream = None self._sample_rate = 0 @@ -77,11 +78,12 @@ class LocalAudioInputTransport(BaseInputTransport): class LocalAudioOutputTransport(BaseOutputTransport): - _params: LocalTransportParams + _params: LocalAudioTransportParams - def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams): + def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams): super().__init__(params) self._py_audio = py_audio + self._out_stream = None self._sample_rate = 0 @@ -117,7 +119,7 @@ class LocalAudioOutputTransport(BaseOutputTransport): class LocalAudioTransport(BaseTransport): - def __init__(self, params: LocalTransportParams): + def __init__(self, params: LocalAudioTransportParams): super().__init__() self._params = params self._pyaudio = pyaudio.PyAudio() diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index 40147d39f..73777c573 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -34,8 +34,15 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +class TkTransportParams(TransportParams): + audio_input_device_index: Optional[int] = None + audio_output_device_index: Optional[int] = None + + class TkInputTransport(BaseInputTransport): - def __init__(self, py_audio: pyaudio.PyAudio, params: TransportParams): + _params: TkTransportParams + + def __init__(self, py_audio: pyaudio.PyAudio, params: TkTransportParams): super().__init__(params) self._py_audio = py_audio self._in_stream = None @@ -54,6 +61,7 @@ class TkInputTransport(BaseInputTransport): frames_per_buffer=num_frames, stream_callback=self._audio_in_callback, input=True, + input_device_index=self._params.audio_input_device_index, ) self._in_stream.start_stream() @@ -76,6 +84,8 @@ class TkInputTransport(BaseInputTransport): class TkOutputTransport(BaseOutputTransport): + _params: TkTransportParams + def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams): super().__init__(params) self._py_audio = py_audio @@ -103,6 +113,7 @@ class TkOutputTransport(BaseOutputTransport): channels=self._params.audio_out_channels, rate=self._sample_rate, output=True, + output_device_index=self._params.audio_output_device_index, ) self._out_stream.start_stream() From 5126d4de92212e4f3addb5bcb205f5a1f5b8559b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 14:34:38 -0800 Subject: [PATCH 150/427] tts: handle incoming frames pausing/resuming from base TTSService class --- src/pipecat/services/ai_services.py | 21 +++++++++++++++++++++ src/pipecat/services/cartesia.py | 13 ------------- src/pipecat/services/elevenlabs.py | 13 ------------- src/pipecat/services/fish.py | 10 ---------- src/pipecat/services/playht.py | 13 ------------- src/pipecat/services/rime.py | 18 +----------------- 6 files changed, 22 insertions(+), 66 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 0fce2482a..7d49c5676 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -15,6 +15,7 @@ from loguru import logger from pipecat.audio.utils import calculate_audio_volume, exp_smoothing from pipecat.frames.frames import ( AudioRawFrame, + BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, @@ -234,6 +235,7 @@ class TTSService(AIService): self._stop_frame_queue: asyncio.Queue = asyncio.Queue() self._current_sentence: str = "" + self._processing_text: bool = False @property def sample_rate(self) -> int: @@ -299,6 +301,7 @@ class TTSService(AIService): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) + if ( isinstance(frame, TextFrame) and not isinstance(frame, InterimTranscriptionFrame) @@ -308,8 +311,15 @@ class TTSService(AIService): elif isinstance(frame, StartInterruptionFrame): await self._handle_interruption(frame, direction) elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): + # We pause processing incoming frames if the LLM response included + # text (it might be that it's only a function calling response). We + # pause to avoid audio overlapping. + if self._processing_text: + await self.pause_processing_frames() + sentence = self._current_sentence self._current_sentence = "" + self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): if self._push_text_frames: @@ -317,10 +327,16 @@ class TTSService(AIService): else: await self.push_frame(frame, direction) elif isinstance(frame, TTSSpeakFrame): + # We pause processing incoming frames because we are sending data to + # the TTS. We pause to avoid audio overlapping. + await self.pause_processing_frames() await self._push_tts_frames(frame.text) await self.flush_audio() + self._processing_text = False elif isinstance(frame, TTSUpdateSettingsFrame): await self._update_settings(frame.settings) + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.resume_processing_frames() else: await self.push_frame(frame, direction) @@ -371,6 +387,11 @@ class TTSService(AIService): if not text.strip(): return + # This is just a flag that indicates if we sent something to the TTS + # service. It will be cleared if we sent text because of a TTSSpeakFrame + # or when we received an LLMFullResponseEndFrame + self._processing_text = True + await self.start_processing_metrics() if self._text_filter: self._text_filter.reset_interruption() diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 49bcac2f1..becd39f79 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -274,19 +274,6 @@ class CartesiaTTSService(AudioContextWordTTSService, WebsocketService): else: logger.error(f"{self} error, unknown message type: {msg}") - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - # If we received a TTSSpeakFrame and the LLM response included text (it - # might be that it's only a function calling response) we pause - # processing more frames until we receive a BotStoppedSpeakingFrame. - if isinstance(frame, TTSSpeakFrame): - await self.pause_processing_frames() - elif isinstance(frame, LLMFullResponseEndFrame) and self._context_id: - await self.pause_processing_frames() - elif isinstance(frame, BotStoppedSpeakingFrame): - await self.resume_processing_frames() - async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating TTS: [{text}]") diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index b2047d34e..d8acedd6c 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -289,19 +289,6 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): if isinstance(frame, TTSStoppedFrame): await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - # If we received a TTSSpeakFrame and the LLM response included text (it - # might be that it's only a function calling response) we pause - # processing more frames until we receive a BotStoppedSpeakingFrame. - if isinstance(frame, TTSSpeakFrame): - await self.pause_processing_frames() - elif isinstance(frame, LLMFullResponseEndFrame) and self._started: - await self.pause_processing_frames() - elif isinstance(frame, BotStoppedSpeakingFrame): - await self.resume_processing_frames() - async def _connect(self): await self._connect_websocket() diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index d61514eb2..e36eaf497 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -166,16 +166,6 @@ class FishAudioTTSService(TTSService, WebsocketService): except Exception as e: logger.error(f"Error processing message: {e}") - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - if isinstance(frame, TTSSpeakFrame): - await self.pause_processing_frames() - elif isinstance(frame, LLMFullResponseEndFrame) and self._request_id: - await self.pause_processing_frames() - elif isinstance(frame, BotStoppedSpeakingFrame): - await self.resume_processing_frames() - async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): await super()._handle_interruption(frame, direction) await self.stop_all_metrics() diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index bd9463f91..3196c306d 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -269,19 +269,6 @@ class PlayHTTTSService(TTSService, WebsocketService): except json.JSONDecodeError: logger.error(f"Invalid JSON message: {message}") - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - # If we received a TTSSpeakFrame and the LLM response included text (it - # might be that it's only a function calling response) we pause - # processing more frames until we receive a BotStoppedSpeakingFrame. - if isinstance(frame, TTSSpeakFrame): - await self.pause_processing_frames() - elif isinstance(frame, LLMFullResponseEndFrame) and self._request_id: - await self.pause_processing_frames() - elif isinstance(frame, BotStoppedSpeakingFrame): - await self.resume_processing_frames() - async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating TTS: [{text}]") diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 1634578cd..79e557f48 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -126,7 +126,6 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): # State tracking self._context_id = None # Tracks current turn self._receive_task = None - self._started = False self._cumulative_time = 0 # Accumulates time across messages def can_generate_metrics(self) -> bool: @@ -200,7 +199,6 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): await self._websocket.send(json.dumps(self._build_eos_msg())) await self._websocket.close() self._websocket = None - self._started = False self._context_id = None except Exception as e: logger.error(f"{self} error closing websocket: {e}") @@ -217,7 +215,6 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): await self.stop_all_metrics() if self._context_id: await self._get_websocket().send(json.dumps(self._build_clear_msg())) - self._started = False self._context_id = None def _calculate_word_times(self, words: list, starts: list, ends: list) -> list: @@ -300,21 +297,9 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): """Push frame and handle end-of-turn conditions.""" await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): - self._started = False if isinstance(frame, TTSStoppedFrame): await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames and manage turn state.""" - await super().process_frame(frame, direction) - - if isinstance(frame, TTSSpeakFrame): - await self.pause_processing_frames() - elif isinstance(frame, LLMFullResponseEndFrame) and self._started: - await self.pause_processing_frames() - elif isinstance(frame, BotStoppedSpeakingFrame): - await self.resume_processing_frames() - async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text. @@ -330,10 +315,9 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): await self._connect() try: - if not self._started: + if not self._context_id: await self.start_ttfb_metrics() yield TTSStartedFrame() - self._started = True self._cumulative_time = 0 self._context_id = str(uuid.uuid4()) await self.create_audio_context(self._context_id) From 67da745bb3741db1305a9d04f4163c29e1ddd6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 14:45:35 -0800 Subject: [PATCH 151/427] tts: make frame pausing/resuming optional --- src/pipecat/services/ai_services.py | 24 +++++++++++++++++------- src/pipecat/services/cartesia.py | 1 + src/pipecat/services/elevenlabs.py | 1 + src/pipecat/services/fish.py | 2 +- src/pipecat/services/lmnt.py | 1 + src/pipecat/services/playht.py | 1 + src/pipecat/services/rime.py | 1 + 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 7d49c5676..99fd05481 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -76,13 +76,13 @@ class AIService(FrameProcessor): ) for key, value in settings.items(): - print("Update request for:", key, value) + logger.debug("Update request for:", key, value) if key in self._settings: logger.info(f"Updating LLM setting {key} to: [{value}]") self._settings[key] = value elif key in SessionProperties.model_fields: - print("Attempting to update", key, value) + logger.debug("Attempting to update", key, value) try: from pipecat.services.openai_realtime_beta.events import ( @@ -213,6 +213,8 @@ class TTSService(AIService): push_silence_after_stop: bool = False, # if push_silence_after_stop is True, send this amount of audio silence silence_time_s: float = 2.0, + # if True, we will pause processing frames while we are receiving audio + pause_frame_processing: bool = False, # TTS output sample rate sample_rate: Optional[int] = None, text_filter: Optional[BaseTextFilter] = None, @@ -225,6 +227,7 @@ class TTSService(AIService): self._stop_frame_timeout_s: float = stop_frame_timeout_s self._push_silence_after_stop: bool = push_silence_after_stop self._silence_time_s: float = silence_time_s + self._pause_frame_processing: bool = pause_frame_processing self._init_sample_rate = sample_rate self._sample_rate = 0 self._voice_id: str = "" @@ -314,8 +317,7 @@ class TTSService(AIService): # We pause processing incoming frames if the LLM response included # text (it might be that it's only a function calling response). We # pause to avoid audio overlapping. - if self._processing_text: - await self.pause_processing_frames() + await self._maybe_pause_frame_processing() sentence = self._current_sentence self._current_sentence = "" @@ -327,16 +329,16 @@ class TTSService(AIService): else: await self.push_frame(frame, direction) elif isinstance(frame, TTSSpeakFrame): + await self._push_tts_frames(frame.text) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. - await self.pause_processing_frames() - await self._push_tts_frames(frame.text) + await self._maybe_pause_frame_processing() await self.flush_audio() self._processing_text = False elif isinstance(frame, TTSUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, BotStoppedSpeakingFrame): - await self.resume_processing_frames() + await self._maybe_resume_frame_processing() else: await self.push_frame(frame, direction) @@ -367,6 +369,14 @@ class TTSService(AIService): self._text_filter.handle_interruption() await self.push_frame(frame, direction) + async def _maybe_pause_frame_processing(self): + if self._processing_text and self._pause_frame_processing: + await self.pause_processing_frames() + + async def _maybe_resume_frame_processing(self): + if self._pause_frame_processing: + await self.resume_processing_frames() + async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None if not self._aggregate_sentences: diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index becd39f79..67fc67ebf 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -109,6 +109,7 @@ class CartesiaTTSService(AudioContextWordTTSService, WebsocketService): self, aggregate_sentences=True, push_text_frames=False, + pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index d8acedd6c..3c8cbb3ed 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -192,6 +192,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): push_text_frames=False, push_stop_frames=True, stop_frame_timeout_s=2.0, + pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index e36eaf497..94475aca7 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -60,7 +60,7 @@ class FishAudioTTSService(TTSService, WebsocketService): params: InputParams = InputParams(), **kwargs, ): - super().__init__(sample_rate=sample_rate, **kwargs) + super().__init__(pause_frame_processing=True, sample_rate=sample_rate, **kwargs) self._api_key = api_key self._base_url = "wss://api.fish.audio/v1/tts/live" diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 645494cc1..0272690e8 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -73,6 +73,7 @@ class LmntTTSService(TTSService, WebsocketService): TTSService.__init__( self, push_stop_frames=True, + pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 3196c306d..297e802d3 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -120,6 +120,7 @@ class PlayHTTTSService(TTSService, WebsocketService): ): TTSService.__init__( self, + pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 79e557f48..f87405a63 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -101,6 +101,7 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): push_text_frames=False, push_stop_frames=True, stop_frame_timeout_s=2.0, + pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) From 633a4d4c58bcef03246be08ba92081bedd33c184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 15:42:05 -0800 Subject: [PATCH 152/427] FalImageGenService: load image async to not block the event loop --- CHANGELOG.md | 3 +++ src/pipecat/services/fal.py | 14 +++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5542674ec..c8590e665 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `FalImageGenService` issue that was causing the event loop to be + blocked while loading the downloadded image. + - Fixed a `CartesiaTTSService` service issue that would cause audio overlapping in some cases. diff --git a/src/pipecat/services/fal.py b/src/pipecat/services/fal.py index d229861f1..7173861ab 100644 --- a/src/pipecat/services/fal.py +++ b/src/pipecat/services/fal.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import asyncio import io import os from typing import AsyncGenerator, Dict, Optional, Union @@ -53,6 +54,11 @@ class FalImageGenService(ImageGenService): os.environ["FAL_KEY"] = key async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + def load_image_bytes(encoded_image: bytes): + buffer = io.BytesIO(encoded_image) + image = Image.open(buffer) + return (image.tobytes(), image.size, image.format) + logger.debug(f"Generating image from prompt: {prompt}") response = await fal_client.run_async( @@ -73,10 +79,8 @@ class FalImageGenService(ImageGenService): logger.debug(f"Downloading image {image_url} ...") async with self._aiohttp_session.get(image_url) as response: logger.debug(f"Downloaded image {image_url}") - image_stream = io.BytesIO(await response.content.read()) - image = Image.open(image_stream) + encoded_image = await response.content.read() + (image_bytes, size, format) = await asyncio.to_thread(load_image_bytes, encoded_image) - frame = URLImageRawFrame( - url=image_url, image=image.tobytes(), size=image.size, format=image.format - ) + frame = URLImageRawFrame(url=image_url, image=image_bytes, size=size, format=format) yield frame From f6912c0f9a45efcb9e3d375985c11618aad36a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 15:44:13 -0800 Subject: [PATCH 153/427] utils: don't consider colon an end of sentence --- CHANGELOG.md | 2 ++ src/pipecat/utils/string.py | 4 ++-- tests/test_utils_string.py | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8590e665..0201fdc23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- We don't consider a colon `:` and end of sentence any more. + - Updated `DailyTransport` to respect the `audio_in_stream_on_start` field, ensuring it only starts receiving the audio input if it is enabled. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index c269c5e57..916dfad41 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -13,8 +13,8 @@ ENDOFSENTENCE_PATTERN_STR = r""" (? Date: Fri, 14 Feb 2025 15:51:03 -0800 Subject: [PATCH 154/427] LLMAssistantResponseAggregator: initialize messages --- src/pipecat/processors/aggregators/llm_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 950c155e6..c12b0d90b 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -420,7 +420,7 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): - def __init__(self, messages: List[dict], **kwargs): + def __init__(self, messages: List[dict] = [], **kwargs): super().__init__(context=OpenAILLMContext(messages), **kwargs) async def push_aggregation(self): From 63950912f01c2ffa79d53b6aa8c67aa52ac3043b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 15:55:22 -0800 Subject: [PATCH 155/427] LLMAssistantContextAggregator: add missing variable initialization --- src/pipecat/processors/aggregators/llm_response.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index c12b0d90b..ca73649b2 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -358,6 +358,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): super().__init__(context=context, role="assistant", **kwargs) self._expect_stripped_words = expect_stripped_words + self._started = False + self.reset() async def process_frame(self, frame: Frame, direction: FrameDirection): From a107b1cb4b40c0d2ed6d5c0d35a8eae1ac6e139c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 15:57:18 -0800 Subject: [PATCH 156/427] examples(06a): use CartesiaTTSService --- examples/foundational/06a-image-sync.py | 4 ++-- src/pipecat/processors/aggregators/llm_response.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 841c335f6..03a7fd3b8 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -27,7 +27,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.cartesia import CartesiaHttpTTSService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -91,7 +91,7 @@ async def main(): ), ) - tts = CartesiaHttpTTSService( + tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index ca73649b2..2de3ebae4 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -10,7 +10,6 @@ from abc import abstractmethod from typing import List from pipecat.frames.frames import ( - BotInterruptionFrame, CancelFrame, EmulateUserStartedSpeakingFrame, EmulateUserStoppedSpeakingFrame, @@ -281,6 +280,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await self._cancel_aggregation_task() async def _handle_user_started_speaking(self, _: UserStartedSpeakingFrame): + self._last_user_speaking_time = time.time() self._user_speaking = True async def _handle_user_stopped_speaking(self, _: UserStoppedSpeakingFrame): From 1f5b790dd0f2bc42d24df90b3951a64946d071f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 16:41:10 -0800 Subject: [PATCH 157/427] TTSService: reset processing text during interruptions --- src/pipecat/services/ai_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 99fd05481..0b67cd3ce 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -365,6 +365,7 @@ class TTSService(AIService): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): self._current_sentence = "" + self._processing_text = False if self._text_filter: self._text_filter.handle_interruption() await self.push_frame(frame, direction) From 883410d8ac4a3b89bc06f396c0fddd62f9b2fa01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 16:42:00 -0800 Subject: [PATCH 158/427] FrameProcessor: no need to create an input event every time --- src/pipecat/processors/frame_processor.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 3fb515a87..f27c6d6c2 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -73,10 +73,11 @@ class FrameProcessor: self._metrics.set_processor_name(self.name) # Processors have an input queue. The input queue will be processed - # immediately (default) or it will block if `pause_processing_frames()` is - # called. To resume processing frames we need to call - # `resume_processing_frames()`. + # immediately (default) or it will block if `pause_processing_frames()` + # is called. To resume processing frames we need to call + # `resume_processing_frames()` which will wake up the event. self.__should_block_frames = False + self.__input_event = asyncio.Event() self.__input_frame_task: Optional[asyncio.Task] = None # Every processor in Pipecat should only output frames from a single @@ -335,8 +336,8 @@ class FrameProcessor: def __create_input_task(self): if not self.__input_frame_task: self.__should_block_frames = False + self.__input_event.clear() self.__input_queue = asyncio.Queue() - self.__input_event = asyncio.Event() self.__input_frame_task = self.create_task(self.__input_frame_task_handler()) async def __cancel_input_task(self): From 329e89c1d938be4f297b5f84e6f10668b301f537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 17:17:47 -0800 Subject: [PATCH 159/427] TTSService: push BotStoppedSpeakingFrame --- src/pipecat/services/ai_services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 0b67cd3ce..c5adc89e0 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -313,6 +313,7 @@ class TTSService(AIService): await self._process_text_frame(frame) elif isinstance(frame, StartInterruptionFrame): await self._handle_interruption(frame, direction) + await self.push_frame(frame, direction) elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)): # We pause processing incoming frames if the LLM response included # text (it might be that it's only a function calling response). We @@ -339,6 +340,7 @@ class TTSService(AIService): await self._update_settings(frame.settings) elif isinstance(frame, BotStoppedSpeakingFrame): await self._maybe_resume_frame_processing() + await self.push_frame(frame, direction) else: await self.push_frame(frame, direction) @@ -368,7 +370,6 @@ class TTSService(AIService): self._processing_text = False if self._text_filter: self._text_filter.handle_interruption() - await self.push_frame(frame, direction) async def _maybe_pause_frame_processing(self): if self._processing_text and self._pause_frame_processing: From 01c06c5cac5e4e53016a42e363af531f00d6afd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 14 Feb 2025 14:12:02 -0800 Subject: [PATCH 160/427] update CHANGELOG for 0.0.57 --- CHANGELOG.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0201fdc23..a8731571f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.57] - 2025-02-14 ### Added @@ -139,9 +139,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue[#1192] in 11labs where we are trying to reconnect/disconnect the websocket connection even when the connection is already closed. -- Fixed an issue where `has_regular_messages` condition was always been true in - `GoogleLLMContext` due to `Part` having `function_call` & `function_response` with - `None` values. +- Fixed an issue where `has_regular_messages` condition was always true in + `GoogleLLMContext` due to `Part` having `function_call` & `function_response` + with `None` values. + +### Other + +- Added new `instant-voice` example. This example showcases how to enable + instant voice communication as soon as a user connects. + +- Added new `local-input-select-stt` example. This examples allows you to play + with local audio inputs by slecting them through a nice text interface. ## [0.0.56] - 2025-02-06 From 5f937b8479e891afa77e607a0c473e96604be1f1 Mon Sep 17 00:00:00 2001 From: Carl Kho <106736711+CarlKho-Minerva@users.noreply.github.com> Date: Fri, 14 Feb 2025 21:14:32 -0800 Subject: [PATCH 161/427] Update test requirements to include Cartesia version 1.3.1 --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index 6a92557e0..82a6fbb25 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ aiohttp~=3.10.3 anthropic~=0.30.0 azure-cognitiveservices-speech~=1.40.0 boto3~=1.35.27 +cartesia~=1.3.1 daily-python~=0.11.0 deepgram-sdk~=3.5.0 fal-client~=0.4.1 From a5cdd5f1b8fd313937d9aaf0ca286b93e87fe52f Mon Sep 17 00:00:00 2001 From: Carl Kho <106736711+CarlKho-Minerva@users.noreply.github.com> Date: Fri, 14 Feb 2025 21:29:37 -0800 Subject: [PATCH 162/427] Add Cartesia API key to dot-env.template --- dot-env.template | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dot-env.template b/dot-env.template index 1e902be60..e31681235 100644 --- a/dot-env.template +++ b/dot-env.template @@ -18,6 +18,9 @@ AZURE_DALLE_API_KEY=... AZURE_DALLE_ENDPOINT=https://... AZURE_DALLE_MODEL=... +# Cartesia +CARTESIA_API_KEY=... + # Daily DAILY_API_KEY=... DAILY_SAMPLE_ROOM_URL=https://... From 90b9dce71043f5c771cc989ed1540f0838994edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 17 Feb 2025 18:59:13 -0800 Subject: [PATCH 163/427] STTMuteFilter: ignore audio frames so no transcriptions are generated --- CHANGELOG.md | 7 ++++ .../processors/filters/stt_mute_filter.py | 2 ++ tests/test_stt_mute_filter.py | 33 ++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8731571f..373a03ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Fixed + +- Fixed a `STTMuteFilter` issue that would not mute user audio frames causing + transcriptions to be generated by the STT service. + ## [0.0.57] - 2025-02-14 ### Added diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index fb1753263..937f3b075 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -23,6 +23,7 @@ from pipecat.frames.frames import ( Frame, FunctionCallInProgressFrame, FunctionCallResultFrame, + InputAudioRawFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, @@ -185,6 +186,7 @@ class STTMuteFilter(FrameProcessor): StopInterruptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, + InputAudioRawFrame, ), ): # Only pass VAD-related frames when not muted diff --git a/tests/test_stt_mute_filter.py b/tests/test_stt_mute_filter.py index 62bdd03c9..a55c4609e 100644 --- a/tests/test_stt_mute_filter.py +++ b/tests/test_stt_mute_filter.py @@ -11,12 +11,13 @@ from pipecat.frames.frames import ( BotStoppedSpeakingFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, + InputAudioRawFrame, STTMuteFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) from pipecat.processors.filters.stt_mute_filter import STTMuteConfig, STTMuteFilter, STTMuteStrategy -from pipecat.tests.utils import run_test +from pipecat.tests.utils import SleepFrame, run_test class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): @@ -26,10 +27,14 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), # First bot speech starts UserStartedSpeakingFrame(), # Should be suppressed + InputAudioRawFrame( + audio=b"", sample_rate=16000, num_channels=1 + ), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # First bot speech ends BotStartedSpeakingFrame(), # Second bot speech UserStartedSpeakingFrame(), # Should pass through + InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStoppedSpeakingFrame(), ] @@ -41,6 +46,7 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): STTMuteFrame, # mute=False BotStartedSpeakingFrame, UserStartedSpeakingFrame, # Now passes through + InputAudioRawFrame, # Now passes through UserStoppedSpeakingFrame, # Now passes through BotStoppedSpeakingFrame, ] @@ -57,12 +63,19 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): frames_to_send = [ BotStartedSpeakingFrame(), # First speech starts UserStartedSpeakingFrame(), # Should be suppressed + InputAudioRawFrame( + audio=b"", sample_rate=16000, num_channels=1 + ), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # First speech ends UserStartedSpeakingFrame(), # Should pass through + InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStartedSpeakingFrame(), # Second speech starts UserStartedSpeakingFrame(), # Should be suppressed again + InputAudioRawFrame( + audio=b"", sample_rate=16000, num_channels=1 + ), # Should be suppressed again UserStoppedSpeakingFrame(), # Should be suppressed again BotStoppedSpeakingFrame(), # Second speech ends ] @@ -73,6 +86,7 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): BotStoppedSpeakingFrame, STTMuteFrame, # mute=False UserStartedSpeakingFrame, + InputAudioRawFrame, UserStoppedSpeakingFrame, BotStartedSpeakingFrame, STTMuteFrame, # mute=True @@ -134,15 +148,23 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): frames_to_send = [ UserStartedSpeakingFrame(), # Should be suppressed (starts muted) + InputAudioRawFrame( + audio=b"", sample_rate=16000, num_channels=1 + ), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStartedSpeakingFrame(), # First bot speech UserStartedSpeakingFrame(), # Should be suppressed + InputAudioRawFrame( + audio=b"", sample_rate=16000, num_channels=1 + ), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # First speech ends, unmutes UserStartedSpeakingFrame(), # Should pass through + InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStartedSpeakingFrame(), # Second speech UserStartedSpeakingFrame(), # Should pass through + InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStoppedSpeakingFrame(), ] @@ -153,9 +175,11 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): BotStoppedSpeakingFrame, STTMuteFrame, # mute=False after first speech UserStartedSpeakingFrame, + InputAudioRawFrame, UserStoppedSpeakingFrame, BotStartedSpeakingFrame, UserStartedSpeakingFrame, + InputAudioRawFrame, UserStoppedSpeakingFrame, BotStoppedSpeakingFrame, ] @@ -190,23 +214,30 @@ class TestSTTMuteFilter(unittest.IsolatedAsyncioTestCase): frames_to_send = [ UserStartedSpeakingFrame(), # Should pass through + InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through UserStoppedSpeakingFrame(), # Should pass through BotStartedSpeakingFrame(), # Bot starts speaking UserStartedSpeakingFrame(), # Should be suppressed + InputAudioRawFrame( + audio=b"", sample_rate=16000, num_channels=1 + ), # Should be suppressed UserStoppedSpeakingFrame(), # Should be suppressed BotStoppedSpeakingFrame(), # Bot stops speaking UserStartedSpeakingFrame(), # Should pass through + InputAudioRawFrame(audio=b"", sample_rate=16000, num_channels=1), # Should pass through UserStoppedSpeakingFrame(), # Should pass through ] expected_returned_frames = [ UserStartedSpeakingFrame, + InputAudioRawFrame, UserStoppedSpeakingFrame, BotStartedSpeakingFrame, STTMuteFrame, # mute=True BotStoppedSpeakingFrame, STTMuteFrame, # mute=False UserStartedSpeakingFrame, + InputAudioRawFrame, UserStoppedSpeakingFrame, ] From c926063d743c14eba99ce87f01b3e507359c5e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Feb 2025 09:39:13 -0800 Subject: [PATCH 164/427] deepgram: handle error event and reconnect --- CHANGELOG.md | 4 ++++ src/pipecat/services/deepgram.py | 33 ++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 373a03ebd..7d4de0b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that would cause `DeepgramSTTService` to stop working after an + error occurred (e.g. sudden network loss). If the network recovered we would + not reconnect. + - Fixed a `STTMuteFilter` issue that would not mute user audio frames causing transcriptions to be generated by the STT service. diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index b9d39fa63..43782e4eb 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -34,6 +34,7 @@ try: AsyncListenWebSocketClient, DeepgramClient, DeepgramClientOptions, + ErrorResponse, LiveOptions, LiveResultResponse, LiveTranscriptionEvents, @@ -155,13 +156,10 @@ class DeepgramSTTService(STTService): options={"keepalive": "true"}, # verbose=logging.DEBUG ), ) - self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1") - self._connection.on(LiveTranscriptionEvents.Transcript, self._on_message) + if self.vad_enabled: self._register_event_handler("on_speech_started") self._register_event_handler("on_utterance_end") - self._connection.on(LiveTranscriptionEvents.SpeechStarted, self._on_speech_started) - self._connection.on(LiveTranscriptionEvents.UtteranceEnd, self._on_utterance_end) @property def vad_enabled(self): @@ -202,6 +200,24 @@ class DeepgramSTTService(STTService): async def _connect(self): logger.debug("Connecting to Deepgram") + + self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1") + + self._connection.on( + LiveTranscriptionEvents(LiveTranscriptionEvents.Transcript), self._on_message + ) + self._connection.on(LiveTranscriptionEvents(LiveTranscriptionEvents.Error), self._on_error) + + if self.vad_enabled: + self._connection.on( + LiveTranscriptionEvents(LiveTranscriptionEvents.SpeechStarted), + self._on_speech_started, + ) + self._connection.on( + LiveTranscriptionEvents(LiveTranscriptionEvents.UtteranceEnd), + self._on_utterance_end, + ) + if not await self._connection.start(self._settings): logger.error(f"{self}: unable to connect to Deepgram") @@ -214,6 +230,15 @@ class DeepgramSTTService(STTService): await self.start_ttfb_metrics() await self.start_processing_metrics() + async def _on_error(self, *args, **kwargs): + error: ErrorResponse = kwargs["error"] + logger.warning(f"{self} connection error, will retry: {error}") + await self.stop_all_metrics() + # NOTE(aleix): we don't disconnect (i.e. call finish on the connection) + # because this triggers more errors internally in the Deepgram SDK. So, + # we just forget about the previous connection and create a new one. + await self._connect() + async def _on_speech_started(self, *args, **kwargs): await self.start_metrics() await self._call_event_handler("on_speech_started", *args, **kwargs) From e809c8680e1c702257e993a95c6df44442de52a2 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Tue, 18 Feb 2025 15:12:44 -0300 Subject: [PATCH 165/427] Upgrading to use the latest node stable version --- examples/bot-ready-signalling/client/react-native/.nvmrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bot-ready-signalling/client/react-native/.nvmrc b/examples/bot-ready-signalling/client/react-native/.nvmrc index 4a58985bb..744ca17ec 100644 --- a/examples/bot-ready-signalling/client/react-native/.nvmrc +++ b/examples/bot-ready-signalling/client/react-native/.nvmrc @@ -1 +1 @@ -18.18 +22.14 From 50b6cc813520a8bdaa46d654ff681b3b1fddef6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Feb 2025 10:09:49 -0800 Subject: [PATCH 166/427] network: added exponential_backoff_time() function --- CHANGELOG.md | 4 +++ src/pipecat/services/websocket_service.py | 24 ++-------------- src/pipecat/utils/network.py | 27 ++++++++++++++++++ tests/test_utils_network.py | 34 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 22 deletions(-) create mode 100644 src/pipecat/utils/network.py create mode 100644 tests/test_utils_network.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d4de0b74..6be017e5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Added `exponential_backoff_time()` to `utils.network` module. + ### Fixed - Fixed an issue that would cause `DeepgramSTTService` to stop working after an diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index c47d6d97e..a8dc97414 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -13,6 +13,7 @@ from loguru import logger from websockets.protocol import State from pipecat.frames.frames import ErrorFrame +from pipecat.utils.network import exponential_backoff_time class WebsocketService(ABC): @@ -51,27 +52,6 @@ class WebsocketService(ABC): await self._connect_websocket() return await self._verify_connection() - def _calculate_wait_time( - self, attempt: int, min_wait: float = 4, max_wait: float = 10, multiplier: float = 1 - ) -> float: - """Calculate exponential backoff wait time. - - Args: - attempt: Current attempt number (1-based) - min_wait: Minimum wait time in seconds - max_wait: Maximum wait time in seconds - multiplier: Base multiplier for exponential calculation - - Returns: - Wait time in seconds - """ - try: - exp = 2 ** (attempt - 1) * multiplier - result = max(0, min(exp, max_wait)) - return max(min_wait, result) - except (ValueError, ArithmeticError): - return max_wait - async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]): """Handles WebSocket message receiving with automatic retry logic. @@ -104,7 +84,7 @@ class WebsocketService(ABC): try: if await self._reconnect_websocket(retry_count): retry_count = 0 # Reset counter on successful reconnection - wait_time = self._calculate_wait_time(retry_count) + wait_time = exponential_backoff_time(retry_count) await asyncio.sleep(wait_time) except Exception as reconnect_error: logger.error(f"{self} reconnection failed: {reconnect_error}") diff --git a/src/pipecat/utils/network.py b/src/pipecat/utils/network.py new file mode 100644 index 000000000..27bec990c --- /dev/null +++ b/src/pipecat/utils/network.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +def exponential_backoff_time( + attempt: int, min_wait: float = 4, max_wait: float = 10, multiplier: float = 1 +) -> float: + """Calculate exponential backoff wait time. + + Args: + attempt: Current attempt number (1-based) + min_wait: Minimum wait time in seconds + max_wait: Maximum wait time in seconds + multiplier: Base multiplier for exponential calculation + + Returns: + Wait time in seconds + """ + try: + exp = 2 ** (attempt - 1) * multiplier + result = max(0, min(exp, max_wait)) + return max(min_wait, result) + except (ValueError, ArithmeticError): + return max_wait diff --git a/tests/test_utils_network.py b/tests/test_utils_network.py new file mode 100644 index 000000000..5b8d96939 --- /dev/null +++ b/tests/test_utils_network.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.utils.network import exponential_backoff_time + + +class TestUtilsNetwork(unittest.IsolatedAsyncioTestCase): + async def test_exponential_backoff_time(self): + # min_wait=4, max_wait=10, multiplier=1 + assert exponential_backoff_time(attempt=1, min_wait=4, max_wait=10, multiplier=1) == 4 + assert exponential_backoff_time(attempt=2, min_wait=4, max_wait=10, multiplier=1) == 4 + assert exponential_backoff_time(attempt=3, min_wait=4, max_wait=10, multiplier=1) == 4 + assert exponential_backoff_time(attempt=4, min_wait=4, max_wait=10, multiplier=1) == 8 + assert exponential_backoff_time(attempt=5, min_wait=4, max_wait=10, multiplier=1) == 10 + assert exponential_backoff_time(attempt=6, min_wait=4, max_wait=10, multiplier=1) == 10 + # min_wait=1, max_wait=10, multiplier=1 + assert exponential_backoff_time(attempt=1, min_wait=1, max_wait=10, multiplier=1) == 1 + assert exponential_backoff_time(attempt=2, min_wait=1, max_wait=10, multiplier=1) == 2 + assert exponential_backoff_time(attempt=3, min_wait=1, max_wait=10, multiplier=1) == 4 + assert exponential_backoff_time(attempt=4, min_wait=1, max_wait=10, multiplier=1) == 8 + assert exponential_backoff_time(attempt=5, min_wait=1, max_wait=10, multiplier=1) == 10 + assert exponential_backoff_time(attempt=6, min_wait=1, max_wait=10, multiplier=1) == 10 + # min_wait=1, max_wait=20, multiplier=2 + assert exponential_backoff_time(attempt=1, min_wait=1, max_wait=20, multiplier=2) == 2 + assert exponential_backoff_time(attempt=2, min_wait=1, max_wait=20, multiplier=2) == 4 + assert exponential_backoff_time(attempt=3, min_wait=1, max_wait=20, multiplier=2) == 8 + assert exponential_backoff_time(attempt=4, min_wait=1, max_wait=20, multiplier=2) == 16 + assert exponential_backoff_time(attempt=5, min_wait=1, max_wait=20, multiplier=2) == 20 + assert exponential_backoff_time(attempt=6, min_wait=1, max_wait=20, multiplier=2) == 20 From 4edda718ed8005178cfe7c36f35b8a6b98d9862d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Feb 2025 13:02:11 -0800 Subject: [PATCH 167/427] deepgram: add ability to provide custom addons --- CHANGELOG.md | 2 ++ src/pipecat/services/deepgram.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6be017e5a..973a57c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `addons` argument to `DeepgramSTTService`. + - Added `exponential_backoff_time()` to `utils.network` module. ### Fixed diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 43782e4eb..59305b1c9 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -5,7 +5,7 @@ # import asyncio -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, Dict, Optional from loguru import logger @@ -121,6 +121,7 @@ class DeepgramSTTService(STTService): url: str = "", sample_rate: Optional[int] = None, live_options: Optional[LiveOptions] = None, + addons: Optional[Dict] = None, **kwargs, ): super().__init__(sample_rate=sample_rate, **kwargs) @@ -148,6 +149,7 @@ class DeepgramSTTService(STTService): merged_options.language = merged_options.language.value self._settings = merged_options.to_dict() + self._addons = addons self._client = DeepgramClient( api_key, @@ -218,7 +220,7 @@ class DeepgramSTTService(STTService): self._on_utterance_end, ) - if not await self._connection.start(self._settings): + if not await self._connection.start(options=self._settings, addons=self._addons): logger.error(f"{self}: unable to connect to Deepgram") async def _disconnect(self): From 1b0bcebef66d012702145440943ef79c05698ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Feb 2025 09:56:09 -0800 Subject: [PATCH 168/427] deepgram: use the new nova-3 model as default --- CHANGELOG.md | 10 ++++++++++ src/pipecat/services/deepgram.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 973a57c33..6b72d15b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `exponential_backoff_time()` to `utils.network` module. +### Changed + +- `DeepgramSTTService` now uses the new `nova-3` model by default. If you want + to use the previous model you can pass `LiveOptions(model="nova-2-general")`. + (see https://deepgram.com/learn/introducing-nova-3-speech-to-text-api) + +```python +stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) +``` + ### Fixed - Fixed an issue that would cause `DeepgramSTTService` to stop working after an diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 59305b1c9..010c2c626 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -129,7 +129,7 @@ class DeepgramSTTService(STTService): default_options = LiveOptions( encoding="linear16", language=Language.EN, - model="nova-2-general", + model="nova-3-general", channels=1, interim_results=True, smart_format=True, From 84ac88cad762073f3132d7beabef04293c172451 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Feb 2025 18:03:37 -0800 Subject: [PATCH 169/427] STTMuteFilter: change suppressed logging to trace --- src/pipecat/processors/filters/stt_mute_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index 937f3b075..3c1ae53af 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -193,7 +193,7 @@ class STTMuteFilter(FrameProcessor): if not self.is_muted: await self.push_frame(frame, direction) else: - logger.debug(f"{frame.__class__.__name__} suppressed - STT currently muted") + logger.trace(f"{frame.__class__.__name__} suppressed - STT currently muted") else: # Pass all other frames through await self.push_frame(frame, direction) From 80f8e05fcf31db14503176bddb27d10c33aa4096 Mon Sep 17 00:00:00 2001 From: James Hush Date: Wed, 19 Feb 2025 16:07:22 +0800 Subject: [PATCH 170/427] docs: fix transcripts in translation chatbot example (#1199) --- examples/translation-chatbot/bot.py | 23 +++++++++++++------ examples/translation-chatbot/requirements.txt | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index d35cf7af2..8d4ee851e 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -24,17 +24,15 @@ from pipecat.frames.frames import ( ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineTask +from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIProcessor from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import ( - DailyParams, - DailyTransport, -) +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -166,21 +164,32 @@ async def main(): async def on_transcript_update(processor, frame): await transcript_handler.on_transcript_update(processor, frame) + rtvi = RTVIProcessor() + pipeline = Pipeline( [ transport.input(), + rtvi, stt, transcript.user(), # User transcripts tp, llm, tts, transport.output(), + transcript.assistant(), context_aggregator.assistant(), - transcript.assistant(), # Assistant transcripts ] ) - task = PipelineTask(pipeline) + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=False, # We don't want to interrupt the translator bot + enable_metrics=True, + enable_usage_metrics=True, + observers=[RTVIObserver(rtvi)], + ), + ) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/translation-chatbot/requirements.txt b/examples/translation-chatbot/requirements.txt index 6f05d3425..da1cbf0af 100644 --- a/examples/translation-chatbot/requirements.txt +++ b/examples/translation-chatbot/requirements.txt @@ -1,4 +1,4 @@ python-dotenv fastapi[all] -pipecat-ai[daily,openai,azure] +pipecat-ai[cartesia,daily,deepgram,openai,silero] aiohttp From 83f1a8830d48fee659c1864561a6a6d1b4fd481a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 09:29:53 -0800 Subject: [PATCH 171/427] daily: add room_url property --- CHANGELOG.md | 2 ++ src/pipecat/transports/services/daily.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b72d15b9..0ff842b1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `room_url` property to `DailyTransport`. + - Added `addons` argument to `DeepgramSTTService`. - Added `exponential_backoff_time()` to `utils.network` module. diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 2274b3e8f..7ce329659 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -329,6 +329,10 @@ class DailyTransportClient(EventHandler): def _speaker_name(self): return f"speaker-{self}" + @property + def room_url(self) -> str: + return self._room_url + @property def participant_id(self) -> str: return self._participant_id @@ -1112,6 +1116,10 @@ class DailyTransport(BaseTransport): # DailyTransport # + @property + def room_url(self) -> str: + return self._client.room_url + @property def participant_id(self) -> str: return self._client.participant_id From 7e7926059c08c1a75ff69bfebfb05da81e4beb42 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 19 Feb 2025 18:04:20 -0300 Subject: [PATCH 172/427] Fixed an issue that `start_callback` was not invoked for some LLM services. --- CHANGELOG.md | 2 ++ examples/foundational/14e-function-calling-gemini.py | 9 ++++++++- src/pipecat/services/ai_services.py | 1 + src/pipecat/services/openai.py | 1 - 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ff842b1f..0fe52e1da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an issue that `start_callback` was not invoked for some LLM services. + - Fixed an issue that would cause `DeepgramSTTService` to stop working after an error occurred (e.g. sudden network loss). If the network recovered we would not reconnect. diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index a05ad7f75..ad5d82b6a 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -14,6 +14,7 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -30,6 +31,12 @@ logger.add(sys.stderr, level="DEBUG") video_participant_id = None +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") @@ -63,7 +70,7 @@ async def main(): ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm.register_function("get_weather", get_weather) + llm.register_function("get_weather", get_weather, start_fetch_weather) llm.register_function("get_image", get_image) tools = [ diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index c5adc89e0..6b739163a 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -175,6 +175,7 @@ class LLMService(AIService): f = self._callbacks[None] else: return None + await self.call_start_function(context, function_name) await context.call_function( f, function_name=function_name, diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 3114b6920..15c1a197a 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -266,7 +266,6 @@ class BaseOpenAILLMService(LLMService): if tool_call.function and tool_call.function.name: function_name += tool_call.function.name tool_call_id = tool_call.id - await self.call_start_function(context, function_name) if tool_call.function and tool_call.function.arguments: # Keep iterating through the response to collect all the argument fragments arguments += tool_call.function.arguments From 293677588deb2c289aa6de460a3157b211f6e023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 21:39:00 -0800 Subject: [PATCH 173/427] tts: make push_stop_frames default to 2.0s --- src/pipecat/services/ai_services.py | 2 +- src/pipecat/services/elevenlabs.py | 1 - src/pipecat/services/rime.py | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 6b739163a..05c637126 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -209,7 +209,7 @@ class TTSService(AIService): # if True, TTSService will push TTSStoppedFrames, otherwise subclass must do it push_stop_frames: bool = False, # if push_stop_frames is True, wait for this idle period before pushing TTSStoppedFrame - stop_frame_timeout_s: float = 1.0, + stop_frame_timeout_s: float = 2.0, # if True, TTSService will push silence audio frames after TTSStoppedFrame push_silence_after_stop: bool = False, # if push_silence_after_stop is True, send this amount of audio silence diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 3c8cbb3ed..5c312e7fe 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -191,7 +191,6 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): aggregate_sentences=True, push_text_frames=False, push_stop_frames=True, - stop_frame_timeout_s=2.0, pause_frame_processing=True, sample_rate=sample_rate, **kwargs, diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index f87405a63..e12844a96 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -14,16 +14,13 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, Frame, - LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, TTSAudioRawFrame, - TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, ) @@ -100,7 +97,6 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): aggregate_sentences=True, push_text_frames=False, push_stop_frames=True, - stop_frame_timeout_s=2.0, pause_frame_processing=True, sample_rate=sample_rate, **kwargs, From 6e3f96aa834394dbcb5850ae7e32edd84362618e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 21:39:36 -0800 Subject: [PATCH 174/427] fish: automatically send TTSStoppedFrame after timeout --- CHANGELOG.md | 3 +++ src/pipecat/services/fish.py | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe52e1da..5e5137f07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed a `FishAudioTTSService` issue where `TTSStoppedFrame` was not being + pushed. + - Fixed an issue that `start_callback` was not invoked for some LLM services. - Fixed an issue that would cause `DeepgramSTTService` to stop working after an diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 94475aca7..e2a75bdb2 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -11,16 +11,13 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, Frame, - LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, TTSAudioRawFrame, - TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, ) @@ -60,7 +57,12 @@ class FishAudioTTSService(TTSService, WebsocketService): params: InputParams = InputParams(), **kwargs, ): - super().__init__(pause_frame_processing=True, sample_rate=sample_rate, **kwargs) + super().__init__( + push_stop_frames=True, + pause_frame_processing=True, + sample_rate=sample_rate, + **kwargs, + ) self._api_key = api_key self._base_url = "wss://api.fish.audio/v1/tts/live" From d0f67fc189e42094d07ff1e5b680cff013aa2366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 21:43:29 -0800 Subject: [PATCH 175/427] BaseInputTransport: remove VAD logging These logs are very verbose. They were added to try to find an issue that resulted in being because of low CPU/memory resources, but these logs were not helpful to determine that. --- src/pipecat/transports/base_input.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 42eb162da..ee7ae7f7c 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -174,11 +174,9 @@ class BaseInputTransport(FrameProcessor): async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState: state = VADState.QUIET if self.vad_analyzer: - logger.trace(f"{self}: analyzing VAD on {audio_frame}") state = await self.get_event_loop().run_in_executor( self._executor, self.vad_analyzer.analyze_audio, audio_frame.audio ) - logger.trace(f"{self}: done analyzing VAD on {audio_frame}") return state async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState): From 039d144c797501adc6472eb599b0c00cbd58935d Mon Sep 17 00:00:00 2001 From: Dominic Stewart <45786774+DominicStewart@users.noreply.github.com> Date: Wed, 19 Feb 2025 22:03:37 -0800 Subject: [PATCH 176/427] examples(phone-bot): updated example to use Gemini (#1233) --- examples/phone-chatbot/README.md | 17 ++ examples/phone-chatbot/bot_daily.py | 5 +- examples/phone-chatbot/bot_daily_gemini.py | 234 +++++++++++++++++++++ examples/phone-chatbot/bot_runner.py | 37 ++++ 4 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 examples/phone-chatbot/bot_daily_gemini.py diff --git a/examples/phone-chatbot/README.md b/examples/phone-chatbot/README.md index f207ebd9d..a8a1cee8e 100644 --- a/examples/phone-chatbot/README.md +++ b/examples/phone-chatbot/README.md @@ -1,3 +1,5 @@ + +
 pipecat
@@ -104,6 +106,21 @@ curl -X POST "http://localhost:7860/daily_start_bot" \ -d '{"dialoutNumber": "+18057145330", "detectVoicemail": true}' ``` +### New! Using Gemini with Daily + +We have introduced a new example file that uses Gemini. You can find the code within bot_daily_gemini.py. +If you want to spin up a Gemini-based bot for this demo, instead of an OpenAI-based bot, call the same properties above but on the `daily_gemini_start_bot` endpoint instead. + +For example: + +```shell +curl -X POST "http://localhost:7860/daily_gemini_start_bot" \ py pipecat + -H "Content-Type: application/json" \ + -d '{"detectVoicemail": true}' +``` + +Any request body properties supported by `/daily_start_bot` (such as "detectVoicemail", "dialoutnumber", etc) can also be passed to `/daily_gemini_start_bot`. The only difference is that calling the Gemini endpoint will start a Gemini bot session. + ### More information For more configuration options, please consult [Daily's API documentation](https://docs.daily.co). diff --git a/examples/phone-chatbot/bot_daily.py b/examples/phone-chatbot/bot_daily.py index cacd17e5f..0ba631296 100644 --- a/examples/phone-chatbot/bot_daily.py +++ b/examples/phone-chatbot/bot_daily.py @@ -98,6 +98,7 @@ async def main( - **"Record your message after the tone."** - **Any phrase that suggests an answering machine or voicemail.** - **ASSUME IT IS A VOICEMAIL. DO NOT WAIT FOR MORE CONFIRMATION.** + - **IF THE CALL SAYS "PLEASE LEAVE A MESSAGE AFTER THE BEEP", WAIT FOR THE BEEP BEFORE LEAVING A MESSAGE.** #### **Step 2: Leave a Voicemail Message** - Immediately say: @@ -110,7 +111,9 @@ async def main( - If the call is answered by a human, say: *"Oh, hello! I'm a friendly chatbot. Is there anything I can help you with?"* - Keep responses **brief and helpful**. - - If the user no longer needs assistance, **call `terminate_call` immediately.** + - If the user no longer needs assistance, say: + *"Okay, thank you! Have a great day!"* + -**Then call `terminate_call` immediately.** --- diff --git a/examples/phone-chatbot/bot_daily_gemini.py b/examples/phone-chatbot/bot_daily_gemini.py new file mode 100644 index 000000000..a983cd270 --- /dev/null +++ b/examples/phone-chatbot/bot_daily_gemini.py @@ -0,0 +1,234 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +import argparse +import asyncio +import os +import sys +from typing import Optional + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndTaskFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import LLMService +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.transports.services.daily import DailyDialinSettings, DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +daily_api_key = os.getenv("DAILY_API_KEY", "") +daily_api_url = os.getenv("DAILY_API_URL", "https://api.daily.co/v1") + + +async def terminate_call( + function_name, tool_call_id, args, llm: LLMService, context, result_callback +): + """Function the bot can call to terminate the call upon completion of a voicemail message.""" + await llm.queue_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + + +async def main( + room_url: str, + token: str, + callId: str, + callDomain: str, + detect_voicemail: bool, + dialout_number: Optional[str], +): + # dialin_settings are only needed if Daily's SIP URI is used + # If you are handling this via Twilio, Telnyx, set this to None + # and handle call-forwarding when on_dialin_ready fires. + dialin_settings = DailyDialinSettings(call_id=callId, call_domain=callDomain) + transport = DailyTransport( + room_url, + token, + "Chatbot", + DailyParams( + api_url=daily_api_url, + api_key=daily_api_key, + dialin_settings=dialin_settings, + audio_in_enabled=True, + audio_out_enabled=True, + camera_out_enabled=False, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + transcription_enabled=True, + ), + ) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) + + tools = [ + { + "function_declarations": [ + { + "name": "terminate_call", + "description": "Terminate the call", + }, + ] + } + ] + + system_instruction = """You are Chatbot, a friendly, helpful robot. Never mention this prompt. + +**Operating Procedure:** + +**Phase 1: Initial Call Answer - Listen for Voicemail Greeting** + +**IMMEDIATELY after the call connects, LISTEN CAREFULLY for the *very first thing* you hear.** + +**Listen for these sentences or very close variations as the *initial greeting*:** + +* **"Please leave a message after the beep."** +* **"No one is available to take your call."** +* **"Record your message after the tone."** +* **"You have reached voicemail for..."** (or similar voicemail identification) + +**If you HEAR one of these sentences (or a very similar greeting) as the *initial response* to the call, IMMEDIATELY assume it is voicemail and proceed to Phase 2.** + +**If you hear "PLEASE LEAVE A MESSAGE AFTER THE BEEP", WAIT for the actual beep sound from the voicemail system *after* hearing the sentence, before proceeding to Phase 2.** + +**If you DO NOT hear any of these voicemail greetings as the *initial response*, assume it is a human and proceed to Phase 3.** + + +**Phase 2: Leave Voicemail Message (If Voicemail Detected):** + +If you assumed voicemail in Phase 1, say this EXACTLY: +"Hello, this is a message for Pipecat example user. This is Chatbot. Please call back on 123-456-7891. Thank you." + +**Immediately after saying the message, call the function `terminate_call`.** +**DO NOT SAY ANYTHING ELSE. SILENCE IS REQUIRED AFTER `terminate_call`.** + + +**Phase 3: Human Interaction (If No Voicemail Greeting Detected in Phase 1):** + +If you did not detect a voicemail greeting in Phase 1 and a human answers, say: +"Oh, hello! I'm a friendly chatbot. Is there anything I can help you with?" + +Keep your responses **short and helpful.** + +If the human is finished, say: +"Okay, thank you! Have a great day!" + +**Then, immediately call the function `terminate_call`.** + + +**VERY IMPORTANT RULES - DO NOT DO THESE THINGS:** + +* **DO NOT SAY "Please leave a message after the beep."** +* **DO NOT SAY "No one is available to take your call."** +* **DO NOT SAY "Record your message after the tone."** +* **DO NOT SAY ANY voicemail greeting yourself.** +* **Only check for voicemail greetings in Phase 1, *immediately after the call connects*.** +* **After voicemail or human interaction, ALWAYS call `terminate_call` immediately.** +* **Do not speak after calling `terminate_call`.** +* Your speech will be audio, so use simple language without special characters. +""" + + llm = GoogleLLMService( + model="models/gemini-2.0-flash-exp", + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=system_instruction, + tools=tools, + ) + llm.register_function("terminate_call", terminate_call) + + context = GoogleLLMContext() + + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams(allow_interruptions=True), + ) + + if dialout_number: + logger.debug("dialout number detected; doing dialout") + + # Configure some handlers for dialing out + @transport.event_handler("on_joined") + async def on_joined(transport, data): + logger.debug(f"Joined; starting dialout to: {dialout_number}") + await transport.start_dialout({"phoneNumber": dialout_number}) + + @transport.event_handler("on_dialout_connected") + async def on_dialout_connected(transport, data): + logger.debug(f"Dial-out connected: {data}") + + @transport.event_handler("on_dialout_answered") + async def on_dialout_answered(transport, data): + logger.debug(f"Dial-out answered: {data}") + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # unlike the dialin case, for the dialout case, the caller will speak first. Presumably + # they will answer the phone and say "Hello?" Since we've captured their transcript, + # That will put a frame into the pipeline and prompt an LLM completion, which is how the + # bot will then greet the user. + elif detect_voicemail: + logger.debug("Detect voicemail example. You can test this in example in Daily Prebuilt") + + # For the voicemail detection case, we do not want the bot to answer the phone. We want it to wait for the voicemail + # machine to say something like 'Leave a message after the beep', or for the user to say 'Hello?'. + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + else: + logger.debug("no dialout number; assuming dialin") + + # Different handlers for dialin + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # For the dialin case, we want the bot to answer the phone and greet the user. We + # can prompt the bot to speak by putting the context into the pipeline. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Pipecat Simple ChatBot") + parser.add_argument("-u", type=str, help="Room URL") + parser.add_argument("-t", type=str, help="Token") + parser.add_argument("-i", type=str, help="Call ID") + parser.add_argument("-d", type=str, help="Call Domain") + parser.add_argument("-v", action="store_true", help="Detect voicemail") + parser.add_argument("-o", type=str, help="Dialout number", default=None) + config = parser.parse_args() + + asyncio.run(main(config.u, config.t, config.i, config.d, config.v, config.o)) diff --git a/examples/phone-chatbot/bot_runner.py b/examples/phone-chatbot/bot_runner.py index 03962b8b1..da64805e3 100644 --- a/examples/phone-chatbot/bot_runner.py +++ b/examples/phone-chatbot/bot_runner.py @@ -110,10 +110,15 @@ async def _create_daily_room( # Spawn a new agent, and join the user session # Note: this is mostly for demonstration purposes (refer to 'deployment' in docs) + print(f"Vendor: {vendor}") if vendor == "daily": bot_proc = f"python3 -m bot_daily -u {room.url} -t {token} -i {callId} -d {callDomain}{' -v' if detect_voicemail else ''}" if dialoutNumber: bot_proc += f" -o {dialoutNumber}" + elif vendor == "daily-gemini": + bot_proc = f"python3 -m bot_daily_gemini -u {room.url} -t {token} -i {callId} -d {callDomain}{' -v' if detect_voicemail else ''}" + if dialoutNumber: + bot_proc += f" -o {dialoutNumber}" else: bot_proc = f"python3 -m bot_twilio -u {room.url} -t {token} -i {callId} -s {room.config.sip_endpoint}" @@ -201,6 +206,38 @@ async def daily_start_bot(request: Request) -> JSONResponse: return JSONResponse({"room_url": room.url, "sipUri": room.config.sip_endpoint}) +@app.post("/daily_gemini_start_bot") +async def daily_gemini_start_bot(request: Request) -> JSONResponse: + # The /daily_start_bot is invoked when a call is received on Daily's SIP URI + # daily_start_bot will create the room, put the call on hold until + # the bot and sip worker are ready. Daily will automatically + # forward the call to the SIP URi when dialin_ready fires. + + # Use specified room URL, or create a new one if not specified + room_url = os.getenv("DAILY_SAMPLE_ROOM_URL", None) + # Get the dial-in properties from the request + try: + data = await request.json() + if "test" in data: + # Pass through any webhook checks + return JSONResponse({"test": True}) + detect_voicemail = data.get("detectVoicemail", False) + callId = data.get("callId", None) + callDomain = data.get("callDomain", None) + dialoutNumber = data.get("dialoutNumber", None) + except Exception: + raise HTTPException( + status_code=500, detail="Missing properties 'callId', 'callDomain', or 'dialoutNumber'" + ) + + room: DailyRoomObject = await _create_daily_room( + room_url, callId, callDomain, dialoutNumber, "daily-gemini", detect_voicemail + ) + + # Grab a token for the user to join with + return JSONResponse({"room_url": room.url, "sipUri": room.config.sip_endpoint}) + + # ----------------- Main ----------------- # From 98259af54eb6ec2fe6e9ccf0204ac130b5eabc02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 22:05:48 -0800 Subject: [PATCH 177/427] update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe52e1da..c057a449a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,10 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Fixed a `STTMuteFilter` issue that would not mute user audio frames causing transcriptions to be generated by the STT service. +### Other + +- Added Gemini support to `examples/phone-chatbot`. + ## [0.0.57] - 2025-02-14 ### Added From 680925496371506ed13d2f85077ea4dbbf9528ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 22:17:26 -0800 Subject: [PATCH 178/427] tts: fix metrics and TTSStoppedFrame frame in HTTP services Fixes #1247 --- CHANGELOG.md | 3 +++ src/pipecat/services/cartesia.py | 18 ++++++++++-------- src/pipecat/services/elevenlabs.py | 6 ++---- src/pipecat/services/playht.py | 5 ++++- src/pipecat/services/rime.py | 12 +++++------- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b816d4ecd..51d0365c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an issue that was causing HTTP TTS services to push `TTSStoppedFrame` + more than once. + - Fixed a `FishAudioTTSService` issue where `TTSStoppedFrame` was not being pushed. diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 67fc67ebf..f9f2bdbe6 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -364,9 +364,6 @@ class CartesiaHttpTTSService(TTSService): async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating TTS: [{text}]") - await self.start_ttfb_metrics() - yield TTSStartedFrame() - try: voice_controls = None if self._settings["speed"] or self._settings["emotion"]: @@ -376,6 +373,8 @@ class CartesiaHttpTTSService(TTSService): if self._settings["emotion"]: voice_controls["emotion"] = self._settings["emotion"] + await self.start_ttfb_metrics() + output = await self._client.tts.sse( model_id=self._model_name, transcript=text, @@ -386,14 +385,17 @@ class CartesiaHttpTTSService(TTSService): _experimental_voice_controls=voice_controls, ) + await self.start_tts_usage_metrics(text) + + yield TTSStartedFrame() + frame = TTSAudioRawFrame( audio=output["audio"], sample_rate=self.sample_rate, num_channels=1 ) + yield frame except Exception as e: logger.error(f"{self} exception: {e}") - - await self.start_tts_usage_metrics(text) - - await self.stop_ttfb_metrics() - yield TTSStoppedFrame() + finally: + await self.stop_ttfb_metrics() + yield TTSStoppedFrame() diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 5c312e7fe..58ec91342 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -566,18 +566,16 @@ class ElevenLabsHttpTTSService(TTSService): return await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() async for chunk in response.content: if chunk: await self.stop_ttfb_metrics() yield TTSAudioRawFrame(chunk, self.sample_rate, 1) - - yield TTSStoppedFrame() - except Exception as e: logger.error(f"Error in run_tts: {e}") yield ErrorFrame(error=str(e)) - finally: + await self.stop_ttfb_metrics() yield TTSStoppedFrame() diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 297e802d3..7f0b29c87 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -397,6 +397,7 @@ class PlayHTHttpTTSService(TTSService): await self.start_tts_usage_metrics(text) yield TTSStartedFrame() + async for chunk in playht_gen: # skip the RIFF header. if in_header: @@ -416,6 +417,8 @@ class PlayHTHttpTTSService(TTSService): await self.stop_ttfb_metrics() frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) yield frame - yield TTSStoppedFrame() except Exception as e: logger.error(f"{self} error generating TTS: {e}") + finally: + await self.stop_ttfb_metrics() + yield TTSStoppedFrame() diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index e12844a96..f920959d8 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -387,9 +387,6 @@ class RimeHttpTTSService(TTSService): try: await self.start_ttfb_metrics() - await self.start_tts_usage_metrics(text) - - yield TTSStartedFrame() async with aiohttp.ClientSession() as session: async with session.post(self._base_url, json=payload, headers=headers) as response: @@ -399,6 +396,10 @@ class RimeHttpTTSService(TTSService): yield ErrorFrame(error=error_message) return + await self.start_tts_usage_metrics(text) + + yield TTSStartedFrame() + # Process the streaming response chunk_size = 8192 first_chunk = True @@ -411,12 +412,9 @@ class RimeHttpTTSService(TTSService): if chunk: frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) yield frame - - yield TTSStoppedFrame() - except Exception as e: logger.exception(f"Error generating TTS: {e}") yield ErrorFrame(error=f"Rime TTS error: {str(e)}") - finally: + await self.stop_ttfb_metrics() yield TTSStoppedFrame() From 69e6f3fdb72204057163a227364b527bfe367241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 22:21:02 -0800 Subject: [PATCH 179/427] rime: pass aiohttp session to constructor --- CHANGELOG.md | 5 +++++ src/pipecat/services/rime.py | 41 ++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51d0365c7..dc732c569 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `RimeHttpTTSService` needs an `aiohttp.ClientSession` to be passed to the + constructor as all the other HTTP-based services. + +- `RimeHttpTTSService` doesn't use a default voice anymore. + - `DeepgramSTTService` now uses the new `nova-3` model by default. If you want to use the previous model you can pass `LiveOptions(model="nova-2-general")`. (see https://deepgram.com/learn/introducing-nova-3-speech-to-text-api) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index f920959d8..cc59b132d 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -345,7 +345,8 @@ class RimeHttpTTSService(TTSService): self, *, api_key: str, - voice_id: str = "eva", + voice_id: str, + aiohttp_session: aiohttp.ClientSession, model: str = "mistv2", sample_rate: Optional[int] = None, params: InputParams = InputParams(), @@ -354,6 +355,7 @@ class RimeHttpTTSService(TTSService): super().__init__(sample_rate=sample_rate, **kwargs) self._api_key = api_key + self._session = aiohttp_session self._base_url = "https://users.rime.ai/v1/rime-tts" self._settings = { "speedAlpha": params.speed_alpha, @@ -388,30 +390,27 @@ class RimeHttpTTSService(TTSService): try: await self.start_ttfb_metrics() - async with aiohttp.ClientSession() as session: - async with session.post(self._base_url, json=payload, headers=headers) as response: - if response.status != 200: - error_message = f"Rime TTS error: HTTP {response.status}" - logger.error(error_message) - yield ErrorFrame(error=error_message) - return + async with self._session.post( + self._base_url, json=payload, headers=headers + ) as response: + if response.status != 200: + error_message = f"Rime TTS error: HTTP {response.status}" + logger.error(error_message) + yield ErrorFrame(error=error_message) + return - await self.start_tts_usage_metrics(text) + await self.start_tts_usage_metrics(text) - yield TTSStartedFrame() + yield TTSStartedFrame() - # Process the streaming response - chunk_size = 8192 - first_chunk = True + # Process the streaming response + chunk_size = 8192 - async for chunk in response.content.iter_chunked(chunk_size): - if first_chunk: - await self.stop_ttfb_metrics() - first_chunk = False - - if chunk: - frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) - yield frame + async for chunk in response.content.iter_chunked(chunk_size): + if chunk: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) + yield frame except Exception as e: logger.exception(f"Error generating TTS: {e}") yield ErrorFrame(error=f"Rime TTS error: {str(e)}") From df57202a051fb99c5ef92adac6734cd6f4a56f8c Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Feb 2025 13:57:18 -0500 Subject: [PATCH 180/427] Since the `openai` package is used by pretty much everything in pipecat (due to `OpenAILLMContext` being the standard context representation), let's make it a non-optional dependency. This change solves an issue faced by users who aren't intending to use OpenAI getting scary error messages saying that they need the `openai` optional dependency "in order to use OpenAI", along with an instruction to set the OPENAI_API_KEY environment variable. Note that with this change we could theoretically remove from pyproject.toml a number of defined optional dependencies that list only the `openai` package as a dependency (like `deepseek`, for example), but I didn't want to "break the API" in terms of how users install/consume pipecat and its set of built-in services. Finally, I removed the `python-deepcompare` dependency from the `openai` optional dependency, since it appears to me like it was added by mistake (my guess is it was used for debugging during development and then never removed). --- pyproject.toml | 5 ++-- .../aggregators/openai_llm_context.py | 20 +++++---------- src/pipecat/services/base_whisper.py | 12 ++------- src/pipecat/services/openai.py | 25 ++++++------------- .../services/openai_realtime_beta/openai.py | 10 +++++++- 5 files changed, 28 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c5d6262b..c5b1ba28a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,8 @@ dependencies = [ "pydantic~=2.10.5", "pyloudnorm~=0.1.1", "resampy~=0.4.3", - "soxr~=0.5.0" + "soxr~=0.5.0", + "openai~=1.59.6" ] [project.urls] @@ -69,7 +70,7 @@ local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.13", "transformers~=4.48.0" ] nim = [ "openai~=1.59.6" ] noisereduce = [ "noisereduce~=3.0.3" ] -openai = [ "openai~=1.59.6", "websockets~=13.1", "python-deepcompare~=2.1.0" ] +openai = [ "openai~=1.59.6", "websockets~=13.1" ] openpipe = [ "openpipe~=4.45.0" ] perplexity = [ "openai~=1.59.6" ] playht = [ "pyht~=0.1.6", "websockets~=13.1" ] diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index cc14c5ac7..5ef8c090f 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -12,6 +12,12 @@ from dataclasses import dataclass from typing import Any, Awaitable, Callable, List, Optional from loguru import logger +from openai._types import NOT_GIVEN, NotGiven +from openai.types.chat import ( + ChatCompletionMessageParam, + ChatCompletionToolChoiceOptionParam, + ChatCompletionToolParam, +) from PIL import Image from pipecat.frames.frames import ( @@ -22,20 +28,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -try: - from openai._types import NOT_GIVEN, NotGiven - from openai.types.chat import ( - ChatCompletionMessageParam, - ChatCompletionToolChoiceOptionParam, - ChatCompletionToolParam, - ) -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - # JSON custom encoder to handle bytes arrays so that we can log contexts # with images to the console. diff --git a/src/pipecat/services/base_whisper.py b/src/pipecat/services/base_whisper.py index eae38be64..4de62f08c 100644 --- a/src/pipecat/services/base_whisper.py +++ b/src/pipecat/services/base_whisper.py @@ -7,22 +7,14 @@ from typing import AsyncGenerator, Optional from loguru import logger +from openai import AsyncOpenAI +from openai.types.audio import Transcription from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame from pipecat.services.ai_services import SegmentedSTTService from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 -try: - from openai import AsyncOpenAI - from openai.types.audio import Transcription -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - def language_to_whisper_language(language: Language) -> Optional[str]: """Language support for Whisper API. diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 15c1a197a..9c7854d51 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -13,6 +13,14 @@ from typing import Any, AsyncGenerator, Dict, List, Literal, Optional import aiohttp import httpx from loguru import logger +from openai import ( + NOT_GIVEN, + AsyncOpenAI, + AsyncStream, + BadRequestError, + DefaultAsyncHttpxClient, +) +from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from PIL import Image from pydantic import BaseModel, Field @@ -57,23 +65,6 @@ from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription from pipecat.transcriptions.language import Language from pipecat.utils.time import time_now_iso8601 -try: - from openai import ( - NOT_GIVEN, - AsyncOpenAI, - AsyncStream, - BadRequestError, - DefaultAsyncHttpxClient, - ) - from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - - ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] VALID_VOICES: Dict[str, ValidVoice] = { diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index bc173d765..ff6f24c66 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -10,9 +10,17 @@ import json import time from dataclasses import dataclass -import websockets from loguru import logger +try: + import websockets +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, From d04c4b36f3c37602e33cf5f402f22528412307f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Feb 2025 19:25:43 -0800 Subject: [PATCH 181/427] AssistantContextAggregator: append aggregation and tools in the same turn --- CHANGELOG.md | 3 +++ src/pipecat/processors/aggregators/llm_response.py | 3 +++ src/pipecat/services/anthropic.py | 9 ++++----- src/pipecat/services/google/google.py | 12 ++++++------ src/pipecat/services/grok.py | 8 ++++---- src/pipecat/services/openai.py | 8 ++++---- 6 files changed, 24 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc732c569..dd4ca7269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed a context aggregator issue that would not append the LLM text response + to the context if a function call happened in the same LLM turn. + - Fixed an issue that was causing HTTP TTS services to push `TTSStoppedFrame` more than once. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 2de3ebae4..6f1da5af4 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -145,6 +145,9 @@ class LLMResponseAggregator(BaseLLMResponseAggregator): frame = LLMMessagesFrame(self._messages) await self.push_frame(frame) + # Reset our accumulator state. + self.reset() + class LLMContextResponseAggregator(BaseLLMResponseAggregator): """This is a base LLM aggregator that uses an LLM context to store the diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 80235199b..a7c5fd5f0 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -743,18 +743,19 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) + if self._function_call_result: frame = self._function_call_result properties = frame.properties self._function_call_result = None if frame.result: assistant_message = {"role": "assistant", "content": []} - if aggregation: - assistant_message["content"].append({"type": "text", "text": aggregation}) assistant_message["content"].append( { "type": "tool_use", @@ -782,8 +783,6 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): else: # Default behavior run_llm = True - elif aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) if self._pending_image_frame_message: frame = self._pending_image_frame_message diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 0a8a86580..67a2b3954 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -565,10 +565,15 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message( + glm.Content(role="model", parts=[glm.Part(text=aggregation)]) + ) + if self._function_call_result: frame = self._function_call_result properties = frame.properties @@ -608,11 +613,6 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): else: # Default behavior is to run the LLM if there are no function calls in progress run_llm = not bool(self._function_calls_in_progress) - else: - if aggregation.strip(): - self._context.add_message( - glm.Content(role="model", parts=[glm.Part(text=aggregation)]) - ) if self._pending_image_frame_message: frame = self._pending_image_frame_message diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 064dd8829..0011a1f4e 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -37,10 +37,13 @@ class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) + if self._function_call_result: frame = self._function_call_result properties = frame.properties @@ -77,9 +80,6 @@ class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): # Default behavior is to run the LLM if there are no function calls in progress run_llm = not bool(self._function_calls_in_progress) - else: - self._context.add_message({"role": "assistant", "content": aggregation}) - if self._pending_image_frame_message: frame = self._pending_image_frame_message self._pending_image_frame_message = None diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 15c1a197a..4077f4354 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -631,10 +631,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): run_llm = False properties: Optional[FunctionCallResultProperties] = None - aggregation = self._aggregation + aggregation = self._aggregation.strip() self.reset() try: + if aggregation: + self._context.add_message({"role": "assistant", "content": aggregation}) + if self._function_call_result: frame = self._function_call_result properties = frame.properties @@ -669,9 +672,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): # Default behavior is to run the LLM if there are no function calls in progress run_llm = not bool(self._function_calls_in_progress) - else: - self._context.add_message({"role": "assistant", "content": aggregation}) - if self._pending_image_frame_message: frame = self._pending_image_frame_message self._pending_image_frame_message = None From 9c5fe5c85e8ef61d1d1e3b338b7868b530f9f1c9 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Fri, 21 Feb 2025 09:32:40 +0530 Subject: [PATCH 182/427] fixing deepgram mismatch --- CHANGELOG.md | 4 ++++ src/pipecat/services/deepgram.py | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd4ca7269..79c663e7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,10 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Fixed a `STTMuteFilter` issue that would not mute user audio frames causing transcriptions to be generated by the STT service. +- Fixes an issue in `DeepgramSTTService` where `sample_rate` passed to the + `LiveOptions` was not being used, causing the service to use the default + sample rate of pipeline. + ### Other - Added Gemini support to `examples/phone-chatbot`. diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 010c2c626..22879ab0e 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -124,6 +124,7 @@ class DeepgramSTTService(STTService): addons: Optional[Dict] = None, **kwargs, ): + sample_rate = sample_rate or (live_options.sample_rate if live_options else None) super().__init__(sample_rate=sample_rate, **kwargs) default_options = LiveOptions( From 01f083b7fcbe0bd29678925945072da944b5c1c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Feb 2025 23:33:06 -0800 Subject: [PATCH 183/427] transports: remove TransportParams.audio_out_is_live --- CHANGELOG.md | 6 +++++- examples/foundational/18-gstreamer-filesrc.py | 1 - src/pipecat/transports/base_transport.py | 2 -- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c663e7f..997591754 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ``` +### Removed + +- Remove `TransportParams.audio_out_is_live` since it was not being used at all. + ### Fixed - Fixed a context aggregator issue that would not append the LLM text response @@ -51,7 +55,7 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) transcriptions to be generated by the STT service. - Fixes an issue in `DeepgramSTTService` where `sample_rate` passed to the - `LiveOptions` was not being used, causing the service to use the default + `LiveOptions` was not being used, causing the service to use the default sample rate of pipeline. ### Other diff --git a/examples/foundational/18-gstreamer-filesrc.py b/examples/foundational/18-gstreamer-filesrc.py index b74846d53..12bec25bd 100644 --- a/examples/foundational/18-gstreamer-filesrc.py +++ b/examples/foundational/18-gstreamer-filesrc.py @@ -38,7 +38,6 @@ async def main(): "GStreamer", DailyParams( audio_out_enabled=True, - audio_out_is_live=True, camera_out_enabled=True, camera_out_width=1280, camera_out_height=720, diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index e07b41c22..c28579784 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import inspect from abc import ABC, abstractmethod from typing import Optional @@ -30,7 +29,6 @@ class TransportParams(BaseModel): camera_out_framerate: int = 30 camera_out_color_format: str = "RGB" audio_out_enabled: bool = False - audio_out_is_live: bool = False audio_out_sample_rate: Optional[int] = None audio_out_channels: int = 1 audio_out_bitrate: int = 96000 From 7c7b4c52af63339f2be4bf0bf8c267a8bff3149f Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 21 Feb 2025 09:11:58 -0300 Subject: [PATCH 184/427] Fixed an issue where EndTaskFrame was not triggering on_client_disconnected or closing the WebSocket in FastAPI. --- CHANGELOG.md | 2 + .../transports/network/fastapi_websocket.py | 120 +++++++++++++----- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c663e7f..6c9e2292f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an issue where `EndTaskFrame` was not triggering `on_client_disconnected` or closing the WebSocket in FastAPI. + - Fixed a context aggregator issue that would not append the LLM text response to the context if a function call happened in the same LLM turn. diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index 9c4c170f2..565d1906c 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -55,45 +55,89 @@ class FastAPIWebsocketCallbacks(BaseModel): on_session_timeout: Callable[[WebSocket], Awaitable[None]] +class FastAPIWebsocketClient: + def __init__(self, websocket: WebSocket, is_binary: bool, callbacks: FastAPIWebsocketCallbacks): + self._websocket = websocket + self._closing = False + self._is_binary = is_binary + self._callbacks = callbacks + + def receive(self) -> typing.AsyncIterator[bytes | str]: + return self._websocket.iter_bytes() if self._is_binary else self._websocket.iter_text() + + async def send(self, data: str | bytes): + if self._can_send(): + if self._is_binary: + await self._websocket.send_bytes(data) + else: + await self._websocket.send_text(data) + + async def disconnect(self): + if self.is_connected and not self.is_closing: + self._closing = True + await self._websocket.close() + await self.trigger_client_disconnected() + + async def trigger_client_disconnected(self): + await self._callbacks.on_client_disconnected(self._websocket) + + async def trigger_client_connected(self): + await self._callbacks.on_client_connected(self._websocket) + + async def trigger_client_timout(self): + await self._callbacks.on_session_timeout(self._websocket) + + def _can_send(self): + return self.is_connected and not self.is_closing + + @property + def is_connected(self) -> bool: + return self._websocket.client_state == WebSocketState.CONNECTED + + @property + def is_closing(self) -> bool: + return self._closing + + class FastAPIWebsocketInputTransport(BaseInputTransport): def __init__( self, - websocket: WebSocket, + client: FastAPIWebsocketClient, params: FastAPIWebsocketParams, - callbacks: FastAPIWebsocketCallbacks, **kwargs, ): super().__init__(params, **kwargs) - - self._websocket = websocket + self._client = client self._params = params - self._callbacks = callbacks + self._receive_task = None + self._monitor_websocket_task = None async def start(self, frame: StartFrame): await super().start(frame) await self._params.serializer.setup(frame) if self._params.session_timeout: self._monitor_websocket_task = self.create_task(self._monitor_websocket()) - await self._callbacks.on_client_connected(self._websocket) + await self._client.trigger_client_connected() self._receive_task = self.create_task(self._receive_messages()) + async def _stop_tasks(self): + if self._monitor_websocket_task: + await self.cancel_task(self._monitor_websocket_task) + await self.cancel_task(self._receive_task) + async def stop(self, frame: EndFrame): await super().stop(frame) - await self.cancel_task(self._receive_task) + await self._stop_tasks() + await self._client.disconnect() async def cancel(self, frame: CancelFrame): await super().cancel(frame) - await self.cancel_task(self._receive_task) - - def _iter_data(self) -> typing.AsyncIterator[bytes | str]: - if self._params.serializer.type == FrameSerializerType.BINARY: - return self._websocket.iter_bytes() - else: - return self._websocket.iter_text() + await self._stop_tasks() + await self._client.disconnect() async def _receive_messages(self): try: - async for message in self._iter_data(): + async for message in self._client.receive(): frame = await self._params.serializer.deserialize(message) if not frame: @@ -106,19 +150,23 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): except Exception as e: logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})") - await self._callbacks.on_client_disconnected(self._websocket) + await self._client.trigger_client_disconnected() async def _monitor_websocket(self): """Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event.""" await asyncio.sleep(self._params.session_timeout) - await self._callbacks.on_session_timeout(self._websocket) + await self._client.trigger_client_timout() class FastAPIWebsocketOutputTransport(BaseOutputTransport): - def __init__(self, websocket: WebSocket, params: FastAPIWebsocketParams, **kwargs): + def __init__( + self, + client: FastAPIWebsocketClient, + params: FastAPIWebsocketParams, + **kwargs, + ): super().__init__(params, **kwargs) - - self._websocket = websocket + self._client = client self._params = params # write_raw_audio_frames() is called quickly, as soon as we get audio @@ -134,6 +182,14 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): await self._params.serializer.setup(frame) self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2 + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._client.disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._client.disconnect() + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -145,7 +201,10 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): await self._write_frame(frame) async def write_raw_audio_frames(self, frames: bytes): - if self._websocket.client_state != WebSocketState.CONNECTED: + if self._client.is_closing: + return + + if not self._client.is_connected: # Simulate audio playback with a sleep. await self._write_audio_sleep() return @@ -172,25 +231,17 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): await self._write_frame(frame) - self._websocket_audio_buffer = bytes() - # Simulate audio playback with a sleep. await self._write_audio_sleep() async def _write_frame(self, frame: Frame): try: payload = await self._params.serializer.serialize(frame) - if payload and self._websocket.client_state == WebSocketState.CONNECTED: - await self._send_data(payload) + if payload: + await self._client.send(payload) except Exception as e: logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})") - def _send_data(self, data: str | bytes): - if self._params.serializer.type == FrameSerializerType.BINARY: - return self._websocket.send_bytes(data) - else: - return self._websocket.send_text(data) - async def _write_audio_sleep(self): # Simulate a clock. current_time = time.monotonic() @@ -219,11 +270,14 @@ class FastAPIWebsocketTransport(BaseTransport): on_session_timeout=self._on_session_timeout, ) + is_binary = self._params.serializer.type == FrameSerializerType.BINARY + self._client = FastAPIWebsocketClient(websocket, is_binary, self._callbacks) + self._input = FastAPIWebsocketInputTransport( - websocket, self._params, self._callbacks, name=self._input_name + self._client, self._params, name=self._input_name ) self._output = FastAPIWebsocketOutputTransport( - websocket, self._params, name=self._output_name + self._client, self._params, name=self._output_name ) # Register supported handlers. The user will only be able to register From 41720b1a13d7c4d90aff389f3aa9352274f272f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Feb 2025 15:07:35 -0800 Subject: [PATCH 185/427] LLMUserContextAggregator: don't reset timer with interim transcription It turns out that in some cases we only get interim transcriptions (e.g. someone is speaking very very softly or someone is talking in the background). In those cases we don't want to interrupt the bot because there's really nothing to interrupt the bot for. We originally thought we should interrupt the bot right at the time we got an interim frame, but this is causing too many false positives. It's actually better to simply wait for a real transcription before interrupting (in case VAD didn't interrupt). --- CHANGELOG.md | 7 ++++++- src/pipecat/processors/aggregators/llm_response.py | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ba16255..760c79cee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,12 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed -- Fixed an issue where `EndTaskFrame` was not triggering `on_client_disconnected` or closing the WebSocket in FastAPI. +- Fixed an issue that would cause undesired interruptions via + `EmulateUserStartedSpeakingFrame` when only interim transcriptions (i.e. no + final transcriptions) where received. + +- Fixed an issue where `EndTaskFrame` was not triggering + `on_client_disconnected` or closing the WebSocket in FastAPI. - Fixed a context aggregator issue that would not append the LLM text response to the context if a function call happened in the same LLM turn. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 6f1da5af4..017973fda 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -301,8 +301,6 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): async def _handle_interim_transcription(self, _: InterimTranscriptionFrame): self._seen_interim_results = True - # Reset aggregation timer. - self._aggregation_event.set() def _create_aggregation_task(self): self._aggregation_task = self.create_task(self._aggregation_task_handler()) From 98706d429c6acd22a725d2c3df5c10b89643239f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 21 Feb 2025 10:12:54 -0800 Subject: [PATCH 186/427] LLMUserContextAggregator: make sure incoming transcription has text --- src/pipecat/processors/aggregators/llm_response.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 017973fda..ae0b23507 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -293,7 +293,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await self.push_aggregation() async def _handle_transcription(self, frame: TranscriptionFrame): - self._aggregation += f" {frame.text}" if self._aggregation else frame.text + text = frame.text + + # Make sure we really have some text. + if not text.strip(): + return + + self._aggregation += f" {text}" if self._aggregation else text # We just got a final result, so let's reset interim results. self._seen_interim_results = False # Reset aggregation timer. From 12bce2e8c0ad5766ff2ce278c7727ffaf7e9d44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 21 Feb 2025 09:54:11 -0800 Subject: [PATCH 187/427] utils: add support for ellipses in match_endofsentence() --- CHANGELOG.md | 2 ++ src/pipecat/utils/string.py | 4 ++-- tests/test_utils_string.py | 10 ++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 760c79cee..f2375c8e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed `match_endofsentence` support for ellipses. + - Fixed an issue that would cause undesired interruptions via `EmulateUserStartedSpeakingFrame` when only interim transcriptions (i.e. no final transcriptions) where received. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 916dfad41..154d03174 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -13,8 +13,8 @@ ENDOFSENTENCE_PATTERN_STR = r""" (? Date: Thu, 20 Feb 2025 23:15:09 -0800 Subject: [PATCH 188/427] observers: add LLMLogObserver and TranscriptionLogObserver --- CHANGELOG.md | 11 ++- src/pipecat/observers/loggers/__init__.py | 0 .../observers/loggers/llm_log_observer.py | 85 +++++++++++++++++++ .../loggers/transcription_log_observer.py | 54 ++++++++++++ 4 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 src/pipecat/observers/loggers/__init__.py create mode 100644 src/pipecat/observers/loggers/llm_log_observer.py create mode 100644 src/pipecat/observers/loggers/transcription_log_observer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f2375c8e5..a71ae5f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new log observers `LLMLogObserver` and `TranscriptionLogObserver` that + can be useful for debugging your pipelines. + - Added `room_url` property to `DailyTransport`. - Added `addons` argument to `DeepgramSTTService`. @@ -45,6 +48,10 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Fixed an issue where `EndTaskFrame` was not triggering `on_client_disconnected` or closing the WebSocket in FastAPI. +- Fixed an issue in `DeepgramSTTService` where the `sample_rate` passed to the + `LiveOptions` was not being used, causing the service to use the default + sample rate of pipeline. + - Fixed a context aggregator issue that would not append the LLM text response to the context if a function call happened in the same LLM turn. @@ -63,10 +70,6 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Fixed a `STTMuteFilter` issue that would not mute user audio frames causing transcriptions to be generated by the STT service. -- Fixes an issue in `DeepgramSTTService` where `sample_rate` passed to the - `LiveOptions` was not being used, causing the service to use the default - sample rate of pipeline. - ### Other - Added Gemini support to `examples/phone-chatbot`. diff --git a/src/pipecat/observers/loggers/__init__.py b/src/pipecat/observers/loggers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/observers/loggers/llm_log_observer.py b/src/pipecat/observers/loggers/llm_log_observer.py new file mode 100644 index 000000000..907dce70b --- /dev/null +++ b/src/pipecat/observers/loggers/llm_log_observer.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesFrame, + LLMTextFrame, +) +from pipecat.observers.base_observer import BaseObserver +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.ai_services import LLMService + + +class LLMLogObserver(BaseObserver): + """Observer to log LLM activity to the console. + + Logs all frame instances (only from/to LLM service) of: + + - LLMFullResponseStartFrame + - LLMFullResponseEndFrame + - LLMTextFrame + - FunctionCallInProgressFrame + - LLMMessagesFrame + - OpenAILLMContextFrame + + This allows you to track when the LLM starts responding, what it generates, + and when it finishes. + + """ + + async def on_push_frame( + self, + src: FrameProcessor, + dst: FrameProcessor, + frame: Frame, + direction: FrameDirection, + timestamp: int, + ): + if not isinstance(src, LLMService) and not isinstance(dst, LLMService): + return + + time_sec = timestamp / 1_000_000_000 + + arrow = "→" + + # Log LLM start/end frames (output) + if isinstance(frame, (LLMFullResponseStartFrame, LLMFullResponseEndFrame)): + event = "START" if isinstance(frame, LLMFullResponseStartFrame) else "END" + logger.debug(f"🧠 {src} {arrow} LLM {event} RESPONSE at {time_sec:.2f}s") + # Log all LLMTextFrames (output) + elif isinstance(frame, LLMTextFrame): + logger.debug(f"🧠 {src} {arrow} LLM GENERATING: {frame.text!r} at {time_sec:.2f}s") + # Log function calling (output) + elif ( + isinstance(frame, FunctionCallInProgressFrame) + and direction != FrameDirection.DOWNSTREAM + ): + logger.debug( + f"🧠 {src} {arrow} LLM FUNCTION CALL ({frame.tool_call_id}): {frame.function_name!r}({frame.arguments}) at {time_sec:.2f}s" + ) + # Log LLMMessagesFrame (input) + elif isinstance(frame, LLMMessagesFrame): + logger.debug( + f"🧠 {arrow} {dst} LLM MESSAGES FRAME: {frame.messages} at {time_sec:.2f}s" + ) + # Log OpenAILLMContextFrame (input) + elif isinstance(frame, OpenAILLMContextFrame): + logger.debug( + f"🧠 {arrow} {dst} LLM CONTEXT FRAME: {frame.context.messages} at {time_sec:.2f}s" + ) + # Log function call result (input) + elif isinstance(frame, FunctionCallResultFrame): + logger.debug( + f"🧠 {arrow} {src} LLM FUNCTION CALL RESULT ({frame.tool_call_id}): {frame.result} at {time_sec:.2f}s" + ) diff --git a/src/pipecat/observers/loggers/transcription_log_observer.py b/src/pipecat/observers/loggers/transcription_log_observer.py new file mode 100644 index 000000000..630f7ab33 --- /dev/null +++ b/src/pipecat/observers/loggers/transcription_log_observer.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + InterimTranscriptionFrame, + TranscriptionFrame, +) +from pipecat.observers.base_observer import BaseObserver +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.ai_services import STTService + + +class TranscriptionLogObserver(BaseObserver): + """Observer to log transcription activity to the console. + + Logs all frame instances (only from STT service) of: + + - TranscriptionFrame + - InterimTranscriptionFrame + + This allows you to track when the LLM starts responding, what it generates, + and when it finishes. + + """ + + async def on_push_frame( + self, + src: FrameProcessor, + dst: FrameProcessor, + frame: Frame, + direction: FrameDirection, + timestamp: int, + ): + if not isinstance(src, STTService): + return + + time_sec = timestamp / 1_000_000_000 + + arrow = "→" + + if isinstance(frame, TranscriptionFrame): + logger.debug( + f"💬 {src} {arrow} TRANSCRIPTION: {frame.text!r} from {frame.user_id!r} at {time_sec:.2f}s" + ) + elif isinstance(frame, InterimTranscriptionFrame): + logger.debug( + f"💬 {src} {arrow} INTERIM TRANSCRIPTION: {frame.text!r} from {frame.user_id!r} at {time_sec:.2f}s" + ) From 65f548b2ece23442acbb50e8e98517dd5b94312e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Feb 2025 23:15:30 -0800 Subject: [PATCH 189/427] examples(30-observer): update to use LLMLogObserver --- examples/foundational/30-observer.py | 36 +--------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 4b672ba0e..0988be76d 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -18,12 +18,10 @@ from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, StartInterruptionFrame, ) from pipecat.observers.base_observer import BaseObserver +from pipecat.observers.loggers.llm_log_observer import LLMLogObserver from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -73,38 +71,6 @@ class DebugObserver(BaseObserver): logger.info(f"🤖 BOT STOP SPEAKING: {src} {arrow} {dst} at {time_sec:.2f}s") -class LLMLogObserver(BaseObserver): - """Observer to log LLM activity to the console. - - Logs all frame instances of: - - LLMFullResponseStartFrame (only from LLM service) - - LLMTextFrame - - LLMFullResponseEndFrame (only from LLM service) - - This allows you to track when the LLM starts responding, what it generates, and when it finishes. - Log format: [LLM EVENT]: [details] at [timestamp]s - """ - - async def on_push_frame( - self, - src: FrameProcessor, - dst: FrameProcessor, - frame: Frame, - direction: FrameDirection, - timestamp: int, - ): - time_sec = timestamp / 1_000_000_000 - - # Only log start/end frames from OpenAILLMService - if isinstance(frame, (LLMFullResponseStartFrame, LLMFullResponseEndFrame)): - if isinstance(src, OpenAILLMService): - event = "START" if isinstance(frame, LLMFullResponseStartFrame) else "END" - logger.info(f"🧠 LLM {event} RESPONSE at {time_sec:.2f}s") - # Log all LLMTextFrames - elif isinstance(frame, LLMTextFrame): - logger.info(f"🧠 LLM GENERATING: {frame.text!r} at {time_sec:.2f}s") - - async def main(): async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) From b5662520aacd0427245a5ef2322734031135c2ac Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sun, 23 Feb 2025 11:04:24 -0500 Subject: [PATCH 190/427] Add one additional ellipsis test to test_utils_string --- tests/test_utils_string.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index 4b08bd411..e4cdb4cb4 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -17,6 +17,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is a sentence;") == 19 assert match_endofsentence("This is a sentence...") == 21 assert match_endofsentence("This is a sentence . . .") == 24 + assert match_endofsentence("This is a sentence. ..") == 22 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") From 5b637bd8260bfedc4dd86fa6b4651abcfdd8edec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 21 Feb 2025 15:00:06 -0800 Subject: [PATCH 191/427] services: fix some TTS websocket service interruption handling --- CHANGELOG.md | 4 + src/pipecat/services/ai_services.py | 100 ++++++++++++++++++++-- src/pipecat/services/cartesia.py | 10 +-- src/pipecat/services/elevenlabs.py | 12 +-- src/pipecat/services/fish.py | 18 ++-- src/pipecat/services/lmnt.py | 13 ++- src/pipecat/services/playht.py | 16 ++-- src/pipecat/services/rime.py | 10 +-- src/pipecat/services/websocket_service.py | 22 ++++- 9 files changed, 147 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71ae5f02..3127a6a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,10 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed a `ElevenLabsTTSService`, `FishAudioTTSService`, `LMNTTTSService` and + `PlayHTTTSService` issue that was resulting in audio requested before an + interruption being played after an interruption. + - Fixed `match_endofsentence` support for ellipses. - Fixed an issue that would cause undesired interruptions via diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 05c637126..30c5d137f 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -15,6 +15,7 @@ from loguru import logger from pipecat.audio.utils import calculate_audio_volume, exp_smoothing from pipecat.frames.frames import ( AudioRawFrame, + BotStartedSpeakingFrame, BotStoppedSpeakingFrame, CancelFrame, EndFrame, @@ -40,6 +41,7 @@ from pipecat.frames.frames import ( from pipecat.metrics.metrics import MetricsData from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language from pipecat.utils.string import match_endofsentence from pipecat.utils.text.base_text_filter import BaseTextFilter @@ -434,6 +436,12 @@ class TTSService(AIService): class WordTTSService(TTSService): + """This is a base class for TTS services that support word timestamps. Word + timestamps are useful to synchronize audio with text of the spoken + words. This way only the spoken words are added to the conversation context. + + """ + def __init__(self, **kwargs): super().__init__(**kwargs) self._initial_word_timestamp = -1 @@ -503,11 +511,93 @@ class WordTTSService(TTSService): self._words_queue.task_done() -class AudioContextWordTTSService(WordTTSService): - """This services allow us to send multiple TTS request to the services. Each - request could be multiple sentences long which are grouped by context. For - this to work, the TTS service needs to support handling multiple requests at - once (i.e. multiple simultaneous contexts). +class WebsocketTTSService(TTSService, WebsocketService): + """This is a base class for websocket-based TTS services.""" + + def __init__(self, **kwargs): + TTSService.__init__(self, **kwargs) + WebsocketService.__init__(self) + + +class InterruptibleTTSService(WebsocketTTSService): + """This is a base class for websocket-based TTS services that don't support + word timestamps and that don't offer a way to correlate the generated audio + to the requested text. + + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # Indicates if the bot is speaking. If the bot is not speaking we don't + # need to reconnect when the user speaks. If the bot is speaking and the + # user interrupts we need to reconnect. + self._bot_speaking = False + + async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + await super()._handle_interruption(frame, direction) + if self._bot_speaking: + await self._disconnect() + await self._connect() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, BotStartedSpeakingFrame): + self._bot_speaking = True + elif isinstance(frame, BotStoppedSpeakingFrame): + self._bot_speaking = False + + +class WebsocketWordTTSService(WordTTSService, WebsocketService): + """This is a base class for websocket-based TTS services that support word + timestamps. + + """ + + def __init__(self, **kwargs): + WordTTSService.__init__(self, **kwargs) + WebsocketService.__init__(self) + + +class InterruptibleWordTTSService(WebsocketWordTTSService): + """This is a base class for websocket-based TTS services that support word + timestamps but don't offer a way to correlate the generated audio to the + requested text. + + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # Indicates if the bot is speaking. If the bot is not speaking we don't + # need to reconnect when the user speaks. If the bot is speaking and the + # user interrupts we need to reconnect. + self._bot_speaking = False + + async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + await super()._handle_interruption(frame, direction) + if self._bot_speaking: + await self._disconnect() + await self._connect() + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, BotStartedSpeakingFrame): + self._bot_speaking = True + elif isinstance(frame, BotStoppedSpeakingFrame): + self._bot_speaking = False + + +class AudioContextWordTTSService(WebsocketWordTTSService): + """This is a base class for websocket-based TTS services that support word + timestamps and also allow correlating the generated audio with the requested + text. + + Each request could be multiple sentences long which are grouped by + context. For this to work, the TTS service needs to support handling + multiple requests at once (i.e. multiple simultaneous contexts). The audio received from the TTS will be played in context order. That is, if we requested audio for a context "A" and then audio for context "B", the diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index f9f2bdbe6..0ec55aa0c 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -13,22 +13,18 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, Frame, - LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, TTSAudioRawFrame, - TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AudioContextWordTTSService, TTSService -from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language # See .env.example for Cartesia configuration needed @@ -75,7 +71,7 @@ def language_to_cartesia_language(language: Language) -> Optional[str]: return result -class CartesiaTTSService(AudioContextWordTTSService, WebsocketService): +class CartesiaTTSService(AudioContextWordTTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN speed: Optional[Union[str, float]] = "" @@ -105,15 +101,13 @@ class CartesiaTTSService(AudioContextWordTTSService, WebsocketService): # if we're interrupted. Cartesia gives us word-by-word timestamps. We # can use those to generate text frames ourselves aligned with the # playout timing of the audio! - AudioContextWordTTSService.__init__( - self, + super().__init__( aggregate_sentences=True, push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) self._api_key = api_key self._cartesia_version = cartesia_version diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 58ec91342..47fa87b5a 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -14,22 +14,18 @@ from loguru import logger from pydantic import BaseModel, model_validator from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, Frame, - LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, TTSAudioRawFrame, - TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService, WordTTSService -from pipecat.services.websocket_service import WebsocketService +from pipecat.services.ai_services import InterruptibleWordTTSService, TTSService from pipecat.transcriptions.language import Language # See .env.example for ElevenLabs configuration needed @@ -141,7 +137,7 @@ def calculate_word_times( return word_times -class ElevenLabsTTSService(WordTTSService, WebsocketService): +class ElevenLabsTTSService(InterruptibleWordTTSService): class InputParams(BaseModel): language: Optional[Language] = None optimize_streaming_latency: Optional[str] = None @@ -186,8 +182,7 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): # Finally, ElevenLabs doesn't provide information on when the bot stops # speaking for a while, so we want the parent class to send TTSStopFrame # after a short period not receiving any audio. - WordTTSService.__init__( - self, + super().__init__( aggregate_sentences=True, push_text_frames=False, push_stop_frames=True, @@ -195,7 +190,6 @@ class ElevenLabsTTSService(WordTTSService, WebsocketService): sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) self._api_key = api_key self._url = url diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index e2a75bdb2..1c5e6a08a 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -22,8 +22,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService -from pipecat.services.websocket_service import WebsocketService +from pipecat.services.ai_services import InterruptibleTTSService from pipecat.transcriptions.language import Language try: @@ -40,7 +39,7 @@ except ModuleNotFoundError as e: FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] -class FishAudioTTSService(TTSService, WebsocketService): +class FishAudioTTSService(InterruptibleTTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN latency: Optional[str] = "normal" # "normal" or "balanced" @@ -110,11 +109,12 @@ class FishAudioTTSService(TTSService, WebsocketService): self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): - await self._disconnect_websocket() if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None + await self._disconnect_websocket() + async def _connect_websocket(self): try: logger.debug("Connecting to Fish Audio") @@ -149,6 +149,11 @@ class FishAudioTTSService(TTSService, WebsocketService): return self._websocket raise Exception("Websocket not connected") + async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + await super()._handle_interruption(frame, direction) + await self.stop_all_metrics() + self._request_id = None + async def _receive_messages(self): async for message in self._get_websocket(): try: @@ -168,11 +173,6 @@ class FishAudioTTSService(TTSService, WebsocketService): except Exception as e: logger.error(f"Error processing message: {e}") - async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): - await super()._handle_interruption(frame, direction) - await self.stop_all_metrics() - self._request_id = None - async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating Fish TTS: [{text}]") try: diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 0272690e8..f6010eeb7 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -21,8 +21,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService -from pipecat.services.websocket_service import WebsocketService +from pipecat.services.ai_services import InterruptibleTTSService from pipecat.transcriptions.language import Language # See .env.example for LMNT configuration needed @@ -60,7 +59,7 @@ def language_to_lmnt_language(language: Language) -> Optional[str]: return result -class LmntTTSService(TTSService, WebsocketService): +class LmntTTSService(InterruptibleTTSService): def __init__( self, *, @@ -70,14 +69,12 @@ class LmntTTSService(TTSService, WebsocketService): language: Language = Language.EN, **kwargs, ): - TTSService.__init__( - self, + super().__init__( push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) self._api_key = api_key self._voice_id = voice_id @@ -116,12 +113,12 @@ class LmntTTSService(TTSService, WebsocketService): self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): - await self._disconnect_websocket() - if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None + await self._disconnect_websocket() + async def _connect_websocket(self): """Connect to LMNT websocket.""" try: diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 7f0b29c87..828acbdb7 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -16,22 +16,18 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, CancelFrame, EndFrame, ErrorFrame, Frame, - LLMFullResponseEndFrame, StartFrame, StartInterruptionFrame, TTSAudioRawFrame, - TTSSpeakFrame, TTSStartedFrame, TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService -from pipecat.services.websocket_service import WebsocketService +from pipecat.services.ai_services import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language try: @@ -100,7 +96,7 @@ def language_to_playht_language(language: Language) -> Optional[str]: return result -class PlayHTTTSService(TTSService, WebsocketService): +class PlayHTTTSService(InterruptibleTTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -118,13 +114,11 @@ class PlayHTTTSService(TTSService, WebsocketService): params: InputParams = InputParams(), **kwargs, ): - TTSService.__init__( - self, + super().__init__( pause_frame_processing=True, sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) self._api_key = api_key self._user_id = user_id @@ -168,12 +162,12 @@ class PlayHTTTSService(TTSService, WebsocketService): self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): - await self._disconnect_websocket() - if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None + await self._disconnect_websocket() + async def _connect_websocket(self): try: logger.debug("Connecting to PlayHT") diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index cc59b132d..6e2daccd6 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -26,7 +26,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AudioContextWordTTSService, TTSService -from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language try: @@ -55,7 +54,7 @@ def language_to_rime_language(language: Language) -> str: return LANGUAGE_MAP.get(language, "eng") -class RimeTTSService(AudioContextWordTTSService, WebsocketService): +class RimeTTSService(AudioContextWordTTSService): """Text-to-Speech service using Rime's websocket API. Uses Rime's websocket JSON API to convert text to speech with word-level timing @@ -92,8 +91,7 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): params: Additional configuration parameters. """ # Initialize with parent class settings for proper frame handling - AudioContextWordTTSService.__init__( - self, + super().__init__( aggregate_sentences=True, push_text_frames=False, push_stop_frames=True, @@ -101,7 +99,6 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) # Store service configuration self._api_key = api_key @@ -172,11 +169,12 @@ class RimeTTSService(AudioContextWordTTSService, WebsocketService): async def _disconnect(self): """Close websocket connection and clean up tasks.""" - await self._disconnect_websocket() if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None + await self._disconnect_websocket() + async def _connect_websocket(self): """Connect to Rime websocket API with configured settings.""" try: diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index a8dc97414..19fb4ae01 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -90,14 +90,32 @@ class WebsocketService(ABC): logger.error(f"{self} reconnection failed: {reconnect_error}") continue + @abstractmethod + async def _connect(self): + """Implement service-specific connection logic. This function will + connect to the websocket via _connect_websocket() among other connection + logic.""" + pass + + @abstractmethod + async def _disconnect(self): + """Implement service-specific disconnection logic. This function will + disconnect to the websocket via _connect_websocket() among other + connection logic. + + """ + pass + @abstractmethod async def _connect_websocket(self): - """Implement service-specific websocket connection logic.""" + """Implement service-specific websocket connection logic. This function + should only connect to the websocket.""" pass @abstractmethod async def _disconnect_websocket(self): - """Implement service-specific websocket disconnection logic.""" + """Implement service-specific websocket disconnection logic. This + function should only disconnect from the websocket.""" pass @abstractmethod From 45058d4a9479c10a7ac618c4092a34051d4b7caa Mon Sep 17 00:00:00 2001 From: allenmylath Date: Mon, 24 Feb 2025 22:11:19 +0530 Subject: [PATCH 192/427] Update audio_buffer_processor.py (#1266) --- src/pipecat/processors/audio/audio_buffer_processor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index cc4ac6cd7..2f9b975e3 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -22,9 +22,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class AudioBufferProcessor(FrameProcessor): """This processor buffers audio raw frames (input and output). The mixed - audio can be obtained by calling `get_audio()` (if `buffer_size` is 0) or by - registering an "on_audio_data" event handler. The event handler will be - called every time `buffer_size` is reached. + audio can be obtained by registering an "on_audio_data" event handler. + The event handler will be called every time `buffer_size` is reached. You can provide the desired output `sample_rate` and incoming audio frames will resampled to match it. Also, you can provide the number of channels, 1 From d6f29a0f4b7243ce9a57af0017537511740a75f3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 24 Feb 2025 14:32:00 -0500 Subject: [PATCH 193/427] Update AnthropicLLMService to use claude-3-7-sonnet-20250219 by default --- CHANGELOG.md | 3 +++ src/pipecat/services/anthropic.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71ae5f02..bdd305fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `AnthropicLLMService` now uses `claude-3-7-sonnet-20250219` as the default + model. + - `RimeHttpTTSService` needs an `aiohttp.ClientSession` to be passed to the constructor as all the other HTTP-based services. diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index a7c5fd5f0..dcf8bc242 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -96,7 +96,7 @@ class AnthropicLLMService(LLMService): self, *, api_key: str, - model: str = "claude-3-5-sonnet-20241022", + model: str = "claude-3-7-sonnet-20250219", params: InputParams = InputParams(), client=None, **kwargs, From 3b9b9200ea16de38737917f3ba1294f4c976cde6 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 24 Feb 2025 15:05:42 -0500 Subject: [PATCH 194/427] Remove openai optional dependency from services as it's now required --- pyproject.toml | 22 +++++++++++----------- src/pipecat/services/azure.py | 2 +- src/pipecat/services/cerebras.py | 14 ++------------ src/pipecat/services/deepseek.py | 14 ++------------ src/pipecat/services/fireworks.py | 10 +--------- src/pipecat/services/openpipe.py | 2 +- src/pipecat/services/perplexity.py | 15 ++------------- 7 files changed, 20 insertions(+), 59 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5b1ba28a..768c9c816 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,11 +45,11 @@ Website = "https://pipecat.ai" anthropic = [ "anthropic~=0.45.2" ] assemblyai = [ "assemblyai~=0.36.0" ] aws = [ "boto3~=1.35.99" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0", "openai~=1.59.6" ] +azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -cerebras = [ "openai~=1.59.6" ] -deepseek = [ "openai~=1.59.6" ] +cerebras = [] +deepseek = [] daily = [ "daily-python~=0.14.2" ] deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] @@ -57,10 +57,10 @@ fal = [ "fal-client~=0.5.6" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.2.0", "google-generativeai~=0.8.4" ] -grok = [ "openai~=1.59.6" ] -groq = [ "openai~=1.59.6" ] +grok = [] +groq = [] gstreamer = [ "pygobject~=3.50.0" ] -fireworks = [ "openai~=1.59.6" ] +fireworks = [] krisp = [ "pipecat-ai-krisp~=0.3.0" ] koala = [ "pvkoala~=2.0.3" ] langchain = [ "langchain~=0.3.14", "langchain-community~=0.3.14", "langchain-openai~=0.3.0" ] @@ -68,11 +68,11 @@ livekit = [ "livekit~=0.19.1", "livekit-api~=0.8.1", "tenacity~=9.0.0" ] lmnt = [ "websockets~=13.1" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.13", "transformers~=4.48.0" ] -nim = [ "openai~=1.59.6" ] +nim = [] noisereduce = [ "noisereduce~=3.0.3" ] -openai = [ "openai~=1.59.6", "websockets~=13.1" ] +openai = [ "websockets~=13.1" ] openpipe = [ "openpipe~=4.45.0" ] -perplexity = [ "openai~=1.59.6" ] +perplexity = [] playht = [ "pyht~=0.1.6", "websockets~=13.1" ] rime = [ "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.18.0" ] @@ -80,10 +80,10 @@ sentry = [ "sentry-sdk~=2.20.0" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] -together = [ "openai~=1.59.6" ] +together = [] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] whisper = [ "faster-whisper~=1.1.1" ] -openrouter = [ "openai~=1.59.6" ] +openrouter = [] [tool.setuptools.packages.find] # All the following settings are optional: diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index d898f4b7c..fbdd08ab1 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -10,6 +10,7 @@ from typing import AsyncGenerator, Optional import aiohttp from loguru import logger +from openai import AsyncAzureOpenAI from PIL import Image from pydantic import BaseModel @@ -48,7 +49,6 @@ try: PushAudioInputStream, ) from azure.cognitiveservices.speech.dialog import AudioConfig - from openai import AsyncAzureOpenAI except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py index fce3c22bf..b5e34afe5 100644 --- a/src/pipecat/services/cerebras.py +++ b/src/pipecat/services/cerebras.py @@ -7,22 +7,12 @@ from typing import List from loguru import logger +from openai import AsyncStream +from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService -try: - from openai import ( - AsyncStream, - ) - from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use Cerebras, you need to `pip install pipecat-ai[cerebras]`. Also, set `CEREBRAS_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - class CerebrasLLMService(OpenAILLMService): """A service for interacting with Cerebras's API using the OpenAI-compatible interface. diff --git a/src/pipecat/services/deepseek.py b/src/pipecat/services/deepseek.py index 67433dfaa..2537f2f7c 100644 --- a/src/pipecat/services/deepseek.py +++ b/src/pipecat/services/deepseek.py @@ -8,22 +8,12 @@ from typing import List from loguru import logger +from openai import AsyncStream +from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService -try: - from openai import ( - AsyncStream, - ) - from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use DeepSeek, you need to `pip install pipecat-ai[deepseek]`. Also, set `DEEPSEEK_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - class DeepSeekLLMService(OpenAILLMService): """A service for interacting with DeepSeek's API using the OpenAI-compatible interface. diff --git a/src/pipecat/services/fireworks.py b/src/pipecat/services/fireworks.py index 0e50fc980..8e40e1a7b 100644 --- a/src/pipecat/services/fireworks.py +++ b/src/pipecat/services/fireworks.py @@ -8,19 +8,11 @@ from typing import List from loguru import logger +from openai.types.chat import ChatCompletionMessageParam from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService -try: - from openai.types.chat import ChatCompletionMessageParam -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use Fireworks, you need to `pip install pipecat-ai[fireworks]`. Also, set `FIREWORKS_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - class FireworksLLMService(OpenAILLMService): """A service for interacting with Fireworks AI using the OpenAI-compatible interface. diff --git a/src/pipecat/services/openpipe.py b/src/pipecat/services/openpipe.py index 98241eb50..c89c9d6a3 100644 --- a/src/pipecat/services/openpipe.py +++ b/src/pipecat/services/openpipe.py @@ -7,12 +7,12 @@ from typing import Dict, List, Optional from loguru import logger +from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService try: - from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from openpipe import AsyncOpenAI as OpenPipeAI from openpipe import AsyncStream except ModuleNotFoundError as e: diff --git a/src/pipecat/services/perplexity.py b/src/pipecat/services/perplexity.py index 8152461c3..b0c560ce4 100644 --- a/src/pipecat/services/perplexity.py +++ b/src/pipecat/services/perplexity.py @@ -7,24 +7,13 @@ from typing import List from loguru import logger +from openai import NOT_GIVEN, AsyncStream +from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService -try: - from openai import ( - NOT_GIVEN, - AsyncStream, - ) - from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use Perplexity, you need to `pip install pipecat-ai[perplexity]`. Also, set `PERPLEXITY_API_KEY` environment variable." - ) - raise Exception(f"Missing module: {e}") - class PerplexityLLMService(OpenAILLMService): """A service for interacting with Perplexity's API. From cefc2a1088515ff8b49f6dc312ca7d03ceadd269 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 24 Feb 2025 15:06:13 -0500 Subject: [PATCH 195/427] Fix test-requirements.text ordering --- test-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 82a6fbb25..fe45b9ff2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -16,6 +16,7 @@ langchain~=0.2.14 livekit~=0.13.1 lmnt~=1.1.4 loguru~=0.7.2 +Markdown~=3.7 numpy~=1.26.4 openai~=1.37.2 openpipe~=4.24.0 @@ -29,5 +30,4 @@ silero-vad~=5.1 soxr~=0.5.0 together~=1.2.7 transformers~=4.48.0 -websockets~=13.1 -Markdown~=3.7 \ No newline at end of file +websockets~=13.1 \ No newline at end of file From 408270b6477497d26c7969bdf10d68da0057e5fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 14:37:37 -0800 Subject: [PATCH 196/427] lmnt: don't send "eof" before closing the socket --- src/pipecat/services/lmnt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index f6010eeb7..dd6b9e3b8 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -150,8 +150,9 @@ class LmntTTSService(InterruptibleTTSService): if self._websocket: logger.debug("Disconnecting from LMNT") - # Send EOF message before closing - await self._websocket.send(json.dumps({"eof": True})) + # NOTE(aleix): sending EOF message before closing is causing + # errors on the websocket, so we just skip it for now. + # await self._websocket.send(json.dumps({"eof": True})) await self._websocket.close() self._websocket = None From 2110b79507388a4c9f29619cf33587e3294c3762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 14:23:03 -0800 Subject: [PATCH 197/427] services(llm): add on_completion_timeout event --- CHANGELOG.md | 5 +++++ src/pipecat/services/ai_services.py | 2 ++ src/pipecat/services/anthropic.py | 7 +++++-- src/pipecat/services/google/google.py | 4 ++++ src/pipecat/services/openai.py | 14 +++++++++----- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c147eae7f..299ead7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `on_completion_timeout` event for LLM services (all OpenAI-based + services, Anthropic and Google). Note that this event will only get triggered + if LLM timeouts are setup and if the timeout was reached. It can be useful to + retrigger another completion and see if the timeout was just a blip. + - Added new log observers `LLMLogObserver` and `TranscriptionLogObserver` that can be useful for debugging your pipelines. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 30c5d137f..8f0df1b37 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -142,6 +142,8 @@ class LLMService(AIService): self._callbacks = {} self._start_callbacks = {} + self._register_event_handler("on_completion_timeout") + # TODO-CB: callback function type def register_function(self, function_name: Optional[str], callback, start_callback=None): # Registering a function with the function_name set to None will run that callback diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index dcf8bc242..1316802de 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -4,15 +4,16 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import asyncio import base64 import copy import io import json import re -from asyncio import CancelledError from dataclasses import dataclass from typing import Any, Dict, List, Optional, Union +import httpx from loguru import logger from PIL import Image from pydantic import BaseModel, Field @@ -251,12 +252,14 @@ class AnthropicLLMService(LLMService): if total_input_tokens >= 1024: context.turns_above_cache_threshold += 1 - except CancelledError: + except asyncio.CancelledError: # If we're interrupted, we won't get a complete usage report. So set our flag to use the # token estimate. The reraise the exception so all the processors running in this task # also get cancelled. use_completion_tokens_estimate = True raise + except httpx.TimeoutException: + await self._call_event_handler("on_completion_timeout") except Exception as e: logger.exception(f"{self} exception: {e}") finally: diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 67a2b3954..58042ab81 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -11,6 +11,8 @@ import json import os import time +from google.api_core.exceptions import DeadlineExceeded + # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" @@ -1126,6 +1128,8 @@ class GoogleLLMService(LLMService): else: logger.exception(f"{self} error: {e}") + except DeadlineExceeded: + await self._call_event_handler("on_completion_timeout") except Exception as e: logger.exception(f"{self} exception: {e}") finally: diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 8c2b7359b..6ed3b4612 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -310,11 +310,15 @@ class BaseOpenAILLMService(LLMService): await self.push_frame(frame, direction) if context: - await self.push_frame(LLMFullResponseStartFrame()) - await self.start_processing_metrics() - await self._process_context(context) - await self.stop_processing_metrics() - await self.push_frame(LLMFullResponseEndFrame()) + try: + await self.push_frame(LLMFullResponseStartFrame()) + await self.start_processing_metrics() + await self._process_context(context) + except httpx.TimeoutException: + await self._call_event_handler("on_completion_timeout") + finally: + await self.stop_processing_metrics() + await self.push_frame(LLMFullResponseEndFrame()) @dataclass From 40c2452d6e5e160a08e022ac9ba3d2d487863583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 15:35:18 -0800 Subject: [PATCH 198/427] google: updgrade OpenAILLMContext to GoogleLLMContext --- CHANGELOG.md | 3 +++ src/pipecat/services/google/google.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 299ead7b0..3a1651808 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an issue that was not allowing to pass an `OpenAILLMContext` to create + `GoogleLLMService`'s context aggregators. + - Fixed a `ElevenLabsTTSService`, `FishAudioTTSService`, `LMNTTTSService` and `PlayHTTTSService` issue that was resulting in audio requested before an interruption being played after an interruption. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 58042ab81..1dafc92bc 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1176,6 +1176,8 @@ class GoogleLLMService(LLMService): def create_context_aggregator( context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True ) -> GoogleContextAggregatorPair: + if isinstance(context, OpenAILLMContext): + context = GoogleLLMContext.upgrade_to_google(context) user = GoogleUserContextAggregator(context) assistant = GoogleAssistantContextAggregator( context, expect_stripped_words=assistant_expect_stripped_words From 07b9be53086172c0ec6a80cffac7edb77051da2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 17:33:10 -0800 Subject: [PATCH 199/427] PipelineTask: add check_dangling_tasks parameter --- CHANGELOG.md | 4 ++++ src/pipecat/pipeline/task.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 299ead7b0..e7ae19967 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a new `PipelineTask` parameter `check_dangling_tasks` to enable or + disable checking for frame processors' dangling tasks when the Pipeline + finishes running. + - Added new `on_completion_timeout` event for LLM services (all OpenAI-based services, Anthropic and Google). Note that this event will only get triggered if LLM timeouts are setup and if the timeout was reached. It can be useful to diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 66b1e7647..1813878e1 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -123,6 +123,7 @@ class PipelineTask(BaseTask): pipeline: The pipeline to execute. params: Configuration parameters for the pipeline. clock: Clock implementation for timing operations. + check_dangling_tasks: Whether to check for processors' tasks finishing properly. """ def __init__( @@ -130,6 +131,7 @@ class PipelineTask(BaseTask): pipeline: BasePipeline, params: PipelineParams = PipelineParams(), clock: BaseClock = SystemClock(), + check_dangling_tasks: bool = True, ): self._id: int = obj_id() self._name: str = f"{self.__class__.__name__}#{obj_count(self)}" @@ -137,6 +139,7 @@ class PipelineTask(BaseTask): self._pipeline = pipeline self._clock = clock self._params = params + self._check_dangling_tasks = check_dangling_tasks self._finished = False # This queue receives frames coming from the pipeline upstream. @@ -220,7 +223,8 @@ class PipelineTask(BaseTask): pass await self._cancel_tasks() await self._cleanup() - self._print_dangling_tasks() + if self._check_dangling_tasks: + self._print_dangling_tasks() self._finished = True async def queue_frame(self, frame: Frame): From f66be2cfa7070f55f49283ebc517855ab9616a47 Mon Sep 17 00:00:00 2001 From: Dominic Stewart <45786774+DominicStewart@users.noreply.github.com> Date: Mon, 24 Feb 2025 20:29:55 -0800 Subject: [PATCH 200/427] Dom/gemini system prompt switching (#1260) * Updated example to use Gemini * Fixed typo * Based on feedback, made the gemini file something that can be called separately * Updated the readme * Updated the readme * Changed example to use gemini 2.0 flash lite * This works * Improvement * I think this works * Updated the code to use the correct prompt broken down into smaller pieces * Added a few more things to detect in the prompt * Fixed import ordering * Updated prompt for non gemini bot to look for more voicemail examples, plus added logic to detect if we're doing dialin or not to avoid a non-fatal dialin related error * moved terminate call to handlers class * Simplified logic for dialin * Forgot to use the same logic for the openai bot * Starting to add logic for native audio input for flash lite * Fixed logic * Fixed some code based on suggestions --- examples/phone-chatbot/README.md | 30 ++- examples/phone-chatbot/bot_daily.py | 13 +- examples/phone-chatbot/bot_daily_gemini.py | 225 +++++++++++++++------ 3 files changed, 202 insertions(+), 66 deletions(-) diff --git a/examples/phone-chatbot/README.md b/examples/phone-chatbot/README.md index a8a1cee8e..040fb7038 100644 --- a/examples/phone-chatbot/README.md +++ b/examples/phone-chatbot/README.md @@ -106,12 +106,12 @@ curl -X POST "http://localhost:7860/daily_start_bot" \ -d '{"dialoutNumber": "+18057145330", "detectVoicemail": true}' ``` -### New! Using Gemini with Daily +### New! Using Gemini 2.0 Flash Lite with Daily -We have introduced a new example file that uses Gemini. You can find the code within bot_daily_gemini.py. -If you want to spin up a Gemini-based bot for this demo, instead of an OpenAI-based bot, call the same properties above but on the `daily_gemini_start_bot` endpoint instead. +We have introduced support for Google's Gemini 2.0 Flash Lite model in this example. This lightweight model offers faster response times and reduced costs while maintaining good conversational capabilities. -For example: +**Quick Start** +To use the Gemini-based bot instead of OpenAI: ```shell curl -X POST "http://localhost:7860/daily_gemini_start_bot" \ py pipecat @@ -119,7 +119,27 @@ curl -X POST "http://localhost:7860/daily_gemini_start_bot" \ -d '{"detectVoicemail": true}' ``` -Any request body properties supported by `/daily_start_bot` (such as "detectVoicemail", "dialoutnumber", etc) can also be passed to `/daily_gemini_start_bot`. The only difference is that calling the Gemini endpoint will start a Gemini bot session. +All request body parameters supported by /daily_start_bot (such as detectVoicemail, dialoutNumber, etc.) are also compatible with /daily_gemini_start_bot. + +This example uses context switching to help steer the bot in the right direction. As Flash Lite is a smaller model, breaking the prompt down into smaller piece helps to improve the bot's accuracy. + +For example, instead of giving one large prompt like: + +```python +system_instruction="""You are a chatbot that needs to detect if you're talking to a voicemail system or human, then either leave a message or have a conversation. If it's voicemail, say "Hello, this is a message..." and hang up. If it's a human, introduce yourself and be helpful until they say goodbye.""" +``` + +We break it into stages: + +First prompt focuses only on detection: "Determine if this is voicemail or human" +After detection, we switch to a new context: either "Leave this specific voicemail message" or "Have a conversation with the human". + +**Implementation Details** +The implementation is available in bot_daily_gemini.py and features: + +- Staged prompting approach: Breaking down complex tasks into smaller, more focused prompts to improve the lightweight model's performance +- Dynamic context switching: The bot can change its behavior in real-time based on what it detects (voicemail vs. human caller) +- Function-based architecture: Uses function calling to trigger context switches and call termination ### More information diff --git a/examples/phone-chatbot/bot_daily.py b/examples/phone-chatbot/bot_daily.py index 0ba631296..16aba1b82 100644 --- a/examples/phone-chatbot/bot_daily.py +++ b/examples/phone-chatbot/bot_daily.py @@ -49,7 +49,11 @@ async def main( # If you are handling this via Twilio, Telnyx, set this to None # and handle call-forwarding when on_dialin_ready fires. - dialin_settings = DailyDialinSettings(call_id=callId, call_domain=callDomain) + # We don't want to specify dial-in settings if we're not dialing in + dialin_settings = None + if callId and callDomain: + dialin_settings = DailyDialinSettings(call_id=callId, call_domain=callDomain) + transport = DailyTransport( room_url, token, @@ -96,6 +100,13 @@ async def main( - **"Please leave a message after the beep."** - **"No one is available to take your call."** - **"Record your message after the tone."** + - **"Please leave a message after the beep"** + - **"You have reached voicemail for..."** + - **"You have reached [phone number]"** + - **"[phone number] is unavailable"** + - **"The person you are trying to reach..."** + - **"The number you have dialed..."** + - **"Your call has been forwarded to an automated voice messaging system"** - **Any phrase that suggests an answering machine or voicemail.** - **ASSUME IT IS A VOICEMAIL. DO NOT WAIT FOR MORE CONFIRMATION.** - **IF THE CALL SAYS "PLEASE LEAVE A MESSAGE AFTER THE BEEP", WAIT FOR THE BEEP BEFORE LEAVING A MESSAGE.** diff --git a/examples/phone-chatbot/bot_daily_gemini.py b/examples/phone-chatbot/bot_daily_gemini.py index a983cd270..b36313c5a 100644 --- a/examples/phone-chatbot/bot_daily_gemini.py +++ b/examples/phone-chatbot/bot_daily_gemini.py @@ -7,17 +7,29 @@ import argparse import asyncio import os import sys +from dataclasses import dataclass from typing import Optional +import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import EndTaskFrame +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + EndTaskFrame, + Frame, + InputAudioRawFrame, + SystemFrame, + TranscriptionFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, +) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.ai_services import LLMService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.google import GoogleLLMContext, GoogleLLMService @@ -33,10 +45,119 @@ daily_api_key = os.getenv("DAILY_API_KEY", "") daily_api_url = os.getenv("DAILY_API_URL", "https://api.daily.co/v1") +class UserAudioCollector(FrameProcessor): + """This FrameProcessor collects audio frames in a buffer, then adds them to the + LLM context when the user stops speaking. + """ + + def __init__(self, context, user_context_aggregator): + super().__init__() + self._context = context + self._user_context_aggregator = user_context_aggregator + self._audio_frames = [] + self._start_secs = 0.2 # this should match VAD start_secs (hardcoding for now) + self._user_speaking = False + + async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + # We could gracefully handle both audio input and text/transcription input ... + # but let's leave that as an exercise to the reader. :-) + return + if isinstance(frame, UserStartedSpeakingFrame): + self._user_speaking = True + elif isinstance(frame, UserStoppedSpeakingFrame): + self._user_speaking = False + self._context.add_audio_frames_message(audio_frames=self._audio_frames) + await self._user_context_aggregator.push_frame( + self._user_context_aggregator.get_context_frame() + ) + elif isinstance(frame, InputAudioRawFrame): + if self._user_speaking: + self._audio_frames.append(frame) + else: + # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest + # frames as necessary. Assume all audio frames have the same duration. + self._audio_frames.append(frame) + frame_duration = len(frame.audio) / 16 * frame.num_channels / frame.sample_rate + buffer_duration = frame_duration * len(self._audio_frames) + while buffer_duration > self._start_secs: + self._audio_frames.pop(0) + buffer_duration -= frame_duration + + await self.push_frame(frame, direction) + + +class ContextSwitcher: + def __init__(self, llm, context_aggregator): + self._llm = llm + self._context_aggregator = context_aggregator + + async def switch_context(self, system_instruction): + """Switch the context to a new system instruction based on what the bot hears.""" + # Create messages with updated system instruction + messages = [ + { + "role": "system", + "content": system_instruction, + } + ] + + # Update context with new messages + self._context_aggregator.set_messages(messages) + # Get the context frame with the updated messages + context_frame = self._context_aggregator.get_context_frame() + # Trigger LLM response by pushing a context frame + await self._llm.push_frame(context_frame) + + +class FunctionHandlers: + def __init__(self, context_switcher): + self.context_switcher = context_switcher + + async def voicemail_response( + self, function_name, tool_call_id, args, llm, context, result_callback + ): + """Function the bot can call to leave a voicemail message.""" + message = """You are Chatbot leaving a voicemail message. Say EXACTLY this message and nothing else: + + "Hello, this is a message for Pipecat example user. This is Chatbot. Please call back on 123-456-7891. Thank you." + + After saying this message, call the terminate_call function.""" + + await self.context_switcher.switch_context(system_instruction=message) + + await result_callback("Leaving a voicemail message") + + async def human_conversation( + self, function_name, tool_call_id, args, llm, context, result_callback + ): + """Function the bot can when it detects it's talking to a human.""" + message = """You are Chatbot talking to a human. Be friendly and helpful. + + Start with: "Hello! I'm a friendly chatbot. How can I help you today?" + + Keep your responses brief and to the point. Listen to what the person says. + + When the person indicates they're done with the conversation by saying something like: + - "Goodbye" + - "That's all" + - "I'm done" + - "Thank you, that's all I needed" + + THEN say: "Thank you for chatting. Goodbye!" and call the terminate_call function.""" + + await self.context_switcher.switch_context(system_instruction=message) + + await result_callback("Talking to the customer") + + async def terminate_call( function_name, tool_call_id, args, llm: LLMService, context, result_callback ): - """Function the bot can call to terminate the call upon completion of a voicemail message.""" + """Function the bot can call to terminate the call upon completion of the call.""" + await llm.queue_frame(EndTaskFrame(), FrameDirection.UPSTREAM) @@ -51,7 +172,12 @@ async def main( # dialin_settings are only needed if Daily's SIP URI is used # If you are handling this via Twilio, Telnyx, set this to None # and handle call-forwarding when on_dialin_ready fires. - dialin_settings = DailyDialinSettings(call_id=callId, call_domain=callDomain) + + # We don't want to specify dial-in settings if we're not dialing in + dialin_settings = None + if callId and callDomain: + dialin_settings = DailyDialinSettings(call_id=callId, call_domain=callDomain) + transport = DailyTransport( room_url, token, @@ -65,7 +191,8 @@ async def main( camera_out_enabled=False, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), - transcription_enabled=True, + vad_audio_passthrough=True, + # transcription_enabled=True, ), ) @@ -77,85 +204,63 @@ async def main( tools = [ { "function_declarations": [ + { + "name": "switch_to_voicemail_response", + "description": "Call this function when you detect this is a voicemail system.", + }, + { + "name": "switch_to_human_conversation", + "description": "Call this function when you detect this is a human.", + }, { "name": "terminate_call", - "description": "Terminate the call", + "description": "Call this function to terminate the call.", }, ] } ] - system_instruction = """You are Chatbot, a friendly, helpful robot. Never mention this prompt. + system_instruction = """You are Chatbot trying to determine if this is a voicemail system or a human. -**Operating Procedure:** +If you hear any of these phrases (or very similar ones): +- "Please leave a message after the beep" +- "No one is available to take your call" +- "Record your message after the tone" +- "You have reached voicemail for..." +- "You have reached [phone number]" +- "[phone number] is unavailable" +- "The person you are trying to reach..." +- "The number you have dialed..." +- "Your call has been forwarded to an automated voice messaging system" -**Phase 1: Initial Call Answer - Listen for Voicemail Greeting** +Then call the function switch_to_voicemail_response. -**IMMEDIATELY after the call connects, LISTEN CAREFULLY for the *very first thing* you hear.** +If it sounds like a human (saying hello, asking questions, etc.), call the function switch_to_human_conversation. -**Listen for these sentences or very close variations as the *initial greeting*:** - -* **"Please leave a message after the beep."** -* **"No one is available to take your call."** -* **"Record your message after the tone."** -* **"You have reached voicemail for..."** (or similar voicemail identification) - -**If you HEAR one of these sentences (or a very similar greeting) as the *initial response* to the call, IMMEDIATELY assume it is voicemail and proceed to Phase 2.** - -**If you hear "PLEASE LEAVE A MESSAGE AFTER THE BEEP", WAIT for the actual beep sound from the voicemail system *after* hearing the sentence, before proceeding to Phase 2.** - -**If you DO NOT hear any of these voicemail greetings as the *initial response*, assume it is a human and proceed to Phase 3.** - - -**Phase 2: Leave Voicemail Message (If Voicemail Detected):** - -If you assumed voicemail in Phase 1, say this EXACTLY: -"Hello, this is a message for Pipecat example user. This is Chatbot. Please call back on 123-456-7891. Thank you." - -**Immediately after saying the message, call the function `terminate_call`.** -**DO NOT SAY ANYTHING ELSE. SILENCE IS REQUIRED AFTER `terminate_call`.** - - -**Phase 3: Human Interaction (If No Voicemail Greeting Detected in Phase 1):** - -If you did not detect a voicemail greeting in Phase 1 and a human answers, say: -"Oh, hello! I'm a friendly chatbot. Is there anything I can help you with?" - -Keep your responses **short and helpful.** - -If the human is finished, say: -"Okay, thank you! Have a great day!" - -**Then, immediately call the function `terminate_call`.** - - -**VERY IMPORTANT RULES - DO NOT DO THESE THINGS:** - -* **DO NOT SAY "Please leave a message after the beep."** -* **DO NOT SAY "No one is available to take your call."** -* **DO NOT SAY "Record your message after the tone."** -* **DO NOT SAY ANY voicemail greeting yourself.** -* **Only check for voicemail greetings in Phase 1, *immediately after the call connects*.** -* **After voicemail or human interaction, ALWAYS call `terminate_call` immediately.** -* **Do not speak after calling `terminate_call`.** -* Your speech will be audio, so use simple language without special characters. -""" +DO NOT say anything until you've determined if this is a voicemail or human.""" llm = GoogleLLMService( - model="models/gemini-2.0-flash-exp", + model="models/gemini-2.0-flash-lite-preview-02-05", api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, ) - llm.register_function("terminate_call", terminate_call) context = GoogleLLMContext() - context_aggregator = llm.create_context_aggregator(context) + audio_collector = UserAudioCollector(context, context_aggregator.user()) + + context_switcher = ContextSwitcher(llm, context_aggregator.user()) + handlers = FunctionHandlers(context_switcher) + + llm.register_function("switch_to_voicemail_response", handlers.voicemail_response) + llm.register_function("switch_to_human_conversation", handlers.human_conversation) + llm.register_function("terminate_call", terminate_call) pipeline = Pipeline( [ transport.input(), # Transport user input + audio_collector, # Collect audio frames context_aggregator.user(), # User responses llm, # LLM tts, # TTS From 6028f0f23a4904910470efe55d6e660decc576ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 17:19:30 -0800 Subject: [PATCH 201/427] PipelineTask: pass observers in contructor parameter --- CHANGELOG.md | 8 ++++++++ examples/instant-voice/server/src/single_bot.py | 6 ++---- examples/news-chatbot/server/news_bot.py | 6 ++---- examples/simple-chatbot/server/bot-gemini.py | 2 +- examples/simple-chatbot/server/bot-openai.py | 2 +- examples/translation-chatbot/bot.py | 2 +- src/pipecat/pipeline/task.py | 14 +++++++++++++- tests/test_pipeline.py | 2 +- 8 files changed, 29 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d8e18746..865f75676 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a new `PipelineTask` parameter `observers` that replaces the previous + `PipelineParams.observers`. + - Added a new `PipelineTask` parameter `check_dangling_tasks` to enable or disable checking for frame processors' dangling tasks when the Pipeline finishes running. @@ -45,6 +48,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ``` +### Deprecated + +- `PipelineParams.observers` is now deprecated, you the new `PipelineTask` + parameter `observers`. + ### Removed - Remove `TransportParams.audio_out_is_live` since it was not being used at all. diff --git a/examples/instant-voice/server/src/single_bot.py b/examples/instant-voice/server/src/single_bot.py index e1e9df68a..55ba20377 100644 --- a/examples/instant-voice/server/src/single_bot.py +++ b/examples/instant-voice/server/src/single_bot.py @@ -92,10 +92,8 @@ async def main(): task = PipelineTask( pipeline, - params=PipelineParams( - allow_interruptions=True, - observers=[rtvi.observer()], - ), + params=PipelineParams(allow_interruptions=True), + observers=[rtvi.observer()], ) @rtvi.event_handler("on_client_ready") diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index 7aed4c3b9..a34b0c3d0 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -140,10 +140,8 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( - allow_interruptions=True, - observers=[GoogleRTVIObserver(rtvi)], - ), + PipelineParams(allow_interruptions=True), + observers=[GoogleRTVIObserver(rtvi)], ) @rtvi.event_handler("on_client_ready") diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index ecf38f159..f80a9770e 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -176,8 +176,8 @@ async def main(): allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, - observers=[RTVIObserver(rtvi)], ), + observers=[RTVIObserver(rtvi)], ) await task.queue_frame(quiet_frame) diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index 51c421c7d..e1d7bb9a0 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -202,8 +202,8 @@ async def main(): allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, - observers=[RTVIObserver(rtvi)], ), + observers=[RTVIObserver(rtvi)], ) await task.queue_frame(quiet_frame) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index 8d4ee851e..5839afb5d 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -187,8 +187,8 @@ async def main(): allow_interruptions=False, # We don't want to interrupt the translator bot enable_metrics=True, enable_usage_metrics=True, - observers=[RTVIObserver(rtvi)], ), + observers=[RTVIObserver(rtvi)], ) @transport.event_handler("on_first_participant_joined") diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 1813878e1..873fc60d6 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -122,6 +122,7 @@ class PipelineTask(BaseTask): Args: pipeline: The pipeline to execute. params: Configuration parameters for the pipeline. + observers: List of observers for monitoring pipeline execution. clock: Clock implementation for timing operations. check_dangling_tasks: Whether to check for processors' tasks finishing properly. """ @@ -130,6 +131,7 @@ class PipelineTask(BaseTask): self, pipeline: BasePipeline, params: PipelineParams = PipelineParams(), + observers: List[BaseObserver] = [], clock: BaseClock = SystemClock(), check_dangling_tasks: bool = True, ): @@ -140,6 +142,16 @@ class PipelineTask(BaseTask): self._clock = clock self._params = params self._check_dangling_tasks = check_dangling_tasks + if self._params.observers: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Field 'observers' is deprecated, use the 'observers' parameter instead.", + DeprecationWarning, + ) + observers = self._params.observers self._finished = False # This queue receives frames coming from the pipeline upstream. @@ -163,7 +175,7 @@ class PipelineTask(BaseTask): self._task_manager = TaskManager() - self._observer = TaskObserver(observers=params.observers, task_manager=self._task_manager) + self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) @property def id(self) -> int: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 0aff922b2..07abef48e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -111,8 +111,8 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): params=PipelineParams( enable_heartbeats=True, heartbeats_period_secs=0.2, - observers=[heartbeats_observer], ), + observers=[heartbeats_observer], ) task.set_event_loop(asyncio.get_event_loop()) From 68789dfcf055093440e99cef3b02569e60d29183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 21:31:07 -0800 Subject: [PATCH 202/427] frames: add new StopFrame --- CHANGELOG.md | 7 +++++++ src/pipecat/frames/frames.py | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 865f75676..60f5c4de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a new `StopFrame` which can be used to stop a pipeline task while + keeping the frame processors running. The frame processors could then be used + in a different pipeline. The difference between a `StopFrame` and a + `StopTaskFrame` is that, as with `EndFrame` and `EndTaskFrame`, the + `StopFrame` is pushed from the task and the `StopTaskFrame` is pushed upstream + inside the pipeline by any processor. + - Added a new `PipelineTask` parameter `observers` that replaces the previous `PipelineParams.observers`. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index c9b812a1c..3e11a7a19 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -513,9 +513,9 @@ class CancelTaskFrame(SystemFrame): @dataclass class StopTaskFrame(SystemFrame): - """Indicates that a pipeline task should be stopped but that the pipeline - processors should be kept in a running state. This is normally queued from - the pipeline task. + """This is used to notify the pipeline task that it should be stopped as + soon as possible (flushing all the queued frames) but that the pipeline + processors should be kept in a running state. """ @@ -722,6 +722,17 @@ class EndFrame(ControlFrame): pass +@dataclass +class StopFrame(ControlFrame): + """Indicates that a pipeline should be stopped but that the pipeline + processors should be kept in a running state. This is normally queued from + the pipeline task. + + """ + + pass + + @dataclass class LLMFullResponseStartFrame(ControlFrame): """Used to indicate the beginning of an LLM response. Following by one or From 376d969a7702ac93f8e308307d1a451f7fc6d2c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 21:32:58 -0800 Subject: [PATCH 203/427] task: handle StopFrame and StopTaskFrame gracefully --- src/pipecat/pipeline/task.py | 44 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 873fc60d6..a2aad1e93 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -22,6 +22,7 @@ from pipecat.frames.frames import ( HeartbeatFrame, MetricsFrame, StartFrame, + StopFrame, StopTaskFrame, ) from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData @@ -163,9 +164,9 @@ class PipelineTask(BaseTask): # This is the heartbeat queue. When a heartbeat frame is received in the # down queue we add it to the heartbeat queue for processing. self._heartbeat_queue = asyncio.Queue() - # This event is used to indicate an EndFrame has been received in the - # down queue. - self._endframe_event = asyncio.Event() + # This event is used to indicate a finalize frame (e.g. EndFrame, + # StopFrame) has been received in the down queue. + self._pipeline_end_event = asyncio.Event() self._source = PipelineTaskSource(self._up_queue) self._source.link(pipeline) @@ -224,9 +225,12 @@ class PipelineTask(BaseTask): """Starts and manages the pipeline execution until completion or cancellation.""" if self.has_finished(): return + cleanup_pipeline = True try: push_task = await self._create_tasks() await self._task_manager.wait_for_task(push_task) + # We have already cleaned up the pipeline inside the task. + cleanup_pipeline = False except asyncio.CancelledError: # We are awaiting on the push task and it might be cancelled # (e.g. Ctrl-C). This means we will get a CancelledError here as @@ -234,7 +238,7 @@ class PipelineTask(BaseTask): # awaiting a task. pass await self._cancel_tasks() - await self._cleanup() + await self._cleanup(cleanup_pipeline) if self._check_dangling_tasks: self._print_dangling_tasks() self._finished = True @@ -305,13 +309,14 @@ class PipelineTask(BaseTask): data.append(ProcessingMetricsData(processor=p.name, value=0.0)) return MetricsFrame(data=data) - async def _wait_for_endframe(self): - await self._endframe_event.wait() - self._endframe_event.clear() + async def _wait_for_pipeline_end(self): + await self._pipeline_end_event.wait() + self._pipeline_end_event.clear() - async def _cleanup(self): + async def _cleanup(self, cleanup_pipeline: bool): await self._source.cleanup() - await self._pipeline.cleanup() + if cleanup_pipeline: + await self._pipeline.cleanup() await self._sink.cleanup() async def _process_push_queue(self): @@ -342,18 +347,16 @@ class PipelineTask(BaseTask): await self._source.queue_frame(self._initial_metrics_frame(), FrameDirection.DOWNSTREAM) running = True - should_cleanup = True + cleanup_pipeline = True while running: frame = await self._push_queue.get() await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM) - if isinstance(frame, EndFrame): - await self._wait_for_endframe() - running = not isinstance(frame, (CancelFrame, EndFrame, StopTaskFrame)) - should_cleanup = not isinstance(frame, StopTaskFrame) + if isinstance(frame, (EndFrame, StopFrame)): + await self._wait_for_pipeline_end() + running = not isinstance(frame, (CancelFrame, EndFrame, StopFrame)) + cleanup_pipeline = not isinstance(frame, StopFrame) self._push_queue.task_done() - # Cleanup only if we need to. - if should_cleanup: - await self._cleanup() + await self._cleanup(cleanup_pipeline) async def _process_up_queue(self): """This is the task that processes frames coming upstream from the @@ -371,7 +374,8 @@ class PipelineTask(BaseTask): # Tell the task we should end right away. await self.queue_frame(CancelFrame()) elif isinstance(frame, StopTaskFrame): - await self.queue_frame(StopTaskFrame()) + # Tell the task we should stop nicely. + await self.queue_frame(StopFrame()) elif isinstance(frame, ErrorFrame): logger.error(f"Error running app: {frame}") if frame.fatal: @@ -390,8 +394,8 @@ class PipelineTask(BaseTask): """ while True: frame = await self._down_queue.get() - if isinstance(frame, EndFrame): - self._endframe_event.set() + if isinstance(frame, (EndFrame, StopFrame)): + self._pipeline_end_event.set() elif isinstance(frame, HeartbeatFrame): await self._heartbeat_queue.put(frame) self._down_queue.task_done() From 699704732c2b0b2b01cfaa25ccd706c0f93908b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 21:35:11 -0800 Subject: [PATCH 204/427] asyncio: re-raise CancelledError in wait_for_task() --- src/pipecat/utils/asyncio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/utils/asyncio.py b/src/pipecat/utils/asyncio.py index ab40a8195..073ab0e50 100644 --- a/src/pipecat/utils/asyncio.py +++ b/src/pipecat/utils/asyncio.py @@ -80,6 +80,7 @@ class TaskManager: logger.warning(f"{name}: timed out waiting for task to finish") except asyncio.CancelledError: logger.trace(f"{name}: unexpected task cancellation (maybe Ctrl-C?)") + raise except Exception as e: logger.exception(f"{name}: unexpected exception while stopping task: {e}") finally: From 4536d03e829ca89496e8499cc29549cdb8c9bccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 21:35:43 -0800 Subject: [PATCH 205/427] FrameProcessor: cancel input/push tasks on CancelFrame --- src/pipecat/processors/frame_processor.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index f27c6d6c2..20a5071f4 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -240,7 +240,7 @@ class FrameProcessor: elif isinstance(frame, StopInterruptionFrame): self._should_report_ttfb = True elif isinstance(frame, CancelFrame): - self._cancelling = True + await self.__cancel(frame) async def push_error(self, error: ErrorFrame): await self.push_frame(error, FrameDirection.UPSTREAM) @@ -275,6 +275,11 @@ class FrameProcessor: self.__create_input_task() self.__create_push_task() + async def __cancel(self, frame: CancelFrame): + self._cancelling = True + await self.__cancel_input_task() + await self.__cancel_push_task() + # # Handle interruptions # From 1ec68bd071cab0879c1ff5895fdf4614221a640a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 22:32:52 -0800 Subject: [PATCH 206/427] make sure we don't create tasks if already created --- .../22b-natural-conversation-proposal.py | 8 +++-- .../22c-natural-conversation-mixed-llms.py | 8 +++-- .../22d-natural-conversation-gemini-audio.py | 8 +++-- src/pipecat/pipeline/parallel_pipeline.py | 31 +++++++++++++------ .../aggregators/gated_openai_llm_context.py | 8 +++-- .../processors/aggregators/llm_response.py | 3 +- src/pipecat/processors/frameworks/rtvi.py | 6 ++-- .../processors/idle_frame_processor.py | 9 ++++-- src/pipecat/processors/user_idle_processor.py | 4 +-- src/pipecat/services/ai_services.py | 12 ++++--- src/pipecat/services/cartesia.py | 4 +-- src/pipecat/services/elevenlabs.py | 10 ++++-- src/pipecat/services/fish.py | 3 +- src/pipecat/services/gladia.py | 12 +++++-- src/pipecat/services/lmnt.py | 4 ++- src/pipecat/services/playht.py | 3 +- src/pipecat/services/rime.py | 4 ++- src/pipecat/services/riva.py | 20 +++++++++--- src/pipecat/services/simli.py | 9 ++++-- src/pipecat/transports/base_input.py | 2 +- src/pipecat/transports/base_output.py | 12 ++++--- .../transports/network/fastapi_websocket.py | 10 ++++-- .../transports/network/websocket_server.py | 9 ++++-- src/pipecat/transports/services/daily.py | 2 +- src/pipecat/transports/services/livekit.py | 2 +- 25 files changed, 142 insertions(+), 61 deletions(-) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index a3c373eac..f828c2878 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -138,6 +138,7 @@ class OutputGate(FrameProcessor): self._gate_open = start_open self._frames_buffer = [] self._notifier = notifier + self._gate_task = None def close_gate(self): self._gate_open = False @@ -178,10 +179,13 @@ class OutputGate(FrameProcessor): async def _start(self): self._frames_buffer = [] - self._gate_task = self.create_task(self._gate_task_handler()) + if not self._gate_task: + self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): - await self.cancel_task(self._gate_task) + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None async def _gate_task_handler(self): while True: diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 92f47ae92..283a7e518 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -342,6 +342,7 @@ class OutputGate(FrameProcessor): self._gate_open = start_open self._frames_buffer = [] self._notifier = notifier + self._gate_task = None def close_gate(self): self._gate_open = False @@ -382,10 +383,13 @@ class OutputGate(FrameProcessor): async def _start(self): self._frames_buffer = [] - self._gate_task = self.create_task(self._gate_task_handler()) + if not self._gate_task: + self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): - await self.cancel_task(self._gate_task) + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None async def _gate_task_handler(self): while True: diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 15ee835e2..0e1feee57 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -555,6 +555,7 @@ class OutputGate(FrameProcessor): self._notifier = notifier self._context = context self._transcription_buffer = user_transcription_buffer + self._gate_task = None def close_gate(self): self._gate_open = False @@ -602,10 +603,13 @@ class OutputGate(FrameProcessor): async def _start(self): self._frames_buffer = [] - self._gate_task = self.create_task(self._gate_task_handler()) + if not self._gate_task: + self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): - await self.cancel_task(self._gate_task) + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None async def _gate_task_handler(self): while True: diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index f77d09cc6..c6f4de2df 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -81,6 +81,8 @@ class ParallelPipeline(BasePipeline): self._seen_ids = set() self._endframe_counter: Dict[int, int] = {} + self._up_task = None + self._down_task = None self._up_queue = asyncio.Queue() self._down_queue = asyncio.Queue() @@ -150,19 +152,30 @@ class ParallelPipeline(BasePipeline): await self._create_tasks() async def _stop(self): - # The up task doesn't receive an EndFrame, so we just cancel it. - await self.cancel_task(self._up_task) - # The down tasks waits for the last EndFrame sent by the internal - # pipelines. - await self._down_task + if self._up_task: + # The up task doesn't receive an EndFrame, so we just cancel it. + await self.cancel_task(self._up_task) + self._up_task = None + + if self._down_task: + # The down tasks waits for the last EndFrame sent by the internal + # pipelines. + await self._down_task + self._down_task = None async def _cancel(self): - await self.cancel_task(self._up_task) - await self.cancel_task(self._down_task) + if self._up_task: + await self.cancel_task(self._up_task) + self._up_task = None + if self._down_task: + await self.cancel_task(self._down_task) + self._down_task = None async def _create_tasks(self): - self._up_task = self.create_task(self._process_up_queue()) - self._down_task = self.create_task(self._process_down_queue()) + if not self._up_task: + self._up_task = self.create_task(self._process_up_queue()) + if not self._down_task: + self._down_task = self.create_task(self._process_down_queue()) async def _drain_queues(self): while not self._up_queue.empty: diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index f3ae78121..9973e3d02 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -21,6 +21,7 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._notifier = notifier self._start_open = start_open self._last_context_frame = None + self._gate_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -41,10 +42,13 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): await self.push_frame(frame, direction) async def _start(self): - self._gate_task = self.create_task(self._gate_task_handler()) + if not self._gate_task: + self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): - await self.cancel_task(self._gate_task) + if self._gate_task: + await self.cancel_task(self._gate_task) + self._gate_task = None async def _gate_task_handler(self): while True: diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index ae0b23507..f25bf7644 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -309,7 +309,8 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self._seen_interim_results = True def _create_aggregation_task(self): - self._aggregation_task = self.create_task(self._aggregation_task_handler()) + if not self._aggregation_task: + self._aggregation_task = self.create_task(self._aggregation_task_handler()) async def _cancel_aggregation_task(self): if self._aggregation_task: diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 5b28846a2..af55fbd42 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -956,8 +956,10 @@ class RTVIProcessor(FrameProcessor): await self._pipeline.cleanup() async def _start(self, frame: StartFrame): - self._action_task = self.create_task(self._action_task_handler()) - self._message_task = self.create_task(self._message_task_handler()) + if not self._action_task: + self._action_task = self.create_task(self._action_task_handler()) + if not self._message_task: + self._message_task = self.create_task(self._message_task_handler()) await self._call_event_handler("on_bot_started") async def _stop(self, frame: EndFrame): diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index 85ea215cf..ab98df771 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -30,6 +30,7 @@ class IdleFrameProcessor(FrameProcessor): self._callback = callback self._timeout = timeout self._types = types + self._idle_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -49,11 +50,13 @@ class IdleFrameProcessor(FrameProcessor): self._idle_event.set() async def cleanup(self): - await self.cancel_task(self._idle_task) + if self._idle_task: + await self.cancel_task(self._idle_task) def _create_idle_task(self): - self._idle_event = asyncio.Event() - self._idle_task = self.create_task(self._idle_task_handler()) + if not self._idle_task: + self._idle_event = asyncio.Event() + self._idle_task = self.create_task(self._idle_task_handler()) async def _idle_task_handler(self): while True: diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index e65bd437f..e88df3540 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -102,7 +102,7 @@ class UserIdleProcessor(FrameProcessor): def _create_idle_task(self) -> None: """Creates the idle task if it hasn't been created yet.""" - if self._idle_task is None: + if not self._idle_task: self._idle_task = self.create_task(self._idle_task_handler()) @property @@ -112,7 +112,7 @@ class UserIdleProcessor(FrameProcessor): async def _stop(self) -> None: """Stops and cleans up the idle monitoring task.""" - if self._idle_task is not None: + if self._idle_task: await self.cancel_task(self._idle_task) self._idle_task = None diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 8f0df1b37..594f7704d 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -273,7 +273,7 @@ class TTSService(AIService): async def start(self, frame: StartFrame): await super().start(frame) self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate - if self._push_stop_frames: + if self._push_stop_frames and not self._stop_frame_task: self._stop_frame_task = self.create_task(self._stop_frame_handler()) async def stop(self, frame: EndFrame): @@ -484,7 +484,8 @@ class WordTTSService(TTSService): self.reset_word_timestamps() def _create_words_task(self): - self._words_task = self.create_task(self._words_task_handler()) + if not self._words_task: + self._words_task = self.create_task(self._words_task_handler()) async def _stop_words_task(self): if self._words_task: @@ -660,9 +661,10 @@ class AudioContextWordTTSService(WebsocketWordTTSService): self._create_audio_context_task() def _create_audio_context_task(self): - self._contexts_queue = asyncio.Queue() - self._contexts: Dict[str, asyncio.Queue] = {} - self._audio_context_task = self.create_task(self._audio_context_task_handler()) + if not self._audio_context_task: + self._contexts_queue = asyncio.Queue() + self._contexts: Dict[str, asyncio.Queue] = {} + self._audio_context_task = self.create_task(self._audio_context_task_handler()) async def _stop_audio_context_task(self): if self._audio_context_task: diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 0ec55aa0c..3aafb4023 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -182,8 +182,8 @@ class CartesiaTTSService(AudioContextWordTTSService): async def _connect(self): await self._connect_websocket() - - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): if self._receive_task: diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 47fa87b5a..6c808f66b 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -214,6 +214,9 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): self._started = False self._cumulative_time = 0 + self._receive_task = None + self._keepalive_task = None + def can_generate_metrics(self) -> bool: return True @@ -286,8 +289,11 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) - self._keepalive_task = self.create_task(self._keepalive_task_handler()) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + + if not self._keepalive_task: + self._keepalive_task = self.create_task(self._keepalive_task_handler()) async def _disconnect(self): if self._receive_task: diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 1c5e6a08a..645ee1496 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -106,7 +106,8 @@ class FishAudioTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): if self._receive_task: diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index c93dfef3b..2fbf31882 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -172,6 +172,7 @@ class GladiaSTTService(STTService): }, } self._confidence = confidence + self._receive_task = None def language_to_service_language(self, language: Language) -> Optional[str]: return language_to_gladia_language(language) @@ -181,18 +182,23 @@ class GladiaSTTService(STTService): self._settings["sample_rate"] = self.sample_rate response = await self._setup_gladia() self._websocket = await websockets.connect(response["url"]) - self._receive_task = self.create_task(self._receive_task_handler()) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) await self._send_stop_recording() await self._websocket.close() - await self.wait_for_task(self._receive_task) + if self._receive_task: + await self.wait_for_task(self._receive_task) + self._receive_task = None async def cancel(self, frame: CancelFrame): await super().cancel(frame) await self._websocket.close() - await self.cancel_task(self._receive_task) + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: await self.start_processing_metrics() diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index dd6b9e3b8..4b3a3ce34 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -83,6 +83,7 @@ class LmntTTSService(InterruptibleTTSService): "format": "raw", # Use raw format for direct PCM data } self._started = False + self._receive_task = None def can_generate_metrics(self) -> bool: return True @@ -110,7 +111,8 @@ class LmntTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): if self._receive_task: diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 828acbdb7..4b4bd6b10 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -159,7 +159,8 @@ class PlayHTTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): if self._receive_task: diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 6e2daccd6..b9a2d20da 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -165,7 +165,9 @@ class RimeTTSService(AudioContextWordTTSService): async def _connect(self): """Establish websocket connection and start receive task.""" await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + + if not self._receive_task: + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) async def _disconnect(self): """Close websocket connection and clean up tasks.""" diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 458cb4919..89c965c12 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -166,6 +166,8 @@ class ParakeetSTTService(STTService): self._asr_service = riva.client.ASRService(auth) self._queue = asyncio.Queue() + self._thread_task = None + self._response_task = None def can_generate_metrics(self) -> bool: return False @@ -205,9 +207,12 @@ class ParakeetSTTService(STTService): self._config = config - self._thread_task = self.create_task(self._thread_task_handler()) - self._response_task = self.create_task(self._response_task_handler()) - self._response_queue = asyncio.Queue() + if not self._thread_task: + self._thread_task = self.create_task(self._thread_task_handler()) + + if not self._response_task: + self._response_queue = asyncio.Queue() + self._response_task = self.create_task(self._response_task_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) @@ -218,8 +223,13 @@ class ParakeetSTTService(STTService): await self._stop_tasks() async def _stop_tasks(self): - await self.cancel_task(self._thread_task) - await self.cancel_task(self._response_task) + if self._thread_task: + await self.cancel_task(self._thread_task) + self._thread_task = None + + if self._response_task: + await self.cancel_task(self._response_task) + self._response_task = None def _response_handler(self): responses = self._asr_service.streaming_response_generator( diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index 01aaeedfb..10f7ae68c 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -49,8 +49,11 @@ class SimliVideoService(FrameProcessor): async def _start_connection(self): await self._simli_client.Initialize() # Create task to consume and process audio and video - self._audio_task = self.create_task(self._consume_and_process_audio()) - self._video_task = self.create_task(self._consume_and_process_video()) + if not self._audio_task: + self._audio_task = self.create_task(self._consume_and_process_audio()) + + if not self._video_task: + self._video_task = self.create_task(self._consume_and_process_video()) async def _consume_and_process_audio(self): await self._pipecat_resampler_event.wait() @@ -117,5 +120,7 @@ class SimliVideoService(FrameProcessor): await self._simli_client.stop() if self._audio_task: await self.cancel_task(self._audio_task) + self._audio_task = None if self._video_task: await self.cancel_task(self._video_task) + self._video_task = None diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index ee7ae7f7c..782ad1333 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -74,7 +74,7 @@ class BaseInputTransport(FrameProcessor): if self._params.audio_in_filter: await self._params.audio_in_filter.start(self._sample_rate) # Create audio input queue and task if needed. - if self._params.audio_in_enabled or self._params.vad_enabled: + if not self._audio_task and (self._params.audio_in_enabled or self._params.vad_enabled): self._audio_in_queue = asyncio.Queue() self._audio_task = self.create_task(self._audio_task_handler()) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 1b6d1e833..2322ade9d 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -238,10 +238,12 @@ class BaseOutputTransport(FrameProcessor): # def _create_sink_tasks(self): - self._sink_queue = asyncio.Queue() - self._sink_clock_queue = asyncio.PriorityQueue() - self._sink_task = self.create_task(self._sink_task_handler()) - self._sink_clock_task = self.create_task(self._sink_clock_task_handler()) + if not self._sink_task: + self._sink_queue = asyncio.Queue() + self._sink_task = self.create_task(self._sink_task_handler()) + if not self._sink_clock_task: + self._sink_clock_queue = asyncio.PriorityQueue() + self._sink_clock_task = self.create_task(self._sink_clock_task_handler()) async def _cancel_sink_tasks(self): # Stop sink tasks. @@ -358,7 +360,7 @@ class BaseOutputTransport(FrameProcessor): def _create_camera_task(self): # Create camera output queue and task if needed. - if self._params.camera_out_enabled: + if not self._camera_out_task and self._params.camera_out_enabled: self._camera_out_queue = asyncio.Queue() self._camera_out_task = self.create_task(self._camera_out_task_handler()) diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index 565d1906c..bfb1e8146 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -115,15 +115,19 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): await super().start(frame) await self._params.serializer.setup(frame) - if self._params.session_timeout: + if not self._monitor_websocket_task and self._params.session_timeout: self._monitor_websocket_task = self.create_task(self._monitor_websocket()) await self._client.trigger_client_connected() - self._receive_task = self.create_task(self._receive_messages()) + if not self._receive_task: + self._receive_task = self.create_task(self._receive_messages()) async def _stop_tasks(self): if self._monitor_websocket_task: await self.cancel_task(self._monitor_websocket_task) - await self.cancel_task(self._receive_task) + self._monitor_websocket_task = None + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None async def stop(self, frame: EndFrame): await super().stop(frame) diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index ca5d07d2e..4d1e07230 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -81,22 +81,27 @@ class WebsocketServerInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): await super().start(frame) await self._params.serializer.setup(frame) - self._server_task = self.create_task(self._server_task_handler()) + if not self._server_task: + self._server_task = self.create_task(self._server_task_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) self._stop_server_event.set() if self._monitor_task: await self.cancel_task(self._monitor_task) + self._monitor_task = None if self._server_task: await self.wait_for_task(self._server_task) + self._server_task = None async def cancel(self, frame: CancelFrame): await super().cancel(frame) if self._monitor_task: await self.cancel_task(self._monitor_task) + self._monitor_task = None if self._server_task: await self.cancel_task(self._server_task) + self._server_task = None async def _server_task_handler(self): logger.info(f"Starting websocket server on {self._host}:{self._port}") @@ -116,7 +121,7 @@ class WebsocketServerInputTransport(BaseInputTransport): await self._callbacks.on_client_connected(websocket) # Create a task to monitor the websocket connection - if self._params.session_timeout: + if not self._monitor_task and self._params.session_timeout: self._monitor_task = self.create_task( self._monitor_websocket(websocket, self._params.session_timeout) ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7ce329659..0bb0dfd6d 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -845,7 +845,7 @@ class DailyInputTransport(BaseInputTransport): def start_audio_in_streaming(self): # Create audio task. It reads audio frames from Daily and push them # internally for VAD processing. - if self._params.audio_in_enabled or self._params.vad_enabled: + if not self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): logger.debug(f"Start receiving audio") self._audio_in_task = self.create_task(self._audio_in_task_handler()) diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index f104e99c8..9d26e1bfc 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -360,7 +360,7 @@ class LiveKitInputTransport(BaseInputTransport): await super().start(frame) await self._client.setup(frame) await self._client.connect() - if self._params.audio_in_enabled or self._params.vad_enabled: + if not self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): self._audio_in_task = self.create_task(self._audio_in_task_handler()) logger.info("LiveKitInputTransport started") From fb7fe540f52e1602b8fe5aa1d3c676e8f2fc6467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 22:33:13 -0800 Subject: [PATCH 207/427] tts: don't connect to websocket if already connected --- src/pipecat/services/cartesia.py | 2 ++ src/pipecat/services/elevenlabs.py | 3 +++ src/pipecat/services/fish.py | 3 +++ src/pipecat/services/lmnt.py | 3 +++ src/pipecat/services/playht.py | 3 +++ src/pipecat/services/rime.py | 3 +++ 6 files changed, 17 insertions(+) diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 3aafb4023..04a35087b 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -194,6 +194,8 @@ class CartesiaTTSService(AudioContextWordTTSService): async def _connect_websocket(self): try: + if self._websocket: + return logger.debug("Connecting to Cartesia") self._websocket = await websockets.connect( f"{self._url}?api_key={self._api_key}&cartesia_version={self._cartesia_version}" diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 6c808f66b..3dc42975e 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -308,6 +308,9 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): async def _connect_websocket(self): try: + if self._websocket: + return + logger.debug("Connecting to ElevenLabs") voice_id = self._voice_id diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 645ee1496..e61579bfb 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -118,6 +118,9 @@ class FishAudioTTSService(InterruptibleTTSService): async def _connect_websocket(self): try: + if self._websocket: + return + logger.debug("Connecting to Fish Audio") headers = {"Authorization": f"Bearer {self._api_key}"} self._websocket = await websockets.connect(self._base_url, extra_headers=headers) diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 4b3a3ce34..8d4ca4803 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -124,6 +124,9 @@ class LmntTTSService(InterruptibleTTSService): async def _connect_websocket(self): """Connect to LMNT websocket.""" try: + if self._websocket: + return + logger.debug("Connecting to LMNT") # Build initial connection message diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 4b4bd6b10..56179e34c 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -171,6 +171,9 @@ class PlayHTTTSService(InterruptibleTTSService): async def _connect_websocket(self): try: + if self._websocket: + return + logger.debug("Connecting to PlayHT") if not self._websocket_url: diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index b9a2d20da..60d2d67ef 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -180,6 +180,9 @@ class RimeTTSService(AudioContextWordTTSService): async def _connect_websocket(self): """Connect to Rime websocket API with configured settings.""" try: + if self._websocket: + return + params = "&".join(f"{k}={v}" for k, v in self._settings.items()) url = f"{self._url}?{params}" headers = {"Authorization": f"Bearer {self._api_key}"} From d2f006682c4158f6463c12c22757cbc3b32fa975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Feb 2025 23:19:57 -0800 Subject: [PATCH 208/427] introduce new BaseTaskManager --- src/pipecat/frames/frames.py | 4 +- src/pipecat/pipeline/task.py | 7 +- src/pipecat/pipeline/task_observer.py | 4 +- src/pipecat/processors/frame_processor.py | 6 +- .../transports/network/websocket_client.py | 6 +- src/pipecat/transports/services/daily.py | 4 +- src/pipecat/transports/services/livekit.py | 4 +- src/pipecat/utils/asyncio.py | 66 ++++++++++++++++++- 8 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 3e11a7a19..b43aaaeae 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -23,7 +23,7 @@ from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.clocks.base_clock import BaseClock from pipecat.metrics.metrics import MetricsData from pipecat.transcriptions.language import Language -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager from pipecat.utils.time import nanoseconds_to_str from pipecat.utils.utils import obj_count, obj_id @@ -438,7 +438,7 @@ class StartFrame(SystemFrame): """This is the first frame that should be pushed down a pipeline.""" clock: BaseClock - task_manager: TaskManager + task_manager: BaseTaskManager audio_in_sample_rate: int = 16000 audio_out_sample_rate: int = 24000 allow_interruptions: bool = False diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index a2aad1e93..da18cca5d 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -5,7 +5,7 @@ # import asyncio -from typing import Any, AsyncIterable, Dict, Iterable, List +from typing import Any, AsyncIterable, Dict, Iterable, List, Optional from loguru import logger from pydantic import BaseModel, ConfigDict @@ -31,7 +31,7 @@ from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.base_task import BaseTask from pipecat.pipeline.task_observer import TaskObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager, TaskManager from pipecat.utils.utils import obj_count, obj_id HEARTBEAT_SECONDS = 1.0 @@ -134,6 +134,7 @@ class PipelineTask(BaseTask): params: PipelineParams = PipelineParams(), observers: List[BaseObserver] = [], clock: BaseClock = SystemClock(), + task_manager: Optional[BaseTaskManager] = None, check_dangling_tasks: bool = True, ): self._id: int = obj_id() @@ -174,7 +175,7 @@ class PipelineTask(BaseTask): self._sink = PipelineTaskSink(self._down_queue) pipeline.link(self._sink) - self._task_manager = TaskManager() + self._task_manager = task_manager or TaskManager() self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index abcbf513b..122038386 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -12,7 +12,7 @@ from attr import dataclass from pipecat.frames.frames import Frame from pipecat.observers.base_observer import BaseObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager from pipecat.utils.utils import obj_count, obj_id @@ -55,7 +55,7 @@ class TaskObserver(BaseObserver): """ - def __init__(self, *, observers: List[BaseObserver] = [], task_manager: TaskManager): + def __init__(self, *, observers: List[BaseObserver] = [], task_manager: BaseTaskManager): self._id: int = obj_id() self._name: str = f"{self.__class__.__name__}#{obj_count(self)}" self._observers = observers diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 20a5071f4..335585ec3 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -23,7 +23,7 @@ from pipecat.frames.frames import ( ) from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager from pipecat.utils.utils import obj_count, obj_id @@ -52,7 +52,7 @@ class FrameProcessor: self._clock: Optional[BaseClock] = None # Task Manager - self._task_manager: Optional[TaskManager] = None + self._task_manager: Optional[BaseTaskManager] = None # Other properties self._allow_interruptions = False @@ -192,7 +192,7 @@ class FrameProcessor: raise Exception(f"{self} Clock is still not initialized.") return self._clock - def get_task_manager(self) -> TaskManager: + def get_task_manager(self) -> BaseTaskManager: if not self._task_manager: raise Exception(f"{self} TaskManager is still not initialized.") return self._task_manager diff --git a/src/pipecat/transports/network/websocket_client.py b/src/pipecat/transports/network/websocket_client.py index d7db9c4bd..eb2b5cfb8 100644 --- a/src/pipecat/transports/network/websocket_client.py +++ b/src/pipecat/transports/network/websocket_client.py @@ -30,7 +30,7 @@ from pipecat.serializers.protobuf import ProtobufFrameSerializer from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager class WebsocketClientParams(TransportParams): @@ -57,12 +57,12 @@ class WebsocketClientSession: self._callbacks = callbacks self._transport_name = transport_name - self._task_manager: Optional[TaskManager] = None + self._task_manager: Optional[BaseTaskManager] = None self._websocket: Optional[websockets.WebSocketClientProtocol] = None @property - def task_manager(self) -> TaskManager: + def task_manager(self) -> BaseTaskManager: if not self._task_manager: raise Exception( f"{self._transport_name}::WebsocketClientSession: TaskManager not initialized (pipeline not started?)" diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 0bb0dfd6d..1d31f348b 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -43,7 +43,7 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager try: from daily import CallClient, Daily, EventHandler @@ -293,7 +293,7 @@ class DailyTransportClient(EventHandler): self._joined_event = asyncio.Event() self._leave_counter = 0 - self._task_manager: Optional[TaskManager] = None + self._task_manager: Optional[BaseTaskManager] = None # We use the executor to cleanup the client. We just do it from one # place, so only one thread is really needed. diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 9d26e1bfc..7018ea520 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -27,7 +27,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio import BaseTaskManager try: from livekit import rtc @@ -88,7 +88,7 @@ class LiveKitTransportClient: self._audio_tracks = {} self._audio_queue = asyncio.Queue() self._other_participant_has_joined = False - self._task_manager: Optional[TaskManager] = None + self._task_manager: Optional[BaseTaskManager] = None @property def participant_id(self) -> str: diff --git a/src/pipecat/utils/asyncio.py b/src/pipecat/utils/asyncio.py index 073ab0e50..acc4acec8 100644 --- a/src/pipecat/utils/asyncio.py +++ b/src/pipecat/utils/asyncio.py @@ -5,12 +5,76 @@ # import asyncio +from abc import ABC, abstractmethod from typing import Coroutine, Optional, Set from loguru import logger -class TaskManager: +class BaseTaskManager(ABC): + @abstractmethod + def set_event_loop(self, loop: asyncio.AbstractEventLoop): + pass + + @abstractmethod + def get_event_loop(self) -> asyncio.AbstractEventLoop: + pass + + @abstractmethod + def create_task(self, coroutine: Coroutine, name: str) -> asyncio.Task: + """ + Creates and schedules a new asyncio Task that runs the given coroutine. + + The task is added to a global set of created tasks. + + Args: + loop (asyncio.AbstractEventLoop): The event loop to use for creating the task. + coroutine (Coroutine): The coroutine to be executed within the task. + name (str): The name to assign to the task for identification. + + Returns: + asyncio.Task: The created task object. + """ + pass + + @abstractmethod + async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None): + """Wait for an asyncio.Task to complete with optional timeout handling. + + This function awaits the specified asyncio.Task and handles scenarios for + timeouts, cancellations, and other exceptions. It also ensures that the task + is removed from the set of registered tasks upon completion or failure. + + Args: + task (asyncio.Task): The asyncio Task to wait for. + timeout (Optional[float], optional): The maximum number of seconds + to wait for the task to complete. If None, waits indefinitely. + Defaults to None. + """ + pass + + @abstractmethod + async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): + """Cancels the given asyncio Task and awaits its completion with an + optional timeout. + + This function removes the task from the set of registered tasks upon + completion or failure. + + Args: + task (asyncio.Task): The task to be cancelled. + timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel. + + """ + pass + + @abstractmethod + def current_tasks(self) -> Set[asyncio.Task]: + """Returns the list of currently created/registered tasks.""" + pass + + +class TaskManager(BaseTaskManager): def __init__(self) -> None: self._tasks: Set[asyncio.Task] = set() self._loop: Optional[asyncio.AbstractEventLoop] = None From 36a729cbfe8a6c214ebea099eb757b784dba491c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 25 Feb 2025 10:00:45 -0500 Subject: [PATCH 209/427] Set grok-2 as default model for GrokLLMSService --- CHANGELOG.md | 4 +++- src/pipecat/services/grok.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 865f75676..aba68530d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [Unreleased] ### Added @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `GrokLLMSService` now uses `grok-2` as the default model. + - `AnthropicLLMService` now uses `claude-3-7-sonnet-20250219` as the default model. diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 0011a1f4e..3a4a5fb5e 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -125,7 +125,7 @@ class GrokLLMService(OpenAILLMService): Args: api_key (str): The API key for accessing Grok's API base_url (str, optional): The base URL for Grok API. Defaults to "https://api.x.ai/v1" - model (str, optional): The model identifier to use. Defaults to "grok-beta" + model (str, optional): The model identifier to use. Defaults to "grok-2" **kwargs: Additional keyword arguments passed to OpenAILLMService """ @@ -134,7 +134,7 @@ class GrokLLMService(OpenAILLMService): *, api_key: str, base_url: str = "https://api.x.ai/v1", - model: str = "grok-beta", + model: str = "grok-2", **kwargs, ): super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) From cfe72143b8052f30c238eadf16fbe3357296c61f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 25 Feb 2025 10:54:25 -0500 Subject: [PATCH 210/427] Example 30: Move observers to PipelineTask --- examples/foundational/30-observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 0988be76d..a6be27b97 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -122,8 +122,8 @@ async def main(): enable_metrics=True, enable_usage_metrics=True, report_only_initial_ttfb=True, - observers=[DebugObserver(), LLMLogObserver()], ), + observers=[DebugObserver(), LLMLogObserver()], ) @transport.event_handler("on_first_participant_joined") From e60b65228bb5928a26d6b4af8afa88547e1662d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 09:52:54 -0800 Subject: [PATCH 211/427] allow multiple StartFrames --- .../22d-natural-conversation-gemini-audio.py | 2 -- .../processors/aggregators/llm_response.py | 8 +++-- src/pipecat/services/ai_services.py | 3 +- src/pipecat/services/assemblyai.py | 3 ++ src/pipecat/services/azure.py | 33 ++++++++++++++++--- src/pipecat/services/gladia.py | 7 +++- src/pipecat/services/riva.py | 4 +++ src/pipecat/services/simli.py | 6 +++- src/pipecat/services/xtts.py | 4 +++ src/pipecat/transports/local/audio.py | 7 ++++ src/pipecat/transports/local/tk.py | 8 +++++ .../transports/network/websocket_server.py | 1 - src/pipecat/transports/services/daily.py | 18 ++++++++++ tests/integration/integration_openai_llm.py | 2 +- 14 files changed, 92 insertions(+), 14 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 0e1feee57..dde280fc1 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -25,10 +25,8 @@ from pipecat.frames.frames import ( InputAudioRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, - LLMMessagesFrame, StartFrame, StartInterruptionFrame, - StopInterruptionFrame, SystemFrame, TextFrame, TranscriptionFrame, diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index f25bf7644..da8ca63aa 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -246,11 +246,15 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): + # Push StartFrame before start(), because we want StartFrame to be + # processed by every processor before any other frame is processed. + await self.push_frame(frame, direction) await self._start(frame) - await self.push_frame(frame, direction) elif isinstance(frame, EndFrame): - await self._stop(frame) + # Push EndFrame before stop(), because stop() waits on the task to + # finish and the task finishes when EndFrame is processed. await self.push_frame(frame, direction) + await self._stop(frame) elif isinstance(frame, CancelFrame): await self._cancel(frame) await self.push_frame(frame, direction) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 594f7704d..71b1e7a92 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -840,7 +840,8 @@ class SegmentedSTTService(STTService): async def start(self, frame: StartFrame): await super().start(frame) - (self._content, self._wave) = self._new_wave() + if not self._wave: + (self._content, self._wave) = self._new_wave() async def stop(self, frame: EndFrame): await super().stop(frame) diff --git a/src/pipecat/services/assemblyai.py b/src/pipecat/services/assemblyai.py index 84113f582..94f081c5b 100644 --- a/src/pipecat/services/assemblyai.py +++ b/src/pipecat/services/assemblyai.py @@ -91,6 +91,9 @@ class AssemblyAISTTService(STTService): AssemblyAI transcriber. """ + if self._transcriber: + return + def on_open(session_opened: aai.RealtimeSessionOpened): """Callback for when the connection to AssemblyAI is opened.""" logger.info(f"{self}: Connected to AssemblyAI") diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index fbdd08ab1..11c34214c 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -535,6 +535,10 @@ class AzureTTSService(AzureBaseTTSService): async def start(self, frame: StartFrame): await super().start(frame) + + if self._speech_config: + return + # Now self.sample_rate is properly initialized self._speech_config = SpeechConfig( subscription=self._api_key, @@ -624,6 +628,10 @@ class AzureHttpTTSService(AzureBaseTTSService): async def start(self, frame: StartFrame): await super().start(frame) + + if self._speech_config: + return + self._speech_config = SpeechConfig( subscription=self._api_key, region=self._region, @@ -678,15 +686,22 @@ class AzureSTTService(STTService): self._speech_config = SpeechConfig(subscription=api_key, region=region) self._speech_config.speech_recognition_language = language + self._audio_stream = None + self._speech_recognizer = None + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: await self.start_processing_metrics() - self._audio_stream.write(audio) + if self._audio_stream: + self._audio_stream.write(audio) await self.stop_processing_metrics() yield None async def start(self, frame: StartFrame): await super().start(frame) + if self._audio_stream: + return + stream_format = AudioStreamFormat(samples_per_second=self.sample_rate, channels=1) self._audio_stream = PushAudioInputStream(stream_format) @@ -700,13 +715,21 @@ class AzureSTTService(STTService): async def stop(self, frame: EndFrame): await super().stop(frame) - self._speech_recognizer.stop_continuous_recognition_async() - self._audio_stream.close() + + if self._speech_recognizer: + self._speech_recognizer.stop_continuous_recognition_async() + + if self._audio_stream: + self._audio_stream.close() async def cancel(self, frame: CancelFrame): await super().cancel(frame) - self._speech_recognizer.stop_continuous_recognition_async() - self._audio_stream.close() + + if self._speech_recognizer: + self._speech_recognizer.stop_continuous_recognition_async() + + if self._audio_stream: + self._audio_stream.close() def _on_handle_recognized(self, event): if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0: diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index 2fbf31882..f0a00b327 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -172,6 +172,7 @@ class GladiaSTTService(STTService): }, } self._confidence = confidence + self._websocket = None self._receive_task = None def language_to_service_language(self, language: Language) -> Optional[str]: @@ -179,6 +180,8 @@ class GladiaSTTService(STTService): async def start(self, frame: StartFrame): await super().start(frame) + if self._websocket: + return self._settings["sample_rate"] = self.sample_rate response = await self._setup_gladia() self._websocket = await websockets.connect(response["url"]) @@ -188,7 +191,9 @@ class GladiaSTTService(STTService): async def stop(self, frame: EndFrame): await super().stop(frame) await self._send_stop_recording() - await self._websocket.close() + if self._websocket: + await self._websocket.close() + self._websocket = None if self._receive_task: await self.wait_for_task(self._receive_task) self._receive_task = None diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 89c965c12..8d5ce79c4 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -166,6 +166,7 @@ class ParakeetSTTService(STTService): self._asr_service = riva.client.ASRService(auth) self._queue = asyncio.Queue() + self._config = None self._thread_task = None self._response_task = None @@ -175,6 +176,9 @@ class ParakeetSTTService(STTService): async def start(self, frame: StartFrame): await super().start(frame) + if self._config: + return + config = riva.client.StreamingRecognitionConfig( config=riva.client.RecognitionConfig( encoding=riva.client.AudioEncoding.LINEAR_PCM, diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index 10f7ae68c..0fb490914 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -43,11 +43,15 @@ class SimliVideoService(FrameProcessor): self._pipecat_resampler: AudioResampler = None self._simli_resampler = AudioResampler("s16", "mono", 16000) + self._initialized = False self._audio_task: asyncio.Task = None self._video_task: asyncio.Task = None async def _start_connection(self): - await self._simli_client.Initialize() + if not self._initialized: + await self._simli_client.Initialize() + self._initialized = True + # Create task to consume and process audio and video if not self._audio_task: self._audio_task = self.create_task(self._consume_and_process_audio()) diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 4d0f3fca6..9d2f3dd1e 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -99,6 +99,10 @@ class XTTSService(TTSService): async def start(self, frame: StartFrame): await super().start(frame) + + if self._studio_speakers: + return + async with self._aiohttp_session.get(self._settings["base_url"] + "/studio_speakers") as r: if r.status != 200: text = await r.text() diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index e7b803f2b..bc8d2dd16 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -44,6 +44,9 @@ class LocalAudioInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): await super().start(frame) + if self._in_stream: + return + self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate num_frames = int(self._sample_rate / 100) * 2 # 20ms of audio @@ -94,6 +97,9 @@ class LocalAudioOutputTransport(BaseOutputTransport): async def start(self, frame: StartFrame): await super().start(frame) + if self._out_stream: + return + self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate self._out_stream = self._py_audio.open( @@ -110,6 +116,7 @@ class LocalAudioOutputTransport(BaseOutputTransport): if self._out_stream: self._out_stream.stop_stream() self._out_stream.close() + self._out_stream = None async def write_raw_audio_frames(self, frames: bytes): if self._out_stream: diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index 73777c573..d270d6395 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -51,6 +51,9 @@ class TkInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): await super().start(frame) + if self._in_stream: + return + self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate num_frames = int(self._sample_rate / 100) * 2 # 20ms of audio @@ -70,6 +73,7 @@ class TkInputTransport(BaseInputTransport): if self._in_stream: self._in_stream.stop_stream() self._in_stream.close() + self._in_stream = None def _audio_in_callback(self, in_data, frame_count, time_info, status): frame = InputAudioRawFrame( @@ -106,6 +110,9 @@ class TkOutputTransport(BaseOutputTransport): async def start(self, frame: StartFrame): await super().start(frame) + if self._out_stream: + return + self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate self._out_stream = self._py_audio.open( @@ -122,6 +129,7 @@ class TkOutputTransport(BaseOutputTransport): if self._out_stream: self._out_stream.stop_stream() self._out_stream.close() + self._out_stream = None async def write_raw_audio_frames(self, frames: bytes): if self._out_stream: diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 4d1e07230..19ebe4a45 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -22,7 +22,6 @@ from pipecat.frames.frames import ( OutputAudioRawFrame, StartFrame, StartInterruptionFrame, - TextFrame, TransportMessageFrame, TransportMessageUrgentFrame, ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 1d31f348b..7735ae91f 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -832,6 +832,9 @@ class DailyInputTransport(BaseInputTransport): self._video_renderers = {} + # Whether we have seen a StartFrame already. + self._initialized = False + # Task that gets audio data from a device or the network and queues it # internally to be processed. self._audio_in_task = None @@ -852,6 +855,12 @@ class DailyInputTransport(BaseInputTransport): async def start(self, frame: StartFrame): # Parent start. await super().start(frame) + + if self._initialized: + return + + self._initialized = True + # Setup client. await self._client.setup(frame) # Join the room. @@ -980,9 +989,18 @@ class DailyOutputTransport(BaseOutputTransport): self._client = client + # Whether we have seen a StartFrame already. + self._initialized = False + async def start(self, frame: StartFrame): # Parent start. await super().start(frame) + + if self._initialized: + return + + self._initialized = True + # Setup client. await self._client.setup(frame) # Join the room. diff --git a/tests/integration/integration_openai_llm.py b/tests/integration/integration_openai_llm.py index c788936a1..2c84c8882 100644 --- a/tests/integration/integration_openai_llm.py +++ b/tests/integration/integration_openai_llm.py @@ -10,7 +10,7 @@ from openai.types.chat import ( ) from pipecat.frames.frames import LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.processors.frame_processor import FrameDirection from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService from pipecat.utils.test_frame_processor import TestFrameProcessor From 34a50033cb475bbfb8130573267d021722e02ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 10:24:24 -0800 Subject: [PATCH 212/427] tk: use TkTransportParams in examples --- examples/foundational/03a-local-still-frame.py | 7 ++++--- examples/foundational/05a-local-sync-speech-and-image.py | 5 ++--- examples/foundational/09a-local-mirror.py | 5 ++--- src/pipecat/transports/local/tk.py | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/examples/foundational/03a-local-still-frame.py b/examples/foundational/03a-local-still-frame.py index 8143c4765..59cc27b7b 100644 --- a/examples/foundational/03a-local-still-frame.py +++ b/examples/foundational/03a-local-still-frame.py @@ -18,8 +18,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.services.fal import FalImageGenService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.local.tk import TkLocalTransport +from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams load_dotenv(override=True) @@ -34,7 +33,9 @@ async def main(): transport = TkLocalTransport( tk_root, - TransportParams(camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024), + TkTransportParams( + camera_out_enabled=True, camera_out_width=1024, camera_out_height=1024 + ), ) imagegen = FalImageGenService( diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 479ddfa94..7bd689fe6 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -30,8 +30,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService from pipecat.services.fal import FalImageGenService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.local.tk import TkLocalTransport, TkOutputTransport +from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams load_dotenv(override=True) @@ -152,7 +151,7 @@ async def main(): transport = TkLocalTransport( tk_root, - TransportParams( + TkTransportParams( audio_out_enabled=True, camera_out_enabled=True, camera_out_width=1024, diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 937ab4867..d9a010766 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -24,8 +24,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transports.base_transport import TransportParams -from pipecat.transports.local.tk import TkLocalTransport +from pipecat.transports.local.tk import TkLocalTransport, TkTransportParams from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -67,7 +66,7 @@ async def main(): tk_transport = TkLocalTransport( tk_root, - TransportParams( + TkTransportParams( audio_out_enabled=True, camera_out_enabled=True, camera_out_is_live=True, diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index d270d6395..4338992e3 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -90,7 +90,7 @@ class TkInputTransport(BaseInputTransport): class TkOutputTransport(BaseOutputTransport): _params: TkTransportParams - def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TransportParams): + def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TkTransportParams): super().__init__(params) self._py_audio = py_audio self._out_stream = None @@ -153,7 +153,7 @@ class TkOutputTransport(BaseOutputTransport): class TkLocalTransport(BaseTransport): - def __init__(self, tk_root: tk.Tk, params: TransportParams): + def __init__(self, tk_root: tk.Tk, params: TkTransportParams): super().__init__() self._tk_root = tk_root self._params = params From 92cc6d39f2728a06d46a527992ccdd1dd5a7c2a0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 25 Feb 2025 12:48:38 -0500 Subject: [PATCH 213/427] TTSService: Remove newlines before sending text to TTS service to generate --- CHANGELOG.md | 4 ++++ src/pipecat/services/ai_services.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aba68530d..72f00278c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The base `TTSService` class now strips leading newlines before sending text + to the TTS provider. This change is to solve issues where some TTS providers, + like Azure, would not output text due to newlines. + - `GrokLLMSService` now uses `grok-2` as the default model. - `AnthropicLLMService` now uses `claude-3-7-sonnet-20250219` as the default diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 8f0df1b37..4fdcd0afa 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -399,6 +399,9 @@ class TTSService(AIService): await self._push_tts_frames(text) async def _push_tts_frames(self, text: str): + # Remove leading newlines only + text = text.lstrip("\n") + # Don't send only whitespace. This causes problems for some TTS models. But also don't # strip all whitespace, as whitespace can influence prosody. if not text.strip(): From 9a50f33e36a8b6b542c22ff8cc208e9b099e2a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 15:35:47 -0800 Subject: [PATCH 214/427] AudioContextWordTTSService: wait for all requested audio --- CHANGELOG.md | 4 ++++ src/pipecat/services/ai_services.py | 36 +++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5d3ae6d1..ade408c14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,10 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an `AudioContextWordTTSService` issue that would cause an `EndFrame` to + disconnect from the TTS service before audio from all the contexts was + received. This affected services like Cartesia and Rime. + - Fixed an issue that was not allowing to pass an `OpenAILLMContext` to create `GoogleLLMService`'s context aggregators. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index a47320b7a..136afb47a 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -652,7 +652,12 @@ class AudioContextWordTTSService(WebsocketWordTTSService): async def stop(self, frame: EndFrame): await super().stop(frame) - await self._stop_audio_context_task() + if self._audio_context_task: + # Indicate no more audio contexts are available. this will end the + # task cleanly after all contexts have been processed. + await self._contexts_queue.put(None) + await self.wait_for_task(self._audio_context_task) + self._audio_context_task = None async def cancel(self, frame: CancelFrame): await super().cancel(frame) @@ -676,22 +681,29 @@ class AudioContextWordTTSService(WebsocketWordTTSService): async def _audio_context_task_handler(self): """In this task we process audio contexts in order.""" - while True: + running = True + while running: context_id = await self._contexts_queue.get() - # Process the audio context until the context doesn't have more - # audio available (i.e. we find None). - await self._handle_audio_context(context_id) + if context_id: + # Process the audio context until the context doesn't have more + # audio available (i.e. we find None). + await self._handle_audio_context(context_id) + + # We just finished processing the context, so we can safely remove it. + del self._contexts[context_id] + + # Append some silence between sentences. + silence = b"\x00" * self.sample_rate + frame = TTSAudioRawFrame( + audio=silence, sample_rate=self.sample_rate, num_channels=1 + ) + await self.push_frame(frame) + else: + running = False - # We just finished processing the context, so we can safely remove it. - del self._contexts[context_id] self._contexts_queue.task_done() - # Append some silence between sentences. - silence = b"\x00" * self.sample_rate - frame = TTSAudioRawFrame(audio=silence, sample_rate=self.sample_rate, num_channels=1) - await self.push_frame(frame) - async def _handle_audio_context(self, context_id: str): # If we don't receive any audio during this time, we consider the context finished. AUDIO_CONTEXT_TIMEOUT = 3.0 From 3e66f2378dbc2225515acb09ea4f9d2b2a7706a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 17:34:33 -0800 Subject: [PATCH 215/427] improve test-requirements.txt --- dev-requirements.txt | 6 ++-- .../processors/filters/stt_mute_filter.py | 3 +- test-requirements.txt | 34 +------------------ 3 files changed, 5 insertions(+), 38 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 6533de95b..2b55adcbe 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,10 +3,10 @@ coverage~=7.6.12 grpcio-tools~=1.67.1 pip-tools~=7.4.1 pre-commit~=4.0.1 -pyright~=1.1.393 +pyright~=1.1.394 pytest~=8.3.4 -pytest-asyncio~=0.25.2 -ruff~=0.9.5 +pytest-asyncio~=0.25.3 +ruff~=0.9.7 setuptools~=70.0.0 setuptools_scm~=8.1.0 python-dotenv~=1.0.1 diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index 3c1ae53af..c74e495ab 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -32,7 +32,6 @@ from pipecat.frames.frames import ( UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.ai_services import STTService class STTMuteStrategy(Enum): @@ -94,7 +93,7 @@ class STTMuteFilter(FrameProcessor): """ def __init__( - self, *, config: STTMuteConfig, stt_service: Optional[STTService] = None, **kwargs + self, *, config: STTMuteConfig, stt_service: Optional[FrameProcessor] = None, **kwargs ): super().__init__(**kwargs) self._config = config diff --git a/test-requirements.txt b/test-requirements.txt index fe45b9ff2..b34a53ab9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,33 +1 @@ -aiohttp~=3.10.3 -anthropic~=0.30.0 -azure-cognitiveservices-speech~=1.40.0 -boto3~=1.35.27 -cartesia~=1.3.1 -daily-python~=0.11.0 -deepgram-sdk~=3.5.0 -fal-client~=0.4.1 -fastapi~=0.115.0 -faster-whisper~=1.0.3 -google-cloud-speech~=2.31.0 -google-cloud-texttospeech~=2.25.0 -google-genai~=1.2.0 -google-generativeai~=0.8.4 -langchain~=0.2.14 -livekit~=0.13.1 -lmnt~=1.1.4 -loguru~=0.7.2 -Markdown~=3.7 -numpy~=1.26.4 -openai~=1.37.2 -openpipe~=4.24.0 -Pillow~=10.4.0 -pyaudio~=0.2.14 -pydantic~=2.8.2 -pyloudnorm~=0.1.1 -pyht~=0.1.4 -python-dotenv~=1.0.1 -silero-vad~=5.1 -soxr~=0.5.0 -together~=1.2.7 -transformers~=4.48.0 -websockets~=13.1 \ No newline at end of file +-e ".[anthropic,google,langchain]" From f8f0578c3da7fedca3b11c36db65afc2ca93a57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 16:32:16 -0800 Subject: [PATCH 216/427] log pipecat version on application startup --- CHANGELOG.md | 3 +++ src/pipecat/__init__.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5d3ae6d1..43ab68988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Pipecat version will now be logged on every application startup. This will + help us identify what version we are running in case of any issues. + - Added a new `StopFrame` which can be used to stop a pipeline task while keeping the frame processors running. The frame processors could then be used in a different pipeline. The difference between a `StopFrame` and a diff --git a/src/pipecat/__init__.py b/src/pipecat/__init__.py index e69de29bb..b1a9c7413 100644 --- a/src/pipecat/__init__.py +++ b/src/pipecat/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from importlib.metadata import version + +from loguru import logger + +__version__ = version("pipecat-ai") + +logger.info(f"ᓚᘏᗢ Pipecat {__version__} ᓚᘏᗢ") From 6722aae59881262bc094b1118c8205e673d028b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 17:56:01 -0800 Subject: [PATCH 217/427] PipelineTask: force constructor keyword arguments --- CHANGELOG.md | 3 +++ examples/bot-ready-signalling/server/signalling_bot.py | 2 +- examples/canonical-metrics/bot.py | 2 +- examples/chatbot-audio-recording/bot.py | 2 +- examples/deployment/flyio-example/bot.py | 2 +- examples/deployment/modal-example/bot.py | 2 +- examples/foundational/03b-still-frame-imagen.py | 3 ++- examples/foundational/06-listen-and-respond.py | 5 ++++- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-vad.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-anthropic.py | 2 +- examples/foundational/07b-interruptible-langchain.py | 2 +- examples/foundational/07c-interruptible-deepgram-vad.py | 2 +- examples/foundational/07c-interruptible-deepgram.py | 2 +- examples/foundational/07d-interruptible-elevenlabs.py | 2 +- examples/foundational/07e-interruptible-playht-http.py | 2 +- examples/foundational/07e-interruptible-playht.py | 2 +- examples/foundational/07f-interruptible-azure.py | 2 +- examples/foundational/07g-interruptible-openai.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07i-interruptible-xtts.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07k-interruptible-lmnt.py | 2 +- examples/foundational/07l-interruptible-together.py | 2 +- examples/foundational/07m-interruptible-polly.py | 2 +- examples/foundational/07n-interruptible-google.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07p-interruptible-krisp.py | 2 +- examples/foundational/07q-interruptible-rime.py | 2 +- examples/foundational/07r-interruptible-riva-nim.py | 2 +- .../foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/07t-interruptible-fish.py | 2 +- examples/foundational/09-mirror.py | 6 +++++- examples/foundational/09a-local-mirror.py | 6 +++++- examples/foundational/10-wake-phrase.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14a-function-calling-anthropic.py | 8 +++++++- .../foundational/14b-function-calling-anthropic-video.py | 8 +++++++- examples/foundational/14e-function-calling-gemini.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14k-function-calling-cerebras.py | 2 +- examples/foundational/14l-function-calling-deepseek.py | 2 +- examples/foundational/14m-function-calling-openrouter.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/15-switch-voices.py | 2 +- examples/foundational/15a-switch-languages.py | 2 +- examples/foundational/16-gpu-container-local-bot.py | 8 +++++++- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/19-openai-realtime-beta.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- .../20b-persistent-context-openai-realtime.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/20d-persistent-context-gemini.py | 2 +- examples/foundational/21-tavus-layer.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- .../foundational/22b-natural-conversation-proposal.py | 2 +- .../foundational/22c-natural-conversation-mixed-llms.py | 2 +- .../foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/24-stt-mute-filter.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- examples/foundational/26-gemini-multimodal-live.py | 2 +- .../26a-gemini-multimodal-live-transcription.py | 2 +- .../26b-gemini-multimodal-live-function-calling.py | 2 +- examples/foundational/26c-gemini-multimodal-live-video.py | 2 +- examples/foundational/26d-gemini-multimodal-live-text.py | 2 +- .../foundational/26e-gemini-multimodal-google-search.py | 6 +++--- examples/foundational/27-simli-layer.py | 2 +- .../foundational/28a-transcription-processor-openai.py | 2 +- .../foundational/28b-transcript-processor-anthropic.py | 2 +- .../foundational/28c-transcription-processor-gemini.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/31-heartbeats.py | 2 +- examples/foundational/32-gemini-grounding-metadata.py | 6 +++--- examples/foundational/33-gemini-rag.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- examples/patient-intake/bot.py | 2 +- examples/phone-chatbot/bot_daily.py | 2 +- examples/phone-chatbot/bot_daily_gemini.py | 2 +- examples/phone-chatbot/bot_twilio.py | 2 +- examples/sentry-metrics/bot.py | 2 +- examples/simple-chatbot/server/bot-gemini.py | 2 +- examples/simple-chatbot/server/bot-openai.py | 2 +- examples/storytelling-chatbot/src/bot.py | 2 +- examples/studypal/studypal.py | 6 ++++-- examples/translation-chatbot/bot.py | 2 +- examples/twilio-chatbot/bot.py | 4 +++- examples/twilio-chatbot/client.py | 4 +++- examples/websocket-server/bot.py | 4 +++- src/pipecat/pipeline/task.py | 1 + 95 files changed, 140 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1f99d3a..afba77a64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- ⚠️ `PipelineTask` now requires keyword arguments (except for the first one for + the pipeline). + - The base `TTSService` class now strips leading newlines before sending text to the TTS provider. This change is to solve issues where some TTS providers, like Azure, would not output text due to newlines. diff --git a/examples/bot-ready-signalling/server/signalling_bot.py b/examples/bot-ready-signalling/server/signalling_bot.py index 4d4d287b5..c6e47b812 100644 --- a/examples/bot-ready-signalling/server/signalling_bot.py +++ b/examples/bot-ready-signalling/server/signalling_bot.py @@ -17,7 +17,7 @@ from runner import configure from pipecat.frames.frames import AudioRawFrame, EndFrame, OutputAudioRawFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/examples/canonical-metrics/bot.py b/examples/canonical-metrics/bot.py index 7e6b62a44..aa376c290 100644 --- a/examples/canonical-metrics/bot.py +++ b/examples/canonical-metrics/bot.py @@ -119,7 +119,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/chatbot-audio-recording/bot.py b/examples/chatbot-audio-recording/bot.py index 47f0917dc..d37d0414e 100644 --- a/examples/chatbot-audio-recording/bot.py +++ b/examples/chatbot-audio-recording/bot.py @@ -124,7 +124,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @audiobuffer.event_handler("on_audio_data") async def on_audio_data(buffer, audio, sample_rate, num_channels): diff --git a/examples/deployment/flyio-example/bot.py b/examples/deployment/flyio-example/bot.py index b74e01231..e01ec7e5c 100644 --- a/examples/deployment/flyio-example/bot.py +++ b/examples/deployment/flyio-example/bot.py @@ -70,7 +70,7 @@ async def main(room_url: str, token: str): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/deployment/modal-example/bot.py b/examples/deployment/modal-example/bot.py index 7855caeb8..02bcff379 100644 --- a/examples/deployment/modal-example/bot.py +++ b/examples/deployment/modal-example/bot.py @@ -62,7 +62,7 @@ async def main(room_url: str, token: str): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/03b-still-frame-imagen.py b/examples/foundational/03b-still-frame-imagen.py index 87e73950e..cb9efb01f 100644 --- a/examples/foundational/03b-still-frame-imagen.py +++ b/examples/foundational/03b-still-frame-imagen.py @@ -44,7 +44,8 @@ async def main(): runner = PipelineRunner() task = PipelineTask( - Pipeline([imagegen, transport.output()]), PipelineParams(enable_metrics=True) + Pipeline([imagegen, transport.output()]), + params=PipelineParams(enable_metrics=True), ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 8c349d567..ee9ae056a 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -105,7 +105,10 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams(enable_metrics=True, enable_usage_metrics=True), + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 03a7fd3b8..077f9b7dd 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -127,7 +127,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index 1aee662c3..d6d3aca7c 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -76,7 +76,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index bb3965159..13e728bfc 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -74,7 +74,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index 62e1186b6..f855f81e6 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -79,7 +79,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 9c093f233..efa631b34 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -103,7 +103,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index 831accfd9..55f421bfa 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -81,7 +81,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index fc805ed56..1e7059657 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -74,7 +74,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index dd2e4735f..ec91ae7cd 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -74,7 +74,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07e-interruptible-playht-http.py b/examples/foundational/07e-interruptible-playht-http.py index 17c49d1be..45795a6ef 100644 --- a/examples/foundational/07e-interruptible-playht-http.py +++ b/examples/foundational/07e-interruptible-playht-http.py @@ -75,7 +75,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index fa4c5d6d4..5ccb96c15 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -77,7 +77,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index 9f66a2811..d30ddc975 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -83,7 +83,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index 01d869e91..f9cac5910 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -81,7 +81,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 0084f9b64..95d786297 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -81,7 +81,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 18980afe9..389f9ee60 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -75,7 +75,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index b4405ebf7..b1e92e787 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -80,7 +80,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index a6fd3aff4..8ce5a7f16 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -71,7 +71,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index 14bcb703c..54ebd4c2c 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -88,7 +88,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07m-interruptible-polly.py b/examples/foundational/07m-interruptible-polly.py index 15035a35f..3d49dc291 100644 --- a/examples/foundational/07m-interruptible-polly.py +++ b/examples/foundational/07m-interruptible-polly.py @@ -81,7 +81,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index f1cf9712e..6026afc45 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -79,7 +79,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 6e3ac5d5c..bffa35322 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -80,7 +80,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 0773acad6..b7b3bb822 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -76,7 +76,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 7bbc4cb25..e0b932a72 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -74,7 +74,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index d54ae4c19..f62717355 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -74,7 +74,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index d4c9bc9d7..e1232a155 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -251,7 +251,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index f7521ac68..ea1456a22 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -74,7 +74,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index 669562742..dc060ccc4 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -78,7 +78,11 @@ async def main(): runner = PipelineRunner() task = PipelineTask( - pipeline, PipelineParams(audio_in_sample_rate=24000, audio_out_sample_rate=24000) + pipeline, + params=PipelineParams( + audio_in_sample_rate=24000, + audio_out_sample_rate=24000, + ), ) await runner.run(task) diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index d9a010766..eb6739f49 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -82,7 +82,11 @@ async def main(): pipeline = Pipeline([daily_transport.input(), MirrorProcessor(), tk_transport.output()]) task = PipelineTask( - pipeline, PipelineParams(audio_in_sample_rate=24000, audio_out_sample_rate=24000) + pipeline, + params=PipelineParams( + audio_in_sample_rate=24000, + audio_out_sample_rate=24000, + ), ) async def run_tk(): diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 0c7cee0a2..1bdeba15f 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -76,7 +76,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 5ce8693cc..7c77378fc 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -112,7 +112,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index a387c038a..4723de004 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -99,7 +99,13 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True)) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index a499ddacd..69f1e5776 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -153,7 +153,13 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True)) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index ad5d82b6a..477d6c5ca 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -152,7 +152,7 @@ indicate you should use the get_image tool are: task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 3e285dba3..f2768a788 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -116,7 +116,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 49b3d6831..7318f455d 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -113,7 +113,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index cd426366c..cfd56b59d 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -117,7 +117,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 331e1c850..53346686f 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -116,7 +116,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 6a2d7d061..712ce0742 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -116,7 +116,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 05f7d2a94..4a65adbef 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -123,7 +123,7 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 8170c7d0f..f67d583ee 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -123,7 +123,7 @@ Start by asking me for my location. Then, use 'get_weather_current' to give me a task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 3325ab240..622337b07 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -117,7 +117,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index ba4d252d4..aee185f75 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -83,7 +83,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index be5b7e3e6..49280449f 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -133,7 +133,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index f3b97f704..217502ea2 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -126,7 +126,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index d025cc7bb..d6984925e 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -85,7 +85,13 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True)) + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) # When a participant joins, start transcription for that participant so the # bot can "hear" and respond to them. diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index e0af16c92..00c969df4 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -108,7 +108,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, report_only_initial_ttfb=True, diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 609b5f036..504f8fbf1 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -154,7 +154,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 28008d3b8..8da683a47 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -212,7 +212,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 3a1c43f47..ef82bd567 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -237,7 +237,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 2e7c61b68..64829450d 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -209,7 +209,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 1fb9d7b21..74c4e69be 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -263,7 +263,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/21-tavus-layer.py b/examples/foundational/21-tavus-layer.py index 0cbc065a0..c78c7eb41 100644 --- a/examples/foundational/21-tavus-layer.py +++ b/examples/foundational/21-tavus-layer.py @@ -87,7 +87,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( # We just use 16000 because that's what Tavus is expecting and # we avoid resampling. audio_in_sample_rate=16000, diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 5e6b0e501..9282f4e4b 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -145,7 +145,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index f828c2878..1eb2dd956 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -355,7 +355,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 283a7e518..362c7554d 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -564,7 +564,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index dde280fc1..9be9ec458 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -742,7 +742,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index c0347ebe3..1b443190d 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -87,7 +87,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index fe17a1085..7ae6c73fa 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -122,7 +122,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index af6e267dc..e3ff21d95 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -354,7 +354,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 62ea5d5f0..71d41304b 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -63,7 +63,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py index ae48ce31b..aa07ace93 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py @@ -89,7 +89,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 03e3096a3..14f2d62c7 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -120,7 +120,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py index 0beac44c9..4ec8663c4 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-multimodal-live-video.py @@ -79,7 +79,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index b243399e9..2e0e27a13 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -106,7 +106,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/26e-gemini-multimodal-google-search.py b/examples/foundational/26e-gemini-multimodal-google-search.py index 84a48d026..7bfd017d1 100644 --- a/examples/foundational/26e-gemini-multimodal-google-search.py +++ b/examples/foundational/26e-gemini-multimodal-google-search.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2024-2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -34,7 +34,7 @@ search_tool = {"google_search": {}} tools = [search_tool] system_instruction = """ -You are an expert at providing the most recent news from any place. Your responses will be converted to audio, so avoid using special characters or overly complex formatting. +You are an expert at providing the most recent news from any place. Your responses will be converted to audio, so avoid using special characters or overly complex formatting. Always use the google search API to retrieve the latest news. You must also use it to check which day is today. @@ -93,7 +93,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 2d8026bc4..cbef6d4a5 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -83,7 +83,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, ), diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index d432c35f2..f341cc0dd 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -150,7 +150,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index 5b45a9c3d..05956fdf7 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -150,7 +150,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index 841be0b98..3760a227f 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -178,7 +178,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index a6be27b97..49885d82c 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -117,7 +117,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/foundational/31-heartbeats.py b/examples/foundational/31-heartbeats.py index fbb959519..2487c13e7 100644 --- a/examples/foundational/31-heartbeats.py +++ b/examples/foundational/31-heartbeats.py @@ -32,7 +32,7 @@ async def main(): pipeline = Pipeline([NullProcessor()]) - task = PipelineTask(pipeline, PipelineParams(enable_heartbeats=True)) + task = PipelineTask(pipeline, params=PipelineParams(enable_heartbeats=True)) runner = PipelineRunner() diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 67694652f..516549694 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2024-2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -38,7 +38,7 @@ search_tool = {"google_search_retrieval": {}} tools = [search_tool] system_instruction = """ -You are an expert at providing the most recent news from any place. Your responses will be converted to audio, so avoid using special characters or overly complex formatting. +You are an expert at providing the most recent news from any place. Your responses will be converted to audio, so avoid using special characters or overly complex formatting. Always use the google search API to retrieve the latest news. You must also use it to check which day is today. @@ -117,7 +117,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/foundational/33-gemini-rag.py b/examples/foundational/33-gemini-rag.py index 72a246cc6..b4d70093a 100644 --- a/examples/foundational/33-gemini-rag.py +++ b/examples/foundational/33-gemini-rag.py @@ -230,7 +230,7 @@ Your response will be turned into speech so use only simple words and punctuatio ) task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index a34b0c3d0..d1cde2f22 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -140,7 +140,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams(allow_interruptions=True), + params=PipelineParams(allow_interruptions=True), observers=[GoogleRTVIObserver(rtvi)], ) diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 9efff46d3..75279fce3 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -346,7 +346,7 @@ async def main(): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=False)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=False)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/phone-chatbot/bot_daily.py b/examples/phone-chatbot/bot_daily.py index 16aba1b82..c3468bbed 100644 --- a/examples/phone-chatbot/bot_daily.py +++ b/examples/phone-chatbot/bot_daily.py @@ -150,7 +150,7 @@ async def main( ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) if dialout_number: logger.debug("dialout number detected; doing dialout") diff --git a/examples/phone-chatbot/bot_daily_gemini.py b/examples/phone-chatbot/bot_daily_gemini.py index b36313c5a..96f6506b6 100644 --- a/examples/phone-chatbot/bot_daily_gemini.py +++ b/examples/phone-chatbot/bot_daily_gemini.py @@ -271,7 +271,7 @@ DO NOT say anything until you've determined if this is a voicemail or human.""" task = PipelineTask( pipeline, - PipelineParams(allow_interruptions=True), + params=PipelineParams(allow_interruptions=True), ) if dialout_number: diff --git a/examples/phone-chatbot/bot_twilio.py b/examples/phone-chatbot/bot_twilio.py index 9da28dfa1..3aab6a6d8 100644 --- a/examples/phone-chatbot/bot_twilio.py +++ b/examples/phone-chatbot/bot_twilio.py @@ -77,7 +77,7 @@ async def main(room_url: str, token: str, callId: str, sipUri: str): ] ) - task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): diff --git a/examples/sentry-metrics/bot.py b/examples/sentry-metrics/bot.py index 0282c7905..89cd8dcf3 100644 --- a/examples/sentry-metrics/bot.py +++ b/examples/sentry-metrics/bot.py @@ -90,7 +90,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams(allow_interruptions=True, enable_metrics=True), + params=PipelineParams(allow_interruptions=True, enable_metrics=True), ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index f80a9770e..753cb2af7 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -172,7 +172,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index e1d7bb9a0..a5e4312ab 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -198,7 +198,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/src/bot.py index f7edbe1d8..92ab7d617 100644 --- a/examples/storytelling-chatbot/src/bot.py +++ b/examples/storytelling-chatbot/src/bot.py @@ -104,7 +104,7 @@ async def main(room_url, token=None): main_task = PipelineTask( main_pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/studypal/studypal.py b/examples/studypal/studypal.py index 0b1496e3d..1ec1d4aa4 100644 --- a/examples/studypal/studypal.py +++ b/examples/studypal/studypal.py @@ -155,8 +155,10 @@ Your task is to help the user understand and learn from this article in 2 senten task = PipelineTask( pipeline, - PipelineParams( - audio_out_sample_rate=44100, allow_interruptions=True, enable_metrics=True + params=PipelineParams( + audio_out_sample_rate=44100, + allow_interruptions=True, + enable_metrics=True, ), ) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index 5839afb5d..dc1d05f3d 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -183,7 +183,7 @@ async def main(): task = PipelineTask( pipeline, - PipelineParams( + params=PipelineParams( allow_interruptions=False, # We don't want to interrupt the translator bot enable_metrics=True, enable_usage_metrics=True, diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index 8d6f813ed..afddc3fe9 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -108,7 +108,9 @@ async def run_bot(websocket_client: WebSocket, stream_sid: str, testing: bool): task = PipelineTask( pipeline, params=PipelineParams( - audio_in_sample_rate=8000, audio_out_sample_rate=8000, allow_interruptions=True + audio_in_sample_rate=8000, + audio_out_sample_rate=8000, + allow_interruptions=True, ), ) diff --git a/examples/twilio-chatbot/client.py b/examples/twilio-chatbot/client.py index cc7e7be36..c4b02c0d2 100644 --- a/examples/twilio-chatbot/client.py +++ b/examples/twilio-chatbot/client.py @@ -142,7 +142,9 @@ async def run_client(client_name: str, server_url: str, duration_secs: int): task = PipelineTask( pipeline, params=PipelineParams( - audio_in_sample_rate=8000, audio_out_sample_rate=8000, allow_interruptions=True + audio_in_sample_rate=8000, + audio_out_sample_rate=8000, + allow_interruptions=True, ), ) diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index 9a0de15c6..d44c77df8 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -125,7 +125,9 @@ async def main(): task = PipelineTask( pipeline, params=PipelineParams( - audio_in_sample_rate=16000, audio_out_sample_rate=16000, allow_interruptions=True + audio_in_sample_rate=16000, + audio_out_sample_rate=16000, + allow_interruptions=True, ), ) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index da18cca5d..9e6b313a6 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -131,6 +131,7 @@ class PipelineTask(BaseTask): def __init__( self, pipeline: BasePipeline, + *, params: PipelineParams = PipelineParams(), observers: List[BaseObserver] = [], clock: BaseClock = SystemClock(), From 8acf9a488bc15152be5104a711919323c1d93484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 12:30:54 -0800 Subject: [PATCH 218/427] tts: some small HTTP-based services improvements --- src/pipecat/services/elevenlabs.py | 8 +++++--- src/pipecat/services/openai.py | 4 +++- src/pipecat/services/playht.py | 13 ++++++------- src/pipecat/services/rime.py | 6 +++--- src/pipecat/services/xtts.py | 4 +++- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 3dc42975e..5e70735f8 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -570,10 +570,12 @@ class ElevenLabsHttpTTSService(TTSService): await self.start_tts_usage_metrics(text) - yield TTSStartedFrame() + # Process the streaming response + CHUNK_SIZE = 1024 - async for chunk in response.content: - if chunk: + yield TTSStartedFrame() + async for chunk in response.content.iter_chunked(CHUNK_SIZE): + if len(chunk) > 0: await self.stop_ttfb_metrics() yield TTSAudioRawFrame(chunk, self.sample_rate, 1) except Exception as e: diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 6ed3b4612..e49bd3a90 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -530,8 +530,10 @@ class OpenAITTSService(TTSService): await self.start_tts_usage_metrics(text) + CHUNK_SIZE = 1024 + yield TTSStartedFrame() - async for chunk in r.iter_bytes(8192): + async for chunk in r.iter_bytes(CHUNK_SIZE): if len(chunk) > 0: await self.stop_ttfb_metrics() frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 56179e34c..c42a44eaa 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -383,8 +383,6 @@ class PlayHTHttpTTSService(TTSService): try: options = self._create_options() - b = bytearray() - in_header = True await self.start_ttfb_metrics() @@ -396,6 +394,8 @@ class PlayHTHttpTTSService(TTSService): yield TTSStartedFrame() + b = bytearray() + in_header = True async for chunk in playht_gen: # skip the RIFF header. if in_header: @@ -410,11 +410,10 @@ class PlayHTHttpTTSService(TTSService): fh.read(size) (data, size) = struct.unpack("<4sI", fh.read(8)) in_header = False - else: - if len(chunk): - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) - yield frame + elif len(chunk) > 0: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) + yield frame except Exception as e: logger.error(f"{self} error generating TTS: {e}") finally: diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 60d2d67ef..007ba958e 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -407,10 +407,10 @@ class RimeHttpTTSService(TTSService): yield TTSStartedFrame() # Process the streaming response - chunk_size = 8192 + CHUNK_SIZE = 1024 - async for chunk in response.content.iter_chunked(chunk_size): - if chunk: + async for chunk in response.content.iter_chunked(CHUNK_SIZE): + if len(chunk) > 0: await self.stop_ttfb_metrics() frame = TTSAudioRawFrame(chunk, self.sample_rate, 1) yield frame diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 9d2f3dd1e..e275b50b0 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -150,8 +150,10 @@ class XTTSService(TTSService): yield TTSStartedFrame() + CHUNK_SIZE = 1024 + buffer = bytearray() - async for chunk in r.content.iter_chunked(1024): + async for chunk in r.content.iter_chunked(CHUNK_SIZE): if len(chunk) > 0: await self.stop_ttfb_metrics() # Append new chunk to the buffer. From 27161f8e3b4e8db40660cb2c36f5663a1f71b02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 13:06:43 -0800 Subject: [PATCH 219/427] BaseOutputTransport: cleanup audio buffer after bot stops talking --- src/pipecat/transports/base_output.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 2322ade9d..a2ee5aa92 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -232,6 +232,9 @@ class BaseOutputTransport(FrameProcessor): await self.push_frame(BotStoppedSpeakingFrame()) await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM) self._bot_speaking = False + # Clean audio buffer (there could be tiny left overs if not multiple + # to our output chunk size). + self._audio_buffer = bytearray() # # Sink tasks From b994a03466d8d15177c8ae9f95542f6bb49d5b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 21:40:41 -0800 Subject: [PATCH 220/427] examples: add more HTTP TTS services examples --- .../07d-interruptible-elevenlabs-http.py | 103 ++++++++++++++++++ .../07q-interruptible-rime-http.py | 103 ++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 examples/foundational/07d-interruptible-elevenlabs-http.py create mode 100644 examples/foundational/07q-interruptible-rime-http.py diff --git a/examples/foundational/07d-interruptible-elevenlabs-http.py b/examples/foundational/07d-interruptible-elevenlabs-http.py new file mode 100644 index 000000000..63894851f --- /dev/null +++ b/examples/foundational/07d-interruptible-elevenlabs-http.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.elevenlabs import ElevenLabsHttpTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = ElevenLabsHttpTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/07q-interruptible-rime-http.py b/examples/foundational/07q-interruptible-rime-http.py new file mode 100644 index 000000000..eaa49179e --- /dev/null +++ b/examples/foundational/07q-interruptible-rime-http.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.openai import OpenAILLMService +from pipecat.services.rime import RimeHttpTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = RimeHttpTTSService( + api_key=os.getenv("RIME_API_KEY", ""), + voice_id="rex", + aiohttp_session=session, + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From bb89a036e5bc95caa21fb711a0c49eccbb15a8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:26:30 -0800 Subject: [PATCH 221/427] google: always send text part when sending inline audio --- CHANGELOG.md | 3 +++ src/pipecat/services/google/google.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afba77a64..3d41cb420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed a `GoogleLLMService` that was causing an exception when sending inline + audio in some cases. + - Fixed an `AudioContextWordTTSService` issue that would cause an `EndFrame` to disconnect from the TTS service before audio from all the contexts was received. This affected services like Cartesia and Rime. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 1dafc92bc..c0941ee33 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -722,7 +722,9 @@ class GoogleLLMContext(OpenAILLMContext): self.add_message(glm.Content(role="user", parts=parts)) - def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): + def add_audio_frames_message( + self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" + ): if not audio_frames: return @@ -731,8 +733,9 @@ class GoogleLLMContext(OpenAILLMContext): parts = [] data = b"".join(frame.audio for frame in audio_frames) - if text: - parts.append(glm.Part(text=text)) + # NOTE(aleix): According to the docs only text or inline_data should be needed. + # (see https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference) + parts.append(glm.Part(text=text)) parts.append( glm.Part( inline_data=glm.Blob( From f83c89c20235b5aa83cf4b365c1fa41358fb3cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:28:02 -0800 Subject: [PATCH 222/427] examples: update google examples --- .../22d-natural-conversation-gemini-audio.py | 2 +- examples/phone-chatbot/bot_daily_gemini.py | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 9be9ec458..6313f951d 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -389,7 +389,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = GoogleLLMContext() - context.add_audio_frames_message(text="Audio follows", audio_frames=self._audio_frames) + context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest diff --git a/examples/phone-chatbot/bot_daily_gemini.py b/examples/phone-chatbot/bot_daily_gemini.py index 96f6506b6..8ada81a9c 100644 --- a/examples/phone-chatbot/bot_daily_gemini.py +++ b/examples/phone-chatbot/bot_daily_gemini.py @@ -7,20 +7,15 @@ import argparse import asyncio import os import sys -from dataclasses import dataclass from typing import Optional -import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( - BotStoppedSpeakingFrame, EndTaskFrame, - Frame, InputAudioRawFrame, - SystemFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, @@ -28,11 +23,11 @@ from pipecat.frames.frames import ( from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.ai_services import LLMService from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.services.google import GoogleLLMService +from pipecat.services.google.google import GoogleLLMContext from pipecat.transports.services.daily import DailyDialinSettings, DailyParams, DailyTransport load_dotenv(override=True) @@ -240,7 +235,7 @@ If it sounds like a human (saying hello, asking questions, etc.), call the funct DO NOT say anything until you've determined if this is a voicemail or human.""" llm = GoogleLLMService( - model="models/gemini-2.0-flash-lite-preview-02-05", + model="models/gemini-2.0-flash-lite", api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, From 4f8b036abec64dcc54a44ff6252dbc18fee09cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:28:21 -0800 Subject: [PATCH 223/427] pyproject: remote httpx old dependency and upgrade anthropic/google-genai --- pyproject.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 768c9c816..1abfff495 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,6 @@ classifiers = [ dependencies = [ "aiohttp~=3.11.11", "audioop-lts~=0.2.1; python_version>='3.13'", - # We need an older version of `httpx` that doesn't remove the deprecated - # `proxies` argument. This is necessary for Azure and Anthropic clients. - "httpx~=0.27.2", "loguru~=0.7.3", "Markdown~=3.7", "numpy~=1.26.4", @@ -42,7 +39,7 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -anthropic = [ "anthropic~=0.45.2" ] +anthropic = [ "anthropic~=0.47.2" ] assemblyai = [ "assemblyai~=0.36.0" ] aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] @@ -56,7 +53,7 @@ elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.6" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] -google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.2.0", "google-generativeai~=0.8.4" ] +google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.3.0", "google-generativeai~=0.8.4" ] grok = [] groq = [] gstreamer = [ "pygobject~=3.50.0" ] From 1753cc99f452c4afa7fcfd83769c0581664a1379 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 25 Feb 2025 16:38:19 -0500 Subject: [PATCH 224/427] PlayHTHttpTTSService now takes a separate protocol input --- CHANGELOG.md | 4 ++++ src/pipecat/services/playht.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afba77a64..341f78a3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ⚠️ `PipelineTask` now requires keyword arguments (except for the first one for the pipeline). +- Updated `PlayHTHttpTTSService` to take a `voice_engine` and `protocol` input + in the constructor. The previous method of providing a `voice_engine` input + that contains the engine and protocol is deprecated by PlayHT. + - The base `TTSService` class now strips leading newlines before sending text to the TTS provider. This change is to solve issues where some TTS providers, like Azure, would not output text due to newlines. diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 56179e34c..f0fd3491e 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -323,7 +323,8 @@ class PlayHTHttpTTSService(TTSService): api_key: str, user_id: str, voice_url: str, - voice_engine: str = "Play3.0-mini-http", # Options: Play3.0-mini-http, Play3.0-mini-ws + voice_engine: str = "Play3.0-mini", + protocol: str = "http", # Options: http, ws sample_rate: Optional[int] = None, params: InputParams = InputParams(), **kwargs, @@ -337,12 +338,24 @@ class PlayHTHttpTTSService(TTSService): user_id=self._user_id, api_key=self._api_key, ) + + # Check if voice_engine contains protocol information (backward compatibility) + if "-http" in voice_engine: + # Extract the base engine name + voice_engine = voice_engine.replace("-http", "") + protocol = "http" + elif "-ws" in voice_engine: + # Extract the base engine name + voice_engine = voice_engine.replace("-ws", "") + protocol = "ws" + self._settings = { "language": self.language_to_service_language(params.language) if params.language else "english", "format": Format.FORMAT_WAV, "voice_engine": voice_engine, + "protocol": protocol, "speed": params.speed, "seed": params.seed, } @@ -389,7 +402,10 @@ class PlayHTHttpTTSService(TTSService): await self.start_ttfb_metrics() playht_gen = self._client.tts( - text, voice_engine=self._settings["voice_engine"], options=options + text, + voice_engine=self._settings["voice_engine"], + protocol=self._settings["protocol"], + options=options, ) await self.start_tts_usage_metrics(text) From 1ca2101e3aa9c05a0881eab33de69db1385a5865 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 26 Feb 2025 10:48:56 -0500 Subject: [PATCH 225/427] Added on_track_audio_data callback to AudioBufferProcessor for track level recording --- CHANGELOG.md | 3 + .../audio/audio_buffer_processor.py | 78 ++++++++++++++++--- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 341f78a3b..8b8ecbbe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Added +- Added track-specific audio event `on_track_audio_data` to + `AudioBufferProcessor` for accessing separate input and output audio tracks. + - Added new `AudioContextWordTTSService`. This is a TTS base class for TTS services that handling multiple separate audio requests. diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 2f9b975e3..1863a0ee6 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -21,20 +21,32 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class AudioBufferProcessor(FrameProcessor): - """This processor buffers audio raw frames (input and output). The mixed - audio can be obtained by registering an "on_audio_data" event handler. - The event handler will be called every time `buffer_size` is reached. + """Processes and buffers audio frames from both input (user) and output (bot) sources. - You can provide the desired output `sample_rate` and incoming audio frames - will resampled to match it. Also, you can provide the number of channels, 1 - for mono and 2 for stereo. With mono audio user and bot audio will be mixed, - in the case of stereo the left channel will be used for the user's audio and - the right channel for the bot. + This processor manages audio buffering and synchronization, providing both merged and + track-specific audio access through event handlers. It supports various audio configurations + including sample rate conversion and mono/stereo output. - Most of the time, user audio will be a continuous stream but it's possible - that in some cases only the spoken audio is sent. To accomodate for those - cases make sure to set `user_continuous_stream` accordingly. + Events: + on_audio_data: Triggered when buffer_size is reached, providing merged audio + on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks + Args: + sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate + num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1 + buffer_size (int): Size of buffer before triggering events. 0 for no buffering + user_continuous_stream (bool): Whether user audio is continuous or speech-only + + Audio handling: + - Mono output (num_channels=1): User and bot audio are mixed + - Stereo output (num_channels=2): User audio on left, bot audio on right + - Automatic resampling of incoming audio to match desired sample_rate + - Silence insertion for non-continuous audio streams + - Buffer synchronization between user and bot audio + + Note: + When user_continuous_stream is False, the processor expects only speech + segments and will handle silence insertion between segments automatically. """ def __init__( @@ -65,21 +77,45 @@ class AudioBufferProcessor(FrameProcessor): self._resampler = create_default_resampler() self._register_event_handler("on_audio_data") + self._register_event_handler("on_track_audio_data") @property def sample_rate(self) -> int: + """Current sample rate of the audio processor. + + Returns: + int: The sample rate in Hz + """ return self._sample_rate @property def num_channels(self) -> int: + """Number of channels in the audio output. + + Returns: + int: Number of channels (1 for mono, 2 for stereo) + """ return self._num_channels def has_audio(self) -> bool: + """Check if both user and bot audio buffers contain data. + + Returns: + bool: True if both buffers contain audio data + """ return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio( self._bot_audio_buffer ) def merge_audio_buffers(self) -> bytes: + """Merge user and bot audio buffers into a single audio stream. + + For mono output, audio is mixed. For stereo output, user audio is placed + on the left channel and bot audio on the right channel. + + Returns: + bytes: Mixed audio data + """ if self._num_channels == 1: return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer)) elif self._num_channels == 2: @@ -90,14 +126,23 @@ class AudioBufferProcessor(FrameProcessor): return b"" async def start_recording(self): + """Start recording audio from both user and bot. + + Initializes recording state and resets audio buffers. + """ self._recording = True self._reset_recording() async def stop_recording(self): + """Stop recording and trigger final audio data handlers. + + Calls audio handlers with any remaining buffered audio before stopping. + """ await self._call_on_audio_data_handler() self._recording = False async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming audio frames and manage audio buffers.""" await super().process_frame(frame, direction) # Update output sample rate if necessary. @@ -160,10 +205,21 @@ class AudioBufferProcessor(FrameProcessor): if not self.has_audio() or not self._recording: return + # Call original handler with merged audio merged_audio = self.merge_audio_buffers() await self._call_event_handler( "on_audio_data", merged_audio, self._sample_rate, self._num_channels ) + + # Call new handler with separate tracks + await self._call_event_handler( + "on_track_audio_data", + bytes(self._user_audio_buffer), + bytes(self._bot_audio_buffer), + self._sample_rate, + self._num_channels, + ) + self._reset_audio_buffers() def _buffer_has_audio(self, buffer: bytearray) -> bool: From 74582bb8d55e81790d666249be107c31334a87c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Feb 2025 22:45:40 -0800 Subject: [PATCH 226/427] pyproject: update daily-python, aiohttp and pydantic --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1abfff495..6c91f7f99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,14 +20,14 @@ classifiers = [ "Topic :: Scientific/Engineering :: Artificial Intelligence" ] dependencies = [ - "aiohttp~=3.11.11", + "aiohttp~=3.11.13", "audioop-lts~=0.2.1; python_version>='3.13'", "loguru~=0.7.3", "Markdown~=3.7", "numpy~=1.26.4", "Pillow~=11.1.0", "protobuf~=5.29.3", - "pydantic~=2.10.5", + "pydantic~=2.10.6", "pyloudnorm~=0.1.1", "resampy~=0.4.3", "soxr~=0.5.0", @@ -47,7 +47,7 @@ canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.14.2" ] +daily = [ "daily-python~=0.15.0" ] deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.6" ] From 530bb5233d89584321a42b2e7fa27d137498faf6 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 26 Feb 2025 10:51:23 -0500 Subject: [PATCH 227/427] example: Added a foundational example (34) for audio recording --- CHANGELOG.md | 9 +- examples/foundational/34-audio-recording.py | 186 ++++++++++++++++++++ 2 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 examples/foundational/34-audio-recording.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b8ecbbe8..83b520583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added track-specific audio event `on_track_audio_data` to + `AudioBufferProcessor` for accessing separate input and output audio tracks. + - Pipecat version will now be logged on every application startup. This will help us identify what version we are running in case of any issues. @@ -128,13 +131,13 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Added Gemini support to `examples/phone-chatbot`. +- Added foundational example `34-audio-recording.py` showing how to use the + AudioBufferProcessor callbacks to save merged and track recordings. + ## [0.0.57] - 2025-02-14 ### Added -- Added track-specific audio event `on_track_audio_data` to - `AudioBufferProcessor` for accessing separate input and output audio tracks. - - Added new `AudioContextWordTTSService`. This is a TTS base class for TTS services that handling multiple separate audio requests. diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py new file mode 100644 index 000000000..1d0431b83 --- /dev/null +++ b/examples/foundational/34-audio-recording.py @@ -0,0 +1,186 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Audio Recording Example with Pipecat. + +This example demonstrates how to record audio from a conversation between a user and an AI assistant, +saving both merged and individual audio tracks. It showcases the AudioBufferProcessor's capabilities +to handle both combined and separate audio streams. + +The example: + 1. Sets up a basic conversation with an AI assistant + 2. Records the entire conversation + 3. Saves three separate WAV files: + - A merged recording of both participants + - Individual recording of user audio + - Individual recording of assistant audio + +Example usage (run from pipecat root directory): + $ pip install "pipecat-ai[daily,openai,cartesia,silero]" + $ pip install -r dev-requirements.txt + $ python examples/foundational/34-audio-recording.py + +Requirements: + - OpenAI API key (for GPT-4) + - Cartesia API key (for text-to-speech) + - Daily API key (for video/audio transport) + + Environment variables (.env file): + OPENAI_API_KEY=your_openai_key + CARTESIA_API_KEY=your_cartesia_key + DAILY_API_KEY=your_daily_key + +The recordings will be saved in a 'recordings' directory with timestamps: + recordings/ + merged_20240315_123456.wav (Combined audio) + user_20240315_123456.wav (User audio only) + bot_20240315_123456.wav (Bot audio only) + +Note: + This example requires the AudioBufferProcessor with track-specific audio support, + which provides both 'on_audio_data' and 'on_track_audio_data' events for + handling merged and separate audio tracks respectively. +""" + +import asyncio +import datetime +import io +import os +import sys +import wave + +import aiofiles +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def save_audio_file(audio: bytes, filename: str, sample_rate: int, num_channels: int): + """Save audio data to a WAV file.""" + if len(audio) > 0: + with io.BytesIO() as buffer: + with wave.open(buffer, "wb") as wf: + wf.setsampwidth(2) + wf.setnchannels(num_channels) + wf.setframerate(sample_rate) + wf.writeframes(audio) + async with aiofiles.open(filename, "wb") as file: + await file.write(buffer.getvalue()) + logger.info(f"Audio saved to {filename}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Recording bot", + DailyParams( + # audio_in_enabled=True, + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, # Enable audio passthrough for recording + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4") + + # Create audio buffer processor + audiobuffer = AudioBufferProcessor() + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant demonstrating audio recording capabilities. Keep your responses brief and clear.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + audiobuffer, # Add audio buffer to pipeline + context_aggregator.assistant(), + ] + ) + + task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + await audiobuffer.start_recording() + messages.append( + { + "role": "system", + "content": "Greet the user and explain that this conversation will be recorded.", + } + ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await audiobuffer.stop_recording() + await task.cancel() + + # Handler for merged audio + @audiobuffer.event_handler("on_audio_data") + async def on_audio_data(buffer, audio, sample_rate, num_channels): + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"recordings/merged_{timestamp}.wav" + os.makedirs("recordings", exist_ok=True) + await save_audio_file(audio, filename, sample_rate, num_channels) + + # Handler for separate tracks + @audiobuffer.event_handler("on_track_audio_data") + async def on_track_audio_data(buffer, user_audio, bot_audio, sample_rate, num_channels): + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + os.makedirs("recordings", exist_ok=True) + + # Save user audio + user_filename = f"recordings/user_{timestamp}.wav" + await save_audio_file(user_audio, user_filename, sample_rate, 1) + + # Save bot audio + bot_filename = f"recordings/bot_{timestamp}.wav" + await save_audio_file(bot_audio, bot_filename, sample_rate, 1) + + runner = PipelineRunner() + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From daa52ff8df6d54cca910949955c8ace6142f18b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 11:09:00 -0800 Subject: [PATCH 228/427] update CHANGELOG for 0.0.58 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebea33515..ff7b62e9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.58] - 2025-02-26 ### Added From d58f398bc4df57a15e4be6dd89368ffd08fb5f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 13:15:07 -0800 Subject: [PATCH 229/427] examples: fix for 22d-natural-conversation-gemini-audio.py --- .../22d-natural-conversation-gemini-audio.py | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 6313f951d..0041ff530 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -23,7 +23,6 @@ from pipecat.frames.frames import ( FunctionCallInProgressFrame, FunctionCallResultFrame, InputAudioRawFrame, - LLMFullResponseEndFrame, LLMFullResponseStartFrame, StartFrame, StartInterruptionFrame, @@ -37,7 +36,7 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_response import LLMResponseAggregator +from pipecat.processors.aggregators.llm_response import LLMAssistantResponseAggregator from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -432,7 +431,11 @@ class CompletenessCheck(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, UserStartedSpeakingFrame): + if isinstance(frame, (EndFrame, CancelFrame)): + if self._idle_task: + await self.cancel_task(self._idle_task) + self._idle_task = None + elif isinstance(frame, UserStartedSpeakingFrame): if self._idle_task: await self.cancel_task(self._idle_task) elif isinstance(frame, TextFrame) and frame.text.startswith("YES"): @@ -474,19 +477,11 @@ class CompletenessCheck(FrameProcessor): self._idle_task = None -class UserAggregatorBuffer(LLMResponseAggregator): +class LLMAggregatorBuffer(LLMAssistantResponseAggregator): """Buffers the output of the transcription LLM. Used by the bot output gate.""" def __init__(self, **kwargs): - super().__init__( - messages=None, - role=None, - start_frame=LLMFullResponseStartFrame, - end_frame=LLMFullResponseEndFrame, - accumulator_frame=TextFrame, - handle_interruptions=True, - expect_stripped_words=False, - ) + super().__init__(expect_stripped_words=False) self._transcription = "" async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -544,7 +539,7 @@ class OutputGate(FrameProcessor): self, notifier: BaseNotifier, context: OpenAILLMContext, - user_transcription_buffer: "UserAggregatorBuffer", + llm_transcription_buffer: LLMAggregatorBuffer, **kwargs, ): super().__init__(**kwargs) @@ -552,7 +547,7 @@ class OutputGate(FrameProcessor): self._frames_buffer = [] self._notifier = notifier self._context = context - self._transcription_buffer = user_transcription_buffer + self._transcription_buffer = llm_transcription_buffer self._gate_task = None def close_gate(self): @@ -699,10 +694,10 @@ async def main(): conversation_audio_context_assembler = ConversationAudioContextAssembler(context=context) - user_aggregator_buffer = UserAggregatorBuffer() + llm_aggregator_buffer = LLMAggregatorBuffer() bot_output_gate = OutputGate( - notifier=notifier, context=context, user_transcription_buffer=user_aggregator_buffer + notifier=notifier, context=context, llm_transcription_buffer=llm_aggregator_buffer ) pipeline = Pipeline( @@ -723,7 +718,7 @@ async def main(): ], [ tx_llm, - user_aggregator_buffer, + llm_aggregator_buffer, ], ) ], From fa010c864414904ded30ff6f89d091b5cf152c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 15:36:25 -0800 Subject: [PATCH 230/427] services: minor LLM and TTS logging improvements --- src/pipecat/services/anthropic.py | 2 +- src/pipecat/services/aws.py | 2 +- src/pipecat/services/azure.py | 4 ++-- src/pipecat/services/cartesia.py | 4 ++-- src/pipecat/services/deepgram.py | 2 +- src/pipecat/services/elevenlabs.py | 4 ++-- src/pipecat/services/fish.py | 2 +- src/pipecat/services/google/google.py | 6 +++--- src/pipecat/services/lmnt.py | 2 +- src/pipecat/services/openai.py | 4 ++-- src/pipecat/services/playht.py | 4 ++-- src/pipecat/services/rime.py | 4 ++-- src/pipecat/services/riva.py | 2 +- src/pipecat/services/xtts.py | 2 +- 14 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 1316802de..f10ee7ccb 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -152,7 +152,7 @@ class AnthropicLLMService(LLMService): await self.start_processing_metrics() logger.debug( - f"Generating chat: {context.system} | {context.get_messages_for_logging()}" + f"{self}: Generating chat [{context.system}] | [{context.get_messages_for_logging()}]" ) messages = context.messages diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index eac3f3e11..b83df2f77 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -197,7 +197,7 @@ class PollyTTSService(TTSService): return audio_data return None - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index 11c34214c..83e5bbe6d 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -578,7 +578,7 @@ class AzureTTSService(AzureBaseTTSService): self._audio_queue.put_nowait(None) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: if self._speech_synthesizer is None: @@ -645,7 +645,7 @@ class AzureHttpTTSService(AzureBaseTTSService): ) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") await self.start_ttfb_metrics() diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 04a35087b..9bbde11bb 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -272,7 +272,7 @@ class CartesiaTTSService(AudioContextWordTTSService): logger.error(f"{self} error, unknown message type: {msg}") async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: if not self._websocket: @@ -358,7 +358,7 @@ class CartesiaHttpTTSService(TTSService): await self._client.close() async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: voice_controls = None diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 22879ab0e..3750164cf 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -70,7 +70,7 @@ class DeepgramTTSService(TTSService): return True async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") options = SpeakOptions( model=self._voice_id, diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 5e70735f8..4e6c84307 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -395,7 +395,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): await self._websocket.send(json.dumps(msg)) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: if not self._websocket: @@ -521,7 +521,7 @@ class ElevenLabsHttpTTSService(TTSService): Yields: Frames containing audio data and status information """ - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") url = f"{self._base_url}/v1/text-to-speech/{self._voice_id}/stream" diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index e61579bfb..0b2729958 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -178,7 +178,7 @@ class FishAudioTTSService(InterruptibleTTSService): logger.error(f"Error processing message: {e}") async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating Fish TTS: [{text}]") + logger.debug(f"{self}: Generating Fish TTS: [{text}]") try: if not self._websocket or self._websocket.closed: await self._connect() diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index c0941ee33..7a48b0e4c 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -998,8 +998,8 @@ class GoogleLLMService(LLMService): try: logger.debug( - # f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}" - f"Generating chat: {context.get_messages_for_logging()}" + # f"{self}: Generating chat [{self._system_instruction}] | [{context.get_messages_for_logging()}]" + f"{self}: Generating chat [{context.get_messages_for_logging()}]" ) messages = context.messages @@ -1297,7 +1297,7 @@ class GoogleTTSService(TTSService): return ssml async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 8d4ca4803..31ddaf123 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -196,7 +196,7 @@ class LmntTTSService(InterruptibleTTSService): async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate TTS audio from text.""" - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: if not self._websocket: diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index e49bd3a90..71da3f5e2 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -178,7 +178,7 @@ class BaseOpenAILLMService(LLMService): async def _stream_chat_completions( self, context: OpenAILLMContext ) -> AsyncStream[ChatCompletionChunk]: - logger.debug(f"Generating chat: {context.get_messages_for_logging()}") + logger.debug(f"{self}: Generating chat [{context.get_messages_for_logging()}]") messages: List[ChatCompletionMessageParam] = context.get_messages() @@ -508,7 +508,7 @@ class OpenAITTSService(TTSService): ) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index e2c3b715d..80d313765 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -269,7 +269,7 @@ class PlayHTTTSService(InterruptibleTTSService): logger.error(f"Invalid JSON message: {message}") async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: # Reconnect if the websocket is closed @@ -392,7 +392,7 @@ class PlayHTHttpTTSService(TTSService): return language_to_playht_language(language) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: options = self._create_options() diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 007ba958e..60083f55d 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -309,7 +309,7 @@ class RimeTTSService(AudioContextWordTTSService): Yields: Frames containing audio data and timing information. """ - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: if not self._websocket: await self._connect() @@ -376,7 +376,7 @@ class RimeHttpTTSService(TTSService): return True async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") headers = { "Accept": "audio/pcm", diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 8d5ce79c4..de065aef4 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -101,7 +101,7 @@ class FastPitchTTSService(TTSService): await self.start_ttfb_metrics() yield TTSStartedFrame() - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") try: queue = asyncio.Queue() diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index e275b50b0..d2702d520 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -118,7 +118,7 @@ class XTTSService(TTSService): self._studio_speakers = await r.json() async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - logger.debug(f"Generating TTS: [{text}]") + logger.debug(f"{self}: Generating TTS [{text}]") if not self._studio_speakers: logger.error(f"{self} no studio speakers available") From 26c68ccd7c3ddcf9ffcea2b576c1a4f273c03426 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 26 Feb 2025 18:59:06 -0500 Subject: [PATCH 231/427] Add a new generic server to client message and frame type --- CHANGELOG.md | 10 ++++++++++ src/pipecat/frames/frames.py | 16 ++++++++++++++-- src/pipecat/processors/frameworks/rtvi.py | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7b62e9b..9b6de3430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added a new frame, `ServerMessageFrame`, and RTVI message `RTVIServerMessage` + which provides a generic mechanism for sending custom messages from server to + client. The `ServerMessageFrame` can be processed by either a `RTVIProcessor` + or `RTVIObserver` and will be delivered to the client's `onServerMessage` + callback. + ## [0.0.58] - 2025-02-26 ### Added diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index b43aaaeae..b771b765f 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -568,7 +568,8 @@ class UserStoppedSpeakingFrame(SystemFrame): @dataclass class EmulateUserStartedSpeakingFrame(SystemFrame): """Emitted by internal processors upstream to emulate VAD behavior when a - user starts speaking.""" + user starts speaking. + """ pass @@ -576,7 +577,8 @@ class EmulateUserStartedSpeakingFrame(SystemFrame): @dataclass class EmulateUserStoppedSpeakingFrame(SystemFrame): """Emitted by internal processors upstream to emulate VAD behavior when a - user stops speaking.""" + user stops speaking. + """ pass @@ -704,6 +706,16 @@ class VisionImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})" +@dataclass +class ServerMessageFrame(SystemFrame): + """A frame for sending server messages to the client.""" + + data: Any + + def __str__(self): + return f"{self.name}(data: {self.data})" + + # # Control frames # diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index af55fbd42..4edef4203 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -38,6 +38,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, MetricsFrame, + ServerMessageFrame, StartFrame, SystemFrame, TranscriptionFrame, @@ -375,6 +376,12 @@ class RTVIMetricsMessage(BaseModel): data: Mapping[str, Any] +class RTVIServerMessage(BaseModel): + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL + type: Literal["server-message"] = "server-message" + data: Any + + class RTVIFrameProcessor(FrameProcessor): def __init__(self, direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs): super().__init__(**kwargs) @@ -710,6 +717,9 @@ class RTVIObserver(BaseObserver): mark_as_seen = False elif isinstance(frame, MetricsFrame): await self._handle_metrics(frame) + elif isinstance(frame, ServerMessageFrame): + message = RTVIServerMessage(data=frame.data) + await self.push_transport_message_urgent(message) if mark_as_seen: self._frames_seen.add(frame.id) @@ -895,6 +905,11 @@ class RTVIProcessor(FrameProcessor): async def handle_message(self, message: RTVIMessage): await self._message_queue.put(message) + async def _handle_server_message(self, frame: ServerMessageFrame): + """Handle server message frame by converting it to a transport message.""" + message = RTVIServerMessage(data=frame.data) + await self._push_transport_message(message) + async def handle_function_call( self, function_name: str, @@ -934,6 +949,9 @@ class RTVIProcessor(FrameProcessor): await self.push_frame(frame, direction) elif isinstance(frame, TransportMessageUrgentFrame): await self._handle_transport_message(frame) + elif isinstance(frame, ServerMessageFrame): + await self._handle_server_message(frame) + await self.push_frame(frame, direction) # All other system frames elif isinstance(frame, SystemFrame): await self.push_frame(frame, direction) From 53d403af4bce22a4e8e77c40a7eac5dfd6319c3e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 27 Feb 2025 12:50:43 -0500 Subject: [PATCH 232/427] Remove the RTVIServerMessage logic from the RTVIProcessor --- CHANGELOG.md | 6 +++--- src/pipecat/processors/frameworks/rtvi.py | 8 -------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b6de3430..cdf100d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a new frame, `ServerMessageFrame`, and RTVI message `RTVIServerMessage` which provides a generic mechanism for sending custom messages from server to - client. The `ServerMessageFrame` can be processed by either a `RTVIProcessor` - or `RTVIObserver` and will be delivered to the client's `onServerMessage` - callback. + client. The `ServerMessageFrame` is processed by the `RTVIObserver` and will + be delivered to the client's `onServerMessage` callback or `ServerMessage` + event. ## [0.0.58] - 2025-02-26 diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 4edef4203..9835a9648 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -905,11 +905,6 @@ class RTVIProcessor(FrameProcessor): async def handle_message(self, message: RTVIMessage): await self._message_queue.put(message) - async def _handle_server_message(self, frame: ServerMessageFrame): - """Handle server message frame by converting it to a transport message.""" - message = RTVIServerMessage(data=frame.data) - await self._push_transport_message(message) - async def handle_function_call( self, function_name: str, @@ -949,9 +944,6 @@ class RTVIProcessor(FrameProcessor): await self.push_frame(frame, direction) elif isinstance(frame, TransportMessageUrgentFrame): await self._handle_transport_message(frame) - elif isinstance(frame, ServerMessageFrame): - await self._handle_server_message(frame) - await self.push_frame(frame, direction) # All other system frames elif isinstance(frame, SystemFrame): await self.push_frame(frame, direction) From 803ea9d8bcfbfa6f18e3590328383cf2551979b1 Mon Sep 17 00:00:00 2001 From: Adrian Cowham Date: Thu, 27 Feb 2025 12:31:02 -0800 Subject: [PATCH 233/427] update the canonical client so that the audio recording is optional as long as there is a transcript --- examples/canonical-metrics/bot.py | 2 +- src/pipecat/services/canonical.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/examples/canonical-metrics/bot.py b/examples/canonical-metrics/bot.py index aa376c290..646636dc1 100644 --- a/examples/canonical-metrics/bot.py +++ b/examples/canonical-metrics/bot.py @@ -113,8 +113,8 @@ async def main(): llm, tts, transport.output(), - audio_buffer_processor, # captures audio into a buffer canonical, # uploads audio buffer to Canonical AI for metrics + audio_buffer_processor, # captures audio into a buffer context_aggregator.assistant(), ] ) diff --git a/src/pipecat/services/canonical.py b/src/pipecat/services/canonical.py index 1c268ab22..7b62273d1 100644 --- a/src/pipecat/services/canonical.py +++ b/src/pipecat/services/canonical.py @@ -62,17 +62,21 @@ class CanonicalMetricsService(AIService): self, *, aiohttp_session: aiohttp.ClientSession, - audio_buffer_processor: AudioBufferProcessor, call_id: str, assistant: str, api_key: str, api_url: str = "https://voiceapp.canonical.chat/api/v1", assistant_speaks_first: bool = True, output_dir: str = "recordings", + audio_buffer_processor: Optional[AudioBufferProcessor] = None, context: Optional[OpenAILLMContext] = None, **kwargs, ): super().__init__(**kwargs) + # Validate that at least one of audio_buffer_processor or context is provided + if audio_buffer_processor is None and context is None: + raise ValueError("At least one of audio_buffer_processor or context must be specified") + self._aiohttp_session = aiohttp_session self._audio_buffer_processor = audio_buffer_processor self._api_key = api_key @@ -85,16 +89,36 @@ class CanonicalMetricsService(AIService): async def stop(self, frame: EndFrame): await super().stop(frame) - await self._process_audio() + await self._process_completion() async def cancel(self, frame: CancelFrame): await super().cancel(frame) - await self._process_audio() + await self._process_completion() async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) await self.push_frame(frame, direction) + async def _process_completion(self): + if self._audio_buffer_processor is not None: + await self._process_audio() + elif self._context is not None: + await self._process_transcript() + + async def _process_transcript(self): + params = { + "callId": self._call_id, + "assistant": {"id": self._assistant, "speaksFirst": self._assistant_speaks_first}, + "transcript": self._context.messages, + } + response = await self._aiohttp_session.post( + f"{self._api_url}/call", + headers=self._request_headers(), + json=params, + ) + if not response.ok: + logger.error(f"Failed to process transcript: {await response.text()}") + async def _process_audio(self): audio_buffer_processor = self._audio_buffer_processor From 6018fc068cb2135ed2f574fbe29df923dcb2c20f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 27 Feb 2025 20:07:07 -0500 Subject: [PATCH 234/427] Rename ServerMessageFrame to RTVIServerMessageFrame and move to rtvi.py --- CHANGELOG.md | 10 +++++----- src/pipecat/frames/frames.py | 10 ---------- src/pipecat/processors/frameworks/rtvi.py | 13 +++++++++++-- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf100d41..7c746f7fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added a new frame, `ServerMessageFrame`, and RTVI message `RTVIServerMessage` - which provides a generic mechanism for sending custom messages from server to - client. The `ServerMessageFrame` is processed by the `RTVIObserver` and will - be delivered to the client's `onServerMessage` callback or `ServerMessage` - event. +- Added a new frame, `RTVIServerMessageFrame`, and RTVI message + `RTVIServerMessage` which provides a generic mechanism for sending custom + messages from server to client. The `RTVIServerMessageFrame` is processed by + the `RTVIObserver` and will be delivered to the client's `onServerMessage` + callback or `ServerMessage` event. ## [0.0.58] - 2025-02-26 diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index b771b765f..74dd2accb 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -706,16 +706,6 @@ class VisionImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})" -@dataclass -class ServerMessageFrame(SystemFrame): - """A frame for sending server messages to the client.""" - - data: Any - - def __str__(self): - return f"{self.name}(data: {self.data})" - - # # Control frames # diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 9835a9648..dac903862 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -38,7 +38,6 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMTextFrame, MetricsFrame, - ServerMessageFrame, StartFrame, SystemFrame, TranscriptionFrame, @@ -382,6 +381,16 @@ class RTVIServerMessage(BaseModel): data: Any +@dataclass +class RTVIServerMessageFrame(SystemFrame): + """A frame for sending server messages to the client.""" + + data: Any + + def __str__(self): + return f"{self.name}(data: {self.data})" + + class RTVIFrameProcessor(FrameProcessor): def __init__(self, direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs): super().__init__(**kwargs) @@ -717,7 +726,7 @@ class RTVIObserver(BaseObserver): mark_as_seen = False elif isinstance(frame, MetricsFrame): await self._handle_metrics(frame) - elif isinstance(frame, ServerMessageFrame): + elif isinstance(frame, RTVIServerMessageFrame): message = RTVIServerMessage(data=frame.data) await self.push_transport_message_urgent(message) From f0330469630cd03f29cc6a0dd4d9bcc6497c6799 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Fri, 28 Feb 2025 08:25:15 +0530 Subject: [PATCH 235/427] using ruff automated formatting to avoid repeated failures --- .pre-commit-config.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6906b571b..d48d8bb8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,8 @@ repos: - - repo: local + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.7 hooks: - - id: ruff-format-hook - name: Check ruff formatting - entry: sh scripts/pre-commit.sh - language: system + - id: ruff + language_version: python3 + args: [ --select, I, ] + - id: ruff-format From 4824220260770f2561e51df5523d1ee486f36d9e Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Thu, 27 Feb 2025 21:30:06 +0530 Subject: [PATCH 236/427] adding GoogleLLMOpenAIBetaService --- CHANGELOG.md | 7 +++++++ src/pipecat/services/google/google.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7b62e9b..d294ab6e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added `GoogleLLMOpenAIBetaService` for Google LLM integration with an + OpenAI-compatible interface. + ## [0.0.58] - 2025-02-26 ### Added diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index c0941ee33..cd51f35e8 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -54,6 +54,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import ImageGenService, LLMService, STTService, TTSService from pipecat.services.google.frames import LLMSearchResponseFrame from pipecat.services.openai import ( + BaseOpenAILLMService, OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) @@ -1188,6 +1189,22 @@ class GoogleLLMService(LLMService): return GoogleContextAggregatorPair(_user=user, _assistant=assistant) +class GoogleLLMOpenAIBetaService(BaseOpenAILLMService): + """This class implements inference with Google's AI LLM models using the OpenAI format. + Ref - https://ai.google.dev/gemini-api/docs/openai + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://generativelanguage.googleapis.com/v1beta/openai/", + model: str = "gemini-2.0-flash", + **kwargs, + ): + super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + + class GoogleTTSService(TTSService): class InputParams(BaseModel): pitch: Optional[str] = None From 59fb6313909b9bb4929ca3e6337010a43bdc1a72 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Thu, 27 Feb 2025 22:55:25 +0530 Subject: [PATCH 237/427] fixing function calling and adding example --- CHANGELOG.md | 3 +- ...o-function-calling-gemini-openai-format.py | 137 ++++++++++++++++++ src/pipecat/services/google/google.py | 104 ++++++++++++- 3 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 examples/foundational/14o-function-calling-gemini-openai-format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d294ab6e5..700ce425c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added `GoogleLLMOpenAIBetaService` for Google LLM integration with an - OpenAI-compatible interface. + OpenAI-compatible interface. Added foundational example + `14o-function-calling-gemini-openai-format.py`. ## [0.0.58] - 2025-02-26 diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py new file mode 100644 index 000000000..4b04f9285 --- /dev/null +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.google import GoogleLLMOpenAIBetaService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) + + llm = GoogleLLMOpenAIBetaService(api_key=os.getenv("GEMINI_API_KEY")) + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function( + "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather + ) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] + messages = [ + { + "role": "user", + "content": "Start a conversation with 'Hey there' to get the current weather.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index cd51f35e8..df8f5836a 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -12,6 +12,8 @@ import os import time from google.api_core.exceptions import DeadlineExceeded +from openai import AsyncStream +from openai.types.chat import ChatCompletionChunk # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" @@ -56,6 +58,8 @@ from pipecat.services.google.frames import LLMSearchResponseFrame from pipecat.services.openai import ( BaseOpenAILLMService, OpenAIAssistantContextAggregator, + OpenAILLMService, + OpenAIUnhandledFunctionException, OpenAIUserContextAggregator, ) from pipecat.transcriptions.language import Language @@ -1189,7 +1193,7 @@ class GoogleLLMService(LLMService): return GoogleContextAggregatorPair(_user=user, _assistant=assistant) -class GoogleLLMOpenAIBetaService(BaseOpenAILLMService): +class GoogleLLMOpenAIBetaService(OpenAILLMService): """This class implements inference with Google's AI LLM models using the OpenAI format. Ref - https://ai.google.dev/gemini-api/docs/openai """ @@ -1204,6 +1208,104 @@ class GoogleLLMOpenAIBetaService(BaseOpenAILLMService): ): super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + async def _process_context(self, context: OpenAILLMContext): + functions_list = [] + arguments_list = [] + tool_id_list = [] + func_idx = 0 + function_name = "" + arguments = "" + tool_call_id = "" + + await self.start_ttfb_metrics() + + chunk_stream: AsyncStream[ChatCompletionChunk] = await self._stream_chat_completions( + context + ) + + async for chunk in chunk_stream: + if chunk.usage: + tokens = LLMTokenUsage( + prompt_tokens=chunk.usage.prompt_tokens, + completion_tokens=chunk.usage.completion_tokens, + total_tokens=chunk.usage.total_tokens, + ) + await self.start_llm_usage_metrics(tokens) + + if chunk.choices is None or len(chunk.choices) == 0: + continue + + await self.stop_ttfb_metrics() + + if not chunk.choices[0].delta: + continue + + if chunk.choices[0].delta.tool_calls: + # We're streaming the LLM response to enable the fastest response times. + # For text, we just yield each chunk as we receive it and count on consumers + # to do whatever coalescing they need (eg. to pass full sentences to TTS) + # + # If the LLM is a function call, we'll do some coalescing here. + # If the response contains a function name, we'll yield a frame to tell consumers + # that they can start preparing to call the function with that name. + # We accumulate all the arguments for the rest of the streamed response, then when + # the response is done, we package up all the arguments and the function name and + # yield a frame containing the function name and the arguments. + logger.debug(f"Tool call: {chunk.choices[0].delta.tool_calls}") + tool_call = chunk.choices[0].delta.tool_calls[0] + if tool_call.index != func_idx: + functions_list.append(function_name) + arguments_list.append(arguments) + tool_id_list.append(tool_call_id) + function_name = "" + arguments = "" + tool_call_id = "" + func_idx += 1 + if tool_call.function and tool_call.function.name: + function_name += tool_call.function.name + tool_call_id = tool_call.id + if tool_call.function and tool_call.function.arguments: + # Keep iterating through the response to collect all the argument fragments + arguments += tool_call.function.arguments + elif chunk.choices[0].delta.content: + await self.push_frame(LLMTextFrame(chunk.choices[0].delta.content)) + + # if we got a function name and arguments, check to see if it's a function with + # a registered handler. If so, run the registered callback, save the result to + # the context, and re-prompt to get a chat answer. If we don't have a registered + # handler, raise an exception. + if function_name and arguments: + # added to the list as last function name and arguments not added to the list + functions_list.append(function_name) + arguments_list.append(arguments) + tool_id_list.append(tool_call_id) + + logger.debug( + f"Function list: {functions_list}, Arguments list: {arguments_list}, Tool ID list: {tool_id_list}" + ) + for index, (function_name, arguments, tool_id) in enumerate( + zip(functions_list, arguments_list, tool_id_list), start=1 + ): + if function_name == "": + # TODO: Remove the _process_context method once Google resolves the bug + # where the index is incorrectly set to None instead of returning the actual index, + # which currently results in an empty function name(''). + continue + if self.has_function(function_name): + run_llm = False + arguments = json.loads(arguments) + await self.call_function( + context=context, + function_name=function_name, + arguments=arguments, + tool_call_id=tool_id, + run_llm=run_llm, + ) + else: + raise OpenAIUnhandledFunctionException( + f"The LLM tried to call a function named '{function_name}', but there isn't a callback registered for that function." + ) + class GoogleTTSService(TTSService): class InputParams(BaseModel): From 8db9d161741b7f1c639b8a4d77dd47dc32eb83ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 27 Feb 2025 15:23:16 -0800 Subject: [PATCH 238/427] add new LLMFullResponseAggregator --- CHANGELOG.md | 3 + .../processors/aggregators/llm_response.py | 54 +++++++ tests/test_llm_response.py | 136 ++++++++++++++++++ 3 files changed, 193 insertions(+) create mode 100644 tests/test_llm_response.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 422fa2246..2b4660d89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `LLMFullResponseAggregator` to aggregate full LLM completions. At + every completion the `on_completion` event handler is triggered. + - Added a new frame, `RTVIServerMessageFrame`, and RTVI message `RTVIServerMessage` which provides a generic mechanism for sending custom messages from server to client. The `RTVIServerMessageFrame` is processed by diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index da8ca63aa..d8582e32f 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -22,6 +22,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, + LLMTextFrame, StartFrame, StartInterruptionFrame, TextFrame, @@ -36,6 +37,59 @@ from pipecat.processors.aggregators.openai_llm_context import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +class LLMFullResponseAggregator(FrameProcessor): + """This is an LLM aggregator that aggregates a full LLM completion. It + aggregates LLM text frames (tokens) received between + `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. Every full + completion is returned via the "on_completion" event handler: + + @aggregator.event_handler("on_completion") + async def on_completion( + aggregator: LLMFullResponseAggregator, + completion: str, + completed: bool, + ) + + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self._aggregation = "" + self._started = False + + self._register_event_handler("on_completion") + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartInterruptionFrame): + await self._call_event_handler("on_completion", self._aggregation, False) + self._aggregation = "" + self._started = False + elif isinstance(frame, LLMFullResponseStartFrame): + await self._handle_llm_start(frame) + elif isinstance(frame, LLMFullResponseEndFrame): + await self._handle_llm_end(frame) + elif isinstance(frame, LLMTextFrame): + await self._handle_llm_text(frame) + + await self.push_frame(frame, direction) + + async def _handle_llm_start(self, _: LLMFullResponseStartFrame): + self._started = True + + async def _handle_llm_end(self, _: LLMFullResponseEndFrame): + await self._call_event_handler("on_completion", self._aggregation, True) + self._started = False + self._aggregation = "" + + async def _handle_llm_text(self, frame: TextFrame): + if not self._started: + return + self._aggregation += frame.text + + class BaseLLMResponseAggregator(FrameProcessor): """This is the base class for all LLM response aggregators. These aggregators process incoming frames and aggregate content until they are diff --git a/tests/test_llm_response.py b/tests/test_llm_response.py new file mode 100644 index 000000000..93838a658 --- /dev/null +++ b/tests/test_llm_response.py @@ -0,0 +1,136 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.frames.frames import ( + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + StartInterruptionFrame, +) +from pipecat.processors.aggregators.llm_response import LLMFullResponseAggregator +from pipecat.tests.utils import SleepFrame, run_test + + +class TestLLMFullResponseAggregator(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + completion_ok = False + + aggregator = LLMFullResponseAggregator() + + @aggregator.event_handler("on_completion") + async def on_completion(aggregator, completion, completed): + nonlocal completion_ok + completion_ok = completion == "" and completed + + frames_to_send = [LLMFullResponseStartFrame(), LLMFullResponseEndFrame()] + expected_down_frames = [LLMFullResponseStartFrame, LLMFullResponseEndFrame] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + assert completion_ok + + async def test_simple(self): + completion_ok = False + + aggregator = LLMFullResponseAggregator() + + @aggregator.event_handler("on_completion") + async def on_completion(aggregator, completion, completed): + nonlocal completion_ok + completion_ok = completion == "Hello from Pipecat!" and completed + + frames_to_send = [ + LLMFullResponseStartFrame(), + LLMTextFrame("Hello from Pipecat!"), + LLMFullResponseEndFrame(), + ] + expected_down_frames = [LLMFullResponseStartFrame, LLMTextFrame, LLMFullResponseEndFrame] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + assert completion_ok + + async def test_multiple(self): + completion_ok = False + + aggregator = LLMFullResponseAggregator() + + @aggregator.event_handler("on_completion") + async def on_completion(aggregator, completion, completed): + nonlocal completion_ok + completion_ok = completion == "Hello from Pipecat!" and completed + + frames_to_send = [ + LLMFullResponseStartFrame(), + LLMTextFrame("Hello "), + LLMTextFrame("from "), + LLMTextFrame("Pipecat!"), + LLMFullResponseEndFrame(), + ] + expected_down_frames = [ + LLMFullResponseStartFrame, + LLMTextFrame, + LLMTextFrame, + LLMTextFrame, + LLMFullResponseEndFrame, + ] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + assert completion_ok + + async def test_interruption(self): + completion_ok = True + + completion_result = [("Hello ", False), ("Hello there!", True)] + completion_index = 0 + + aggregator = LLMFullResponseAggregator() + + @aggregator.event_handler("on_completion") + async def on_completion(aggregator, completion, completed): + nonlocal completion_result, completion_index, completion_ok + (completion_expected, completion_completed) = completion_result[completion_index] + completion_ok = ( + completion_ok + and completion == completion_expected + and completed == completion_completed + ) + completion_index += 1 + + frames_to_send = [ + LLMFullResponseStartFrame(), + LLMTextFrame("Hello "), + SleepFrame(), + StartInterruptionFrame(), + LLMFullResponseStartFrame(), + LLMTextFrame("Hello "), + LLMTextFrame("there!"), + LLMFullResponseEndFrame(), + ] + expected_down_frames = [ + LLMFullResponseStartFrame, + LLMTextFrame, + StartInterruptionFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + LLMTextFrame, + LLMFullResponseEndFrame, + ] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + assert completion_ok From 010d9103d4d8b61262ca56e15a2d0c834fe25c12 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 1 Mar 2025 15:39:19 -0800 Subject: [PATCH 239/427] support for Azure OpenAI Realtime API --- .../foundational/19a-azure-realtime-beta.py | 179 ++++++++++++++++++ .../services/openai_realtime_beta/azure.py | 66 +++++++ 2 files changed, 245 insertions(+) create mode 100644 examples/foundational/19a-azure-realtime-beta.py create mode 100644 src/pipecat/services/openai_realtime_beta/azure.py diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py new file mode 100644 index 000000000..14d034836 --- /dev/null +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -0,0 +1,179 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from datetime import datetime + +import aiohttp +import websockets +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.openai_realtime_beta import ( + AzureRealtimeBetaLLMService, + InputAudioTranscription, + SessionProperties, + TurnDetection, +) +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + temperature = 75 if args["format"] == "fahrenheit" else 24 + await result_callback( + { + "conditions": "nice", + "temperature": temperature, + "format": args["format"], + "timestamp": datetime.now().strftime("%Y%m%d_%H%M%S"), + } + ) + + +tools = [ + { + "type": "function", + "name": "get_current_weather", + "description": "Get the current weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + "required": ["location", "format"], + }, + } +] + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + transcription_enabled=False, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), + vad_audio_passthrough=True, + ), + ) + + session_properties = SessionProperties( + input_audio_transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + # turn_detection=TurnDetection(silence_duration_ms=1000), + # Or set to False to disable openai turn detection and use transport VAD + # turn_detection=False, + # tools=tools, + instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. + +Act like a human, but remember that you aren't a human and that you can't do human +things in the real world. Your voice and personality should be warm and engaging, with a lively and +playful tone. + +If interacting in a non-English language, start by using the standard accent or dialect familiar to +the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, +even if you're asked about them. +- +You are participating in a voice conversation. Keep your responses concise, short, and to the point +unless specifically asked to elaborate on a topic. + +Remember, your responses should be short. Just one or two sentences, usually.""", + ) + + llm = AzureRealtimeBetaLLMService( + api_key=os.getenv("AZURE_REALTIME_API_KEY"), + base_url=os.getenv("AZURE_REALTIME_BASE_URL"), + session_properties=session_properties, + start_audio_paused=False, + ) + + # you can either register a single function for all function calls, or specific functions + # llm.register_function(None, fetch_weather_from_api) + llm.register_function("get_current_weather", fetch_weather_from_api) + + # Create a standard OpenAI LLM context object using the normal messages format. The + # OpenAIRealtimeBetaLLMService will convert this internally to messages that the + # openai WebSocket API can understand. + context = OpenAILLMContext( + [{"role": "user", "content": "Say hello!"}], + # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}], + # [ + # { + # "role": "user", + # "content": [ + # {"type": "text", "text": "Say"}, + # {"type": "text", "text": "yo what's up!"}, + # ], + # } + # ], + tools, + ) + + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), + llm, # LLM + context_aggregator.assistant(), + transport.output(), # Transport bot output + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + # report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py new file mode 100644 index 000000000..5951b1d89 --- /dev/null +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +from loguru import logger + +from .openai import OpenAIRealtimeBetaLLMService + +try: + import websockets +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + + +class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): + """Subclass ofOpenAI Realtime API Service with adjustments for Azure's wss connection.""" + + def __init__( + self, + *, + api_key: str, + base_url: str, + **kwargs, + ): + """Constructor takes the same arguments as the parent class, OpenAIRealtimeBetaLLMService. + + Note that the following are required arguments: + api_key: The API key for the Azure OpenAI service. + base_url: The base URL for the Azure OpenAI service. + + All other constructor args are passed to the parent class OpenAIRealtimeBetaLLMService. + + base_url should be set to the full Azure endpoint URL including the api-version and the deployment name. For example, + + wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment + """ + super().__init__(base_url=base_url, api_key=api_key, **kwargs) + self.api_key = api_key + self.base_url = base_url + + async def _connect(self): + try: + if self._websocket: + # Here we assume that if we have a websocket, we are connected. We + # handle disconnections in the send/recv code paths. + return + + logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") + self._websocket = await websockets.connect( + uri=self.base_url, + extra_headers={ + "api-key": self.api_key, + }, + ) + self._receive_task = self.create_task(self._receive_task_handler()) + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._websocket = None From 6c9bb782b1b7115f157d4a96bdccfcfcd72a71f1 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 1 Mar 2025 15:42:20 -0800 Subject: [PATCH 240/427] add __init__.py --- src/pipecat/services/openai_realtime_beta/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index a1cbafecd..52b00f6c8 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,2 +1,3 @@ +from .azure import AzureRealtimeBetaLLMService from .events import InputAudioTranscription, SessionProperties, TurnDetection from .openai import OpenAIRealtimeBetaLLMService From 2f4d36a146e89f6e933af051b42e664ca9febacf Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 1 Mar 2025 15:44:10 -0800 Subject: [PATCH 241/427] docstring fixup --- src/pipecat/services/openai_realtime_beta/azure.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py index 5951b1d89..5f046b8b0 100644 --- a/src/pipecat/services/openai_realtime_beta/azure.py +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -21,7 +21,7 @@ except ModuleNotFoundError as e: class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): - """Subclass ofOpenAI Realtime API Service with adjustments for Azure's wss connection.""" + """Subclass of OpenAI Realtime API Service with adjustments for Azure's wss connection.""" def __init__( self, @@ -36,8 +36,6 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): api_key: The API key for the Azure OpenAI service. base_url: The base URL for the Azure OpenAI service. - All other constructor args are passed to the parent class OpenAIRealtimeBetaLLMService. - base_url should be set to the full Azure endpoint URL including the api-version and the deployment name. For example, wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment From 29310b4e924470b11d9295d0f418e6a277f589a6 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sun, 2 Mar 2025 08:19:44 -0500 Subject: [PATCH 242/427] Add speed as InputParam to ElevenLabs TTS services --- CHANGELOG.md | 3 +++ src/pipecat/services/elevenlabs.py | 24 ++++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4660d89..415fb1f74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `speed` as an `InputParam` for both `ElevenLabsTTSService` and + `ElevenLabsHttpTTSService`. + - Added new `LLMFullResponseAggregator` to aggregate full LLM completions. At every completion the `on_completion` event handler is triggered. diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 4e6c84307..336be5382 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -145,6 +145,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): similarity_boost: Optional[float] = None style: Optional[float] = None use_speaker_boost: Optional[bool] = None + speed: Optional[float] = None auto_mode: Optional[bool] = True @model_validator(mode="after") @@ -202,6 +203,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): "similarity_boost": params.similarity_boost, "style": params.style, "use_speaker_boost": params.use_speaker_boost, + "speed": params.speed, "auto_mode": str(params.auto_mode).lower(), } self.set_model_name(model) @@ -235,6 +237,8 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): voice_settings["style"] = self._settings["style"] if self._settings["use_speaker_boost"] is not None: voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"] + if self._settings["speed"] is not None: + voice_settings["speed"] = self._settings["speed"] else: if self._settings["style"] is not None: logger.warning( @@ -244,6 +248,10 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): logger.warning( "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." ) + if self._settings["speed"] is not None: + logger.warning( + "'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." + ) return voice_settings or None @@ -441,6 +449,7 @@ class ElevenLabsHttpTTSService(TTSService): similarity_boost: Optional[float] = None style: Optional[float] = None use_speaker_boost: Optional[bool] = None + speed: Optional[float] = None def __init__( self, @@ -470,6 +479,7 @@ class ElevenLabsHttpTTSService(TTSService): "similarity_boost": params.similarity_boost, "style": params.style, "use_speaker_boost": params.use_speaker_boost, + "speed": params.speed, } self.set_model_name(model) self.set_voice(voice_id) @@ -490,12 +500,14 @@ class ElevenLabsHttpTTSService(TTSService): self._settings["stability"] is not None and self._settings["similarity_boost"] is not None ): - voice_settings["stability"] = float(self._settings["stability"]) - voice_settings["similarity_boost"] = float(self._settings["similarity_boost"]) + voice_settings["stability"] = self._settings["stability"] + voice_settings["similarity_boost"] = self._settings["similarity_boost"] if self._settings["style"] is not None: - voice_settings["style"] = float(self._settings["style"]) + voice_settings["style"] = self._settings["style"] if self._settings["use_speaker_boost"] is not None: - voice_settings["use_speaker_boost"] = bool(self._settings["use_speaker_boost"]) + voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"] + if self._settings["speed"] is not None: + voice_settings["speed"] = self._settings["speed"] else: if self._settings["style"] is not None: logger.warning( @@ -505,6 +517,10 @@ class ElevenLabsHttpTTSService(TTSService): logger.warning( "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." ) + if self._settings["speed"] is not None: + logger.warning( + "'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." + ) return voice_settings or None From ad7f1eec121cd03753232c62a297d7e8a9177f11 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sun, 2 Mar 2025 08:27:29 -0500 Subject: [PATCH 243/427] Create a function to build voice_settings dictionary --- src/pipecat/services/elevenlabs.py | 103 ++++++++++++----------------- 1 file changed, 41 insertions(+), 62 deletions(-) diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 336be5382..c41e5de02 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -116,6 +116,44 @@ def output_format_from_sample_rate(sample_rate: int) -> str: return "pcm_16000" +def build_elevenlabs_voice_settings( + settings: Dict[str, Any], +) -> Optional[Dict[str, Union[float, bool]]]: + """Build voice settings dictionary for ElevenLabs based on provided settings. + + Args: + settings: Dictionary containing voice settings parameters + + Returns: + Dictionary of voice settings or None if required parameters are missing + """ + voice_settings = {} + if settings["stability"] is not None and settings["similarity_boost"] is not None: + voice_settings["stability"] = settings["stability"] + voice_settings["similarity_boost"] = settings["similarity_boost"] + if settings["style"] is not None: + voice_settings["style"] = settings["style"] + if settings["use_speaker_boost"] is not None: + voice_settings["use_speaker_boost"] = settings["use_speaker_boost"] + if settings["speed"] is not None: + voice_settings["speed"] = settings["speed"] + else: + if settings["style"] is not None: + logger.warning( + "'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." + ) + if settings["use_speaker_boost"] is not None: + logger.warning( + "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." + ) + if settings["speed"] is not None: + logger.warning( + "'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." + ) + + return voice_settings or None + + def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float ) -> List[Tuple[str, float]]: @@ -226,34 +264,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): return language_to_elevenlabs_language(language) def _set_voice_settings(self): - voice_settings = {} - if ( - self._settings["stability"] is not None - and self._settings["similarity_boost"] is not None - ): - voice_settings["stability"] = self._settings["stability"] - voice_settings["similarity_boost"] = self._settings["similarity_boost"] - if self._settings["style"] is not None: - voice_settings["style"] = self._settings["style"] - if self._settings["use_speaker_boost"] is not None: - voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"] - if self._settings["speed"] is not None: - voice_settings["speed"] = self._settings["speed"] - else: - if self._settings["style"] is not None: - logger.warning( - "'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." - ) - if self._settings["use_speaker_boost"] is not None: - logger.warning( - "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." - ) - if self._settings["speed"] is not None: - logger.warning( - "'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." - ) - - return voice_settings or None + return build_elevenlabs_voice_settings(self._settings) async def set_model(self, model: str): await super().set_model(model) @@ -489,40 +500,8 @@ class ElevenLabsHttpTTSService(TTSService): def can_generate_metrics(self) -> bool: return True - def _set_voice_settings(self) -> Optional[Dict[str, Union[float, bool]]]: - """Configure voice settings if stability and similarity_boost are provided. - - Returns: - Dictionary of voice settings or None if required parameters are missing. - """ - voice_settings: Dict[str, Union[float, bool]] = {} - if ( - self._settings["stability"] is not None - and self._settings["similarity_boost"] is not None - ): - voice_settings["stability"] = self._settings["stability"] - voice_settings["similarity_boost"] = self._settings["similarity_boost"] - if self._settings["style"] is not None: - voice_settings["style"] = self._settings["style"] - if self._settings["use_speaker_boost"] is not None: - voice_settings["use_speaker_boost"] = self._settings["use_speaker_boost"] - if self._settings["speed"] is not None: - voice_settings["speed"] = self._settings["speed"] - else: - if self._settings["style"] is not None: - logger.warning( - "'style' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." - ) - if self._settings["use_speaker_boost"] is not None: - logger.warning( - "'use_speaker_boost' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." - ) - if self._settings["speed"] is not None: - logger.warning( - "'speed' is set but will not be applied because 'stability' and 'similarity_boost' are not both set." - ) - - return voice_settings or None + def _set_voice_settings(self): + return build_elevenlabs_voice_settings(self._settings) async def start(self, frame: StartFrame): await super().start(frame) From 3213e85b7dbfe9d96928ab47dd4e1a6e9d437ce3 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 2 Mar 2025 12:16:50 -0800 Subject: [PATCH 244/427] CHANGELOG.md entry for AzureRealtimeBetaLLMService --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b4660d89..4c000939e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 OpenAI-compatible interface. Added foundational example `14o-function-calling-gemini-openai-format.py`. +- Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added + foundational example `19a-azure-realtime-beta.py`. + ## [0.0.58] - 2025-02-26 ### Added From 8e0dc1f256481499db365dfd7e60462a4aab1021 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 3 Mar 2025 10:08:45 -0500 Subject: [PATCH 245/427] Add the `permissions` property to `DailyMeetingTokenProperties` --- src/pipecat/transports/services/helpers/daily_rest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pipecat/transports/services/helpers/daily_rest.py b/src/pipecat/transports/services/helpers/daily_rest.py index 9becacebc..b64aba2d6 100644 --- a/src/pipecat/transports/services/helpers/daily_rest.py +++ b/src/pipecat/transports/services/helpers/daily_rest.py @@ -195,6 +195,10 @@ class DailyMeetingTokenProperties(BaseModel): default=None, description="Start cloud recording when the user joins the room. This can be used to always record and archive meetings, for example in a customer support context.", ) + permissions: Optional[dict] = Field( + default=None, + description="Specifies the initial default permissions for a non-meeting-owner participant joining a call.", + ) class DailyMeetingTokenParams(BaseModel): From 17a1f305729e24c720b292bc2f51f81d1b338494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 28 Feb 2025 18:02:11 -0800 Subject: [PATCH 246/427] LLMService: add user/assistant args to create_context_aggregator() --- CHANGELOG.md | 4 +++ src/pipecat/services/anthropic.py | 32 +++++++++++++---- .../services/gemini_multimodal_live/gemini.py | 34 +++++++++++++++--- src/pipecat/services/google/google.py | 32 +++++++++++++---- src/pipecat/services/grok.py | 32 +++++++++++++---- src/pipecat/services/openai.py | 32 +++++++++++++---- .../services/openai_realtime_beta/openai.py | 36 +++++++++++++++---- 7 files changed, 168 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61a95b3c2..4e6ae5c21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Allow passing user (`user_kwargs`) and assistant (`assistant_kwargs`) context + aggregator parameters when using `create_context_aggregator()`. The values are + passed as a mapping that will then be converted to arguments. + - Added `speed` as an `InputParam` for both `ElevenLabsTTSService` and `ElevenLabsHttpTTSService`. diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index f10ee7ccb..bae780e62 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -11,7 +11,7 @@ import io import json import re from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Mapping, Optional, Union import httpx from loguru import logger @@ -125,14 +125,34 @@ class AnthropicLLMService(LLMService): @staticmethod def create_context_aggregator( - context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> AnthropicContextAggregatorPair: + """Create an instance of AnthropicContextAggregatorPair from an + OpenAILLMContext. Constructor keyword arguments for both the user and + assistant aggregators can be provided. + + Args: + context (OpenAILLMContext): The LLM context. + user_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the user context aggregator constructor. Defaults + to an empty mapping. + assistant_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the assistant context aggregator + constructor. Defaults to an empty mapping. + + Returns: + AnthropicContextAggregatorPair: A pair of context aggregators, one + for the user and one for the assistant, encapsulated in an + AnthropicContextAggregatorPair. + + """ if isinstance(context, OpenAILLMContext): context = AnthropicLLMContext.from_openai_context(context) - user = AnthropicUserContextAggregator(context) - assistant = AnthropicAssistantContextAggregator( - context, expect_stripped_words=assistant_expect_stripped_words - ) + user = AnthropicUserContextAggregator(context, **user_kwargs) + assistant = AnthropicAssistantContextAggregator(context, **assistant_kwargs) return AnthropicContextAggregatorPair(_user=user, _assistant=assistant) async def _process_context(self, context: OpenAILLMContext): diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 6e7a1c0fa..934117c52 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -9,7 +9,7 @@ import base64 import json from dataclasses import dataclass from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Mapping, Optional import websockets from loguru import logger @@ -701,11 +701,37 @@ class GeminiMultimodalLiveLLMService(LLMService): await self.push_frame(TTSStoppedFrame()) def create_context_aggregator( - self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = False + self, + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> GeminiMultimodalLiveContextAggregatorPair: + """Create an instance of GeminiMultimodalLiveContextAggregatorPair from + an OpenAILLMContext. Constructor keyword arguments for both the user and + assistant aggregators can be provided. + + Args: + context (OpenAILLMContext): The LLM context. + user_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the user context aggregator constructor. Defaults + to an empty mapping. + assistant_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the assistant context aggregator + constructor. Defaults to an empty mapping. + + Returns: + GeminiMultimodalLiveContextAggregatorPair: A pair of context + aggregators, one for the user and one for the assistant, + encapsulated in an GeminiMultimodalLiveContextAggregatorPair. + + """ GeminiMultimodalLiveContext.upgrade(context) - user = GeminiMultimodalLiveUserContextAggregator(context) + user = GeminiMultimodalLiveUserContextAggregator(context, **user_kwargs) + + default_assistant_kwargs = {"expect_stripped_words": False} + default_assistant_kwargs.update(assistant_kwargs) assistant = GeminiMultimodalLiveAssistantContextAggregator( - context, expect_stripped_words=assistant_expect_stripped_words + context, **default_assistant_kwargs ) return GeminiMultimodalLiveContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 50d0aaf83..cbbc73b47 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -19,7 +19,7 @@ from openai.types.chat import ChatCompletionChunk os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" from dataclasses import dataclass -from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Union +from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, Union from loguru import logger from PIL import Image @@ -1182,14 +1182,34 @@ class GoogleLLMService(LLMService): @staticmethod def create_context_aggregator( - context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> GoogleContextAggregatorPair: + """Create an instance of GoogleContextAggregatorPair from an + OpenAILLMContext. Constructor keyword arguments for both the user and + assistant aggregators can be provided. + + Args: + context (OpenAILLMContext): The LLM context. + user_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the user context aggregator constructor. Defaults + to an empty mapping. + assistant_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the assistant context aggregator + constructor. Defaults to an empty mapping. + + Returns: + GoogleContextAggregatorPair: A pair of context aggregators, one for + the user and one for the assistant, encapsulated in an + GoogleContextAggregatorPair. + + """ if isinstance(context, OpenAILLMContext): context = GoogleLLMContext.upgrade_to_google(context) - user = GoogleUserContextAggregator(context) - assistant = GoogleAssistantContextAggregator( - context, expect_stripped_words=assistant_expect_stripped_words - ) + user = GoogleUserContextAggregator(context, **user_kwargs) + assistant = GoogleAssistantContextAggregator(context, **assistant_kwargs) return GoogleContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 3a4a5fb5e..1f1661cf4 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -7,7 +7,7 @@ import json from dataclasses import dataclass -from typing import Optional +from typing import Any, Mapping, Optional from loguru import logger @@ -208,10 +208,30 @@ class GrokLLMService(OpenAILLMService): @staticmethod def create_context_aggregator( - context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> GrokContextAggregatorPair: - user = OpenAIUserContextAggregator(context) - assistant = GrokAssistantContextAggregator( - context, expect_stripped_words=assistant_expect_stripped_words - ) + """Create an instance of GrokContextAggregatorPair from an + OpenAILLMContext. Constructor keyword arguments for both the user and + assistant aggregators can be provided. + + Args: + context (OpenAILLMContext): The LLM context. + user_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the user context aggregator constructor. Defaults + to an empty mapping. + assistant_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the assistant context aggregator + constructor. Defaults to an empty mapping. + + Returns: + GrokContextAggregatorPair: A pair of context aggregators, one for + the user and one for the assistant, encapsulated in an + GrokContextAggregatorPair. + + """ + user = OpenAIUserContextAggregator(context, **user_kwargs) + assistant = GrokAssistantContextAggregator(context, **assistant_kwargs) return GrokContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 71da3f5e2..425882d6f 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -8,7 +8,7 @@ import base64 import io import json from dataclasses import dataclass -from typing import Any, AsyncGenerator, Dict, List, Literal, Optional +from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional import aiohttp import httpx @@ -345,12 +345,32 @@ class OpenAILLMService(BaseOpenAILLMService): @staticmethod def create_context_aggregator( - context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> OpenAIContextAggregatorPair: - user = OpenAIUserContextAggregator(context) - assistant = OpenAIAssistantContextAggregator( - context, expect_stripped_words=assistant_expect_stripped_words - ) + """Create an instance of OpenAIContextAggregatorPair from an + OpenAILLMContext. Constructor keyword arguments for both the user and + assistant aggregators can be provided. + + Args: + context (OpenAILLMContext): The LLM context. + user_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the user context aggregator constructor. Defaults + to an empty mapping. + assistant_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the assistant context aggregator + constructor. Defaults to an empty mapping. + + Returns: + OpenAIContextAggregatorPair: A pair of context aggregators, one for + the user and one for the assistant, encapsulated in an + OpenAIContextAggregatorPair. + + """ + user = OpenAIUserContextAggregator(context, **user_kwargs) + assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs) return OpenAIContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index ff6f24c66..44ce45dd7 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -4,11 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio import base64 import json import time from dataclasses import dataclass +from typing import Any, Mapping from loguru import logger @@ -571,11 +571,35 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) def create_context_aggregator( - self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = False + self, + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> OpenAIContextAggregatorPair: + """Create an instance of OpenAIContextAggregatorPair from an + OpenAILLMContext. Constructor keyword arguments for both the user and + assistant aggregators can be provided. + + Args: + context (OpenAILLMContext): The LLM context. + user_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the user context aggregator constructor. Defaults + to an empty mapping. + assistant_kwargs (Mapping[str, Any], optional): Additional keyword + arguments for the assistant context aggregator + constructor. Defaults to an empty mapping. + + Returns: + OpenAIContextAggregatorPair: A pair of context aggregators, one for + the user and one for the assistant, encapsulated in an + OpenAIContextAggregatorPair. + + """ OpenAIRealtimeLLMContext.upgrade_to_realtime(context) - user = OpenAIRealtimeUserContextAggregator(context) - assistant = OpenAIRealtimeAssistantContextAggregator( - context, expect_stripped_words=assistant_expect_stripped_words - ) + user = OpenAIRealtimeUserContextAggregator(context, **user_kwargs) + + default_assistant_kwargs = {"expect_stripped_words": False} + default_assistant_kwargs.update(assistant_kwargs) + assistant = OpenAIRealtimeAssistantContextAggregator(context, **default_assistant_kwargs) return OpenAIContextAggregatorPair(_user=user, _assistant=assistant) From 5f1848d24b76f9b5afeed480e1aa43f838681979 Mon Sep 17 00:00:00 2001 From: allenmylath Date: Mon, 3 Mar 2025 23:21:11 +0530 Subject: [PATCH 247/427] Update gladia.py (#1317) * Update gladia.py According to gladia docs https://docs.gladia.io/api-reference/v2/live/init speech threshould value close to 1 enables gladia to better isolate speeech from noise. --- src/pipecat/services/gladia.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index f0a00b327..5082e9ea2 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -136,7 +136,7 @@ class GladiaSTTService(STTService): maximum_duration_without_endpointing: Optional[int] = 10 audio_enhancer: Optional[bool] = None words_accurate_timestamps: Optional[bool] = None - + speech_threshold: Optional[float] = .99 def __init__( self, *, @@ -144,11 +144,10 @@ class GladiaSTTService(STTService): url: str = "https://api.gladia.io/v2/live", confidence: float = 0.5, sample_rate: Optional[int] = None, - params: InputParams = InputParams(), + params: InputParams = InputParams(), **kwargs, ): super().__init__(sample_rate=sample_rate, **kwargs) - self._api_key = api_key self._url = url self._settings = { @@ -166,6 +165,7 @@ class GladiaSTTService(STTService): "maximum_duration_without_endpointing": params.maximum_duration_without_endpointing, "pre_processing": { "audio_enhancer": params.audio_enhancer, + "speech_threshold": params.speech_threshold, }, "realtime_processing": { "words_accurate_timestamps": params.words_accurate_timestamps, From ec36fef26ef3297433182010de2ddd4db4a90b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 3 Mar 2025 09:53:03 -0800 Subject: [PATCH 248/427] updated CHANGELOG and fix GladiaSTTService formatting --- CHANGELOG.md | 2 ++ src/pipecat/services/gladia.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6ae5c21..7e30fe4be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `speech_threshold` parameter to `GladiaSTTService`. + - Allow passing user (`user_kwargs`) and assistant (`assistant_kwargs`) context aggregator parameters when using `create_context_aggregator()`. The values are passed as a mapping that will then be converted to arguments. diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index 5082e9ea2..09bcd7aa3 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -136,7 +136,8 @@ class GladiaSTTService(STTService): maximum_duration_without_endpointing: Optional[int] = 10 audio_enhancer: Optional[bool] = None words_accurate_timestamps: Optional[bool] = None - speech_threshold: Optional[float] = .99 + speech_threshold: Optional[float] = 0.99 + def __init__( self, *, @@ -144,7 +145,7 @@ class GladiaSTTService(STTService): url: str = "https://api.gladia.io/v2/live", confidence: float = 0.5, sample_rate: Optional[int] = None, - params: InputParams = InputParams(), + params: InputParams = InputParams(), **kwargs, ): super().__init__(sample_rate=sample_rate, **kwargs) From c14b85c12bf7655ea575e9ed43094ea09945d46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 4 Mar 2025 10:26:11 -0800 Subject: [PATCH 249/427] pyproject: update pyht to 0.1.12 Fixes #1309 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6c91f7f99..dd58b1bfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "websockets~=13.1" ] openpipe = [ "openpipe~=4.45.0" ] perplexity = [] -playht = [ "pyht~=0.1.6", "websockets~=13.1" ] +playht = [ "pyht~=0.1.12", "websockets~=13.1" ] rime = [ "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.18.0" ] sentry = [ "sentry-sdk~=2.20.0" ] From 3fe7c1d73062b1fbb70a45b606d681fbcf24585b Mon Sep 17 00:00:00 2001 From: Michael Louis Date: Tue, 4 Mar 2025 13:59:03 -0500 Subject: [PATCH 250/427] Added ultravox service --- src/pipecat/services/ultravox.py | 393 +++++++++++++++++++++++++++++++ 1 file changed, 393 insertions(+) create mode 100644 src/pipecat/services/ultravox.py diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py new file mode 100644 index 000000000..f39dc348f --- /dev/null +++ b/src/pipecat/services/ultravox.py @@ -0,0 +1,393 @@ +"""This module implements Ultravox speech-to-text with a locally-loaded model.""" + +import json +import time +import os +import numpy as np +from enum import Enum +from typing import AsyncGenerator, Optional, List +from loguru import logger +from pydantic import BaseModel +from huggingface_hub import login + +from pipecat.frames.frames import ( + Frame, + AudioRawFrame, + TranscriptionFrame, + TextFrame, + StartFrame, + EndFrame, + CancelFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + ErrorFrame +) +from pipecat.services.ai_services import AIService +from pipecat.processors.frame_processor import FrameDirection +from pipecat.utils.time import time_now_iso8601 + +try: + from vllm import SamplingParams, AsyncLLMEngine + from vllm.engine.arg_utils import AsyncEngineArgs + from transformers import AutoTokenizer +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Ultravox, you need to `pip install pipecat-ai[ultravox]`.") + raise Exception(f"Missing module: {e}") + +class AudioBuffer: + """Buffer to collect audio frames before processing. + + Attributes: + frames: List of AudioRawFrames to process + started_at: Timestamp when speech started + is_processing: Flag to prevent concurrent processing + """ + def __init__(self): + self.frames: List[AudioRawFrame] = [] + self.started_at: Optional[float] = None + self.is_processing: bool = False + +class UltravoxModel: + """Model wrapper for the Ultravox multimodal model. + + This class handles loading and running the Ultravox model for speech-to-text. + + Args: + model_name: The name or path of the Ultravox model to load + + Attributes: + model_name: The name of the loaded model + engine: The vLLM engine for model inference + tokenizer: The tokenizer for the model + stop_token_ids: Optional token IDs to stop generation + """ + def __init__(self, model_name: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b"): + self.model_name = model_name + self._initialize_engine() + self._initialize_tokenizer() + self.stop_token_ids = None + + def _initialize_engine(self): + """Initialize the vLLM engine for inference.""" + engine_args = AsyncEngineArgs( + model=self.model_name, + gpu_memory_utilization=0.9, + max_model_len=8192, + trust_remote_code=True + ) + self.engine = AsyncLLMEngine.from_engine_args(engine_args) + + def _initialize_tokenizer(self): + """Initialize the tokenizer for the model.""" + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) + + def format_prompt(self, messages: list): + """Format chat messages into a prompt for the model. + + Args: + messages: List of message dictionaries with 'role' and 'content' + + Returns: + str: Formatted prompt string + """ + return self.tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True + ) + + async def generate(self, messages: list, temperature: float = 0.7, max_tokens: int = 100, audio: np.ndarray = None): + """Generate text from audio input using the model. + + Args: + messages: List of message dictionaries + temperature: Sampling temperature + max_tokens: Maximum tokens to generate + audio: Audio data as numpy array + + Yields: + str: JSON chunks of the generated response + """ + sampling_params = SamplingParams( + temperature=temperature, + max_tokens=max_tokens, + stop_token_ids=self.stop_token_ids + ) + + mm_data = { + "audio": audio + } + inputs = {"prompt": self.format_prompt(messages), "multi_modal_data": mm_data} + results_generator = self.engine.generate(inputs, sampling_params, str(time.time())) + + previous_text = "" + first_chunk = True + + async for output in results_generator: + prompt_output = output.outputs + new_text = prompt_output[0].text[len(previous_text):] + previous_text = prompt_output[0].text + + # Construct OpenAI-compatible chunk + chunk = { + "id": str(int(time.time() * 1000)), + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": self.model_name, + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": None, + } + ], + } + + # Include the role in the first chunk + if first_chunk: + chunk["choices"][0]["delta"]["role"] = "assistant" + first_chunk = False + + # Add new text to the delta if any + if new_text: + chunk["choices"][0]["delta"]["content"] = new_text + + # Capture a finish reason if it's provided + finish_reason = prompt_output[0].finish_reason or None + if finish_reason and finish_reason != "none": + chunk["choices"][0]["finish_reason"] = finish_reason + + yield json.dumps(chunk) + +class UltravoxSTTService(AIService): + """Service to transcribe audio using the Ultravox multimodal model. + + This service collects audio frames and processes them with Ultravox + to generate text transcriptions. + + Args: + model_size: The Ultravox model to use (ModelSize enum or string) + hf_token: Hugging Face token for model access + temperature: Sampling temperature for generation + max_tokens: Maximum tokens to generate + **kwargs: Additional arguments passed to AIService + + Attributes: + model: The UltravoxModel instance + buffer: Buffer to collect audio frames + temperature: Temperature for text generation + max_tokens: Maximum tokens to generate + _connection_active: Flag indicating if service is active + """ + def __init__( + self, + *, + model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b", + hf_token: Optional[str] = None, + temperature: float = 0.7, + max_tokens: int = 100, + **kwargs, + ): + super().__init__(**kwargs) + + # Authenticate with Hugging Face if token provided + if hf_token: + login(token=hf_token) + elif os.environ.get("HF_TOKEN"): + login(token=os.environ.get("HF_TOKEN")) + else: + logger.warning("No Hugging Face token provided. Model may not load correctly.") + + # Initialize model + model_name = model_size if isinstance(model_size, str) else model_size.value + self.model = UltravoxModel(model_name=model_name) + + # Initialize service state + self.buffer = AudioBuffer() + self.temperature = temperature + self.max_tokens = max_tokens + self._connection_active = False + + logger.info(f"Initialized UltravoxSTTService with model: {model_name}") + + def can_generate_metrics(self) -> bool: + """Indicates whether this service can generate metrics. + + Returns: + bool: True, as this service supports metric generation. + """ + return True + + async def start(self, frame: StartFrame): + """Handle service start. + + Args: + frame: StartFrame that triggered this method + """ + await super().start(frame) + self._connection_active = True + logger.info("UltravoxSTTService started") + + async def stop(self, frame: EndFrame): + """Handle service stop. + + Args: + frame: EndFrame that triggered this method + """ + await super().stop(frame) + self._connection_active = False + logger.info("UltravoxSTTService stopped") + + async def cancel(self, frame: CancelFrame): + """Handle service cancellation. + + Args: + frame: CancelFrame that triggered this method + """ + await super().cancel(frame) + self._connection_active = False + self.buffer = AudioBuffer() + logger.info("UltravoxSTTService cancelled") + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames. + + This method collects audio frames and processes them when speech ends. + + Args: + frame: The frame to process + direction: Direction of the frame (input/output) + """ + await super().process_frame(frame, direction) + + if isinstance(frame, UserStartedSpeakingFrame): + logger.info("Speech started") + self.buffer = AudioBuffer() + self.buffer.started_at = time.time() + + elif isinstance(frame, AudioRawFrame) and self.buffer.started_at is not None: + self.buffer.frames.append(frame) + + elif isinstance(frame, UserStoppedSpeakingFrame): + if self.buffer.frames and not self.buffer.is_processing: + logger.info("Speech ended, processing buffer...") + await self.process_generator(self._process_audio_buffer()) + return # Return early to avoid pushing None frame + + # Only push the original frame if we haven't processed audio + if frame is not None: + await self.push_frame(frame, direction) + + async def _process_audio_buffer(self) -> AsyncGenerator[Frame, None]: + """Process collected audio frames with Ultravox. + + This method concatenates audio frames, processes them with the model, + and yields the resulting text frames. + + Yields: + Frame: TextFrame containing the transcribed text + """ + try: + self.buffer.is_processing = True + + # Check if we have valid frames before processing + if not self.buffer.frames: + logger.warning("No audio frames to process") + yield ErrorFrame("No audio frames to process") + return + + # Process audio frames + audio_arrays = [] + for f in self.buffer.frames: + if hasattr(f, 'audio') and f.audio: + # Handle bytes data - these are int16 PCM samples + if isinstance(f.audio, bytes): + try: + # Convert bytes to int16 array + arr = np.frombuffer(f.audio, dtype=np.int16) + if arr.size > 0: # Check if array is not empty + audio_arrays.append(arr) + except Exception as e: + logger.error(f"Error processing bytes audio frame: {e}") + # Handle numpy array data + elif isinstance(f.audio, np.ndarray): + if f.audio.size > 0: # Check if array is not empty + # Ensure it's int16 data + if f.audio.dtype != np.int16: + logger.info(f"Converting array from {f.audio.dtype} to int16") + audio_arrays.append(f.audio.astype(np.int16)) + else: + audio_arrays.append(f.audio) + + # Only proceed if we have valid audio arrays + if not audio_arrays: + logger.warning("No valid audio data found in frames") + yield ErrorFrame("No valid audio data found in frames") + return + + # Concatenate audio frames - all should be int16 now + audio_data = np.concatenate(audio_arrays) + + # Generate text using the model + if self.model: + try: + logger.info("Generating text from audio using model...") + full_response = "" + + # Start metrics tracking + await self.start_ttfb_metrics() + await self.start_processing_metrics() + + async for response in self.model.generate( + messages=[{ + 'role': 'user', + 'content': "<|audio|>\n" + }], + temperature=self.temperature, + max_tokens=self.max_tokens, + audio=audio_data + ): + # Stop TTFB metrics after first response + await self.stop_ttfb_metrics() + + chunk = json.loads(response) + if "choices" in chunk and len(chunk["choices"]) > 0: + delta = chunk["choices"][0]["delta"] + if "content" in delta: + new_text = delta["content"] + full_response += new_text + + # Stop processing metrics after completion + await self.stop_processing_metrics() + + logger.info(f"Generated text: {full_response}") + + # Create a transcription frame with the generated text + transcription = full_response.strip() + if transcription: + yield TranscriptionFrame( + text=transcription, + interim_text="", + timestamp=time_now_iso8601() + ) + else: + logger.warning("Empty transcription result") + yield ErrorFrame("Empty transcription result") + + except Exception as e: + logger.error(f"Error generating text from model: {e}") + yield ErrorFrame(f"Error generating text: {str(e)}") + else: + logger.warning("No model available for text generation") + yield ErrorFrame("No model available for text generation") + + except Exception as e: + logger.error(f"Error processing audio buffer: {e}") + import traceback + logger.error(traceback.format_exc()) + yield ErrorFrame(f"Error processing audio: {str(e)}") + finally: + self.buffer.is_processing = False + self.buffer.frames = [] + self.buffer.started_at = None From 5967ac0d4f2e3867eb46c2569d1b77e20521a434 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:10:32 -0300 Subject: [PATCH 251/427] Implementing unified format for function calling. --- src/pipecat/adapters/__init__.py | 0 src/pipecat/adapters/base_llm_adapter.py | 22 ++++++++ src/pipecat/adapters/schemas/__init__.py | 0 .../adapters/schemas/function_schema.py | 55 +++++++++++++++++++ src/pipecat/adapters/schemas/tools_schema.py | 43 +++++++++++++++ src/pipecat/adapters/services/__init__.py | 0 .../adapters/services/anthropic_adapter.py | 34 ++++++++++++ .../adapters/services/gemini_adapter.py | 28 ++++++++++ .../adapters/services/open_ai_adapter.py | 24 ++++++++ .../services/open_ai_realtime_adapter.py | 34 ++++++++++++ .../aggregators/openai_llm_context.py | 19 +++++-- src/pipecat/services/ai_services.py | 17 +++++- src/pipecat/services/anthropic.py | 9 ++- .../services/gemini_multimodal_live/gemini.py | 13 ++++- src/pipecat/services/google/google.py | 9 ++- src/pipecat/services/grok.py | 4 +- src/pipecat/services/openai.py | 3 +- .../services/openai_realtime_beta/openai.py | 7 +++ 18 files changed, 309 insertions(+), 12 deletions(-) create mode 100644 src/pipecat/adapters/__init__.py create mode 100644 src/pipecat/adapters/base_llm_adapter.py create mode 100644 src/pipecat/adapters/schemas/__init__.py create mode 100644 src/pipecat/adapters/schemas/function_schema.py create mode 100644 src/pipecat/adapters/schemas/tools_schema.py create mode 100644 src/pipecat/adapters/services/__init__.py create mode 100644 src/pipecat/adapters/services/anthropic_adapter.py create mode 100644 src/pipecat/adapters/services/gemini_adapter.py create mode 100644 src/pipecat/adapters/services/open_ai_adapter.py create mode 100644 src/pipecat/adapters/services/open_ai_realtime_adapter.py diff --git a/src/pipecat/adapters/__init__.py b/src/pipecat/adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/adapters/base_llm_adapter.py b/src/pipecat/adapters/base_llm_adapter.py new file mode 100644 index 000000000..c26722604 --- /dev/null +++ b/src/pipecat/adapters/base_llm_adapter.py @@ -0,0 +1,22 @@ +from abc import ABC, abstractmethod +from typing import Any, List, Union, cast + +from loguru import logger + +from pipecat.adapters.schemas.tools_schema import ToolsSchema + + +class BaseLLMAdapter(ABC): + @abstractmethod + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Any]: + """Converts tools to the provider's format.""" + pass + + def from_standard_tools(self, tools: Any) -> List[Any]: + if isinstance(tools, ToolsSchema): + logger.debug(f"Retrieving the tools using the adapter: {type(self)}") + return self.to_provider_tools_format(tools) + # Fallback to return the same tools in case they are not in a standard format + return tools + + # TODO: we can move the logic to also handle the Messages here diff --git a/src/pipecat/adapters/schemas/__init__.py b/src/pipecat/adapters/schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/adapters/schemas/function_schema.py b/src/pipecat/adapters/schemas/function_schema.py new file mode 100644 index 000000000..f6e59cef1 --- /dev/null +++ b/src/pipecat/adapters/schemas/function_schema.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Any, Dict, List + + +class FunctionSchema: + def __init__( + self, name: str, description: str, properties: Dict[str, Any], required: List[str] + ) -> None: + """Standardized function schema representation. + + :param name: Name of the function. + :param description: Description of the function. + :param properties: Dictionary defining properties types and descriptions. + :param required: List of required parameters. + """ + self._name = name + self._description = description + self._properties = properties + self._required = required + + def to_default_dict(self) -> Dict[str, Any]: + """Converts the function schema to a dictionary. + + :return: Dictionary representation of the function schema. + """ + return { + "name": self._name, + "description": self._description, + "parameters": { + "type": "object", + "properties": self._properties, + "required": self._required, + }, + } + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return self._description + + @property + def properties(self) -> Dict[str, Any]: + return self._properties + + @property + def required(self) -> List[str]: + return self._required diff --git a/src/pipecat/adapters/schemas/tools_schema.py b/src/pipecat/adapters/schemas/tools_schema.py new file mode 100644 index 000000000..5720535c5 --- /dev/null +++ b/src/pipecat/adapters/schemas/tools_schema.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from enum import Enum +from typing import Any, Dict, List + +from pipecat.adapters.schemas.function_schema import FunctionSchema + + +class AdapterType(Enum): + GEMINI = "gemini" # that is the only service where we are able to add custom tools for now + + +class ToolsSchema: + def __init__( + self, + standard_tools: List[FunctionSchema], + custom_tools: Dict[AdapterType, List[Dict[str, Any]]] = None, + ) -> None: + """ + A schema for tools that includes both standardized function schemas + and custom tools that do not follow the FunctionSchema format. + + :param standard_tools: List of tools following FunctionSchema. + :param custom_tools: List of tools in a custom format (e.g., search_tool). + """ + self._standard_tools = standard_tools + self._custom_tools = custom_tools + + @property + def standard_tools(self) -> List[FunctionSchema]: + return self._standard_tools + + @property + def custom_tools(self) -> Dict[AdapterType, List[Dict[str, Any]]]: + return self._custom_tools + + @custom_tools.setter + def custom_tools(self, value: Dict[AdapterType, List[Dict[str, Any]]]) -> None: + self._custom_tools = value diff --git a/src/pipecat/adapters/services/__init__.py b/src/pipecat/adapters/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/adapters/services/anthropic_adapter.py b/src/pipecat/adapters/services/anthropic_adapter.py new file mode 100644 index 000000000..a699469d3 --- /dev/null +++ b/src/pipecat/adapters/services/anthropic_adapter.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Any, Dict, List, Union + +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema + + +class AnthropicLLMAdapter(BaseLLMAdapter): + @staticmethod + def _to_anthropic_function_format(function: FunctionSchema) -> Dict[str, Any]: + return { + "name": function.name, + "description": function.description, + "input_schema": { + "type": "object", + "properties": function.properties, + "required": function.required, + }, + } + + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]: + """Converts function schemas to Anthropic's function-calling format. + + :return: Anthropic formatted function call definition. + """ + + functions_schema = tools_schema.standard_tools + return [self._to_anthropic_function_format(func) for func in functions_schema] diff --git a/src/pipecat/adapters/services/gemini_adapter.py b/src/pipecat/adapters/services/gemini_adapter.py new file mode 100644 index 000000000..8efca5189 --- /dev/null +++ b/src/pipecat/adapters/services/gemini_adapter.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Any, Dict, List, Union + +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema + + +class GeminiLLMAdapter(BaseLLMAdapter): + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]: + """Converts function schemas to Gemini's function-calling format. + + :return: Gemini formatted function call definition. + """ + + functions_schema = tools_schema.standard_tools + formatted_standard_tools = [ + {"function_declarations": [func.to_default_dict() for func in functions_schema]} + ] + custom_gemini_tools = [] + if tools_schema.custom_tools: + custom_gemini_tools = tools_schema.custom_tools.get(AdapterType.GEMINI, []) + + return formatted_standard_tools + custom_gemini_tools diff --git a/src/pipecat/adapters/services/open_ai_adapter.py b/src/pipecat/adapters/services/open_ai_adapter.py new file mode 100644 index 000000000..909e5103a --- /dev/null +++ b/src/pipecat/adapters/services/open_ai_adapter.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +from typing import List + +from openai.types.chat import ChatCompletionToolParam + +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.schemas.tools_schema import ToolsSchema + + +class OpenAILLMAdapter(BaseLLMAdapter): + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]: + """Converts function schemas to OpenAI's function-calling format. + + :return: OpenAI formatted function call definition. + """ + functions_schema = tools_schema.standard_tools + return [ + ChatCompletionToolParam(type="function", function=func.to_default_dict()) + for func in functions_schema + ] diff --git a/src/pipecat/adapters/services/open_ai_realtime_adapter.py b/src/pipecat/adapters/services/open_ai_realtime_adapter.py new file mode 100644 index 000000000..b7eafaa81 --- /dev/null +++ b/src/pipecat/adapters/services/open_ai_realtime_adapter.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# +from typing import Any, Dict, List, Union + +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema + + +class OpenAIRealtimeLLMAdapter(BaseLLMAdapter): + @staticmethod + def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]: + return { + "type": "function", + "name": function.name, + "description": function.description, + "parameters": { + "type": "object", + "properties": function.properties, + "required": function.required, + }, + } + + def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]: + """Converts function schemas to Openai Realtime function-calling format. + + :return: Openai Realtime formatted function call definition. + """ + + functions_schema = tools_schema.standard_tools + return [self._to_openai_realtime_function_format(func) for func in functions_schema] diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 5ef8c090f..e8391d62b 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -20,6 +20,8 @@ from openai.types.chat import ( ) from PIL import Image +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import ( AudioRawFrame, Frame, @@ -44,13 +46,20 @@ class OpenAILLMContext: def __init__( self, messages: Optional[List[ChatCompletionMessageParam]] = None, - tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN, + tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, ): self._messages: List[ChatCompletionMessageParam] = messages if messages else [] self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice - self._tools: List[ChatCompletionToolParam] | NotGiven = tools + self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools self._user_image_request_context = {} + self._llm_adapter: Optional[BaseLLMAdapter] = None + + def get_llm_adapter(self) -> Optional[BaseLLMAdapter]: + return self._llm_adapter + + def set_llm_adapter(self, llm_adapter: BaseLLMAdapter): + self._llm_adapter = llm_adapter @staticmethod def from_messages(messages: List[dict]) -> "OpenAILLMContext": @@ -67,7 +76,9 @@ class OpenAILLMContext: return self._messages @property - def tools(self) -> List[ChatCompletionToolParam] | NotGiven: + def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]: + if self._llm_adapter: + return self._llm_adapter.from_standard_tools(self._tools) return self._tools @property @@ -152,7 +163,7 @@ class OpenAILLMContext: def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven): self._tool_choice = tool_choice - def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven = NOT_GIVEN): + def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN): if tools != NOT_GIVEN and len(tools) == 0: tools = NOT_GIVEN self._tools = tools diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 136afb47a..65c9b5d92 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -8,10 +8,12 @@ import asyncio import io import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple +from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple, Type from loguru import logger +from pipecat.adapters.base_llm_adapter import BaseLLMAdapter +from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter from pipecat.audio.utils import calculate_audio_volume, exp_smoothing from pipecat.frames.frames import ( AudioRawFrame, @@ -137,10 +139,23 @@ class AIService(FrameProcessor): class LLMService(AIService): """This class is a no-op but serves as a base class for LLM services.""" + # OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations. + # However, subclasses should override this with a more specific adapter when necessary. + adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter + def __init__(self, **kwargs): super().__init__(**kwargs) self._callbacks = {} self._start_callbacks = {} + self._adapter = self.adapter_class() + + def get_llm_adapter(self) -> BaseLLMAdapter: + return self._adapter + + def create_context_aggregator( + self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + ) -> Any: + pass self._register_event_handler("on_completion_timeout") diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index bae780e62..10a2ab7b7 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -18,6 +18,7 @@ from loguru import logger from PIL import Image from pydantic import BaseModel, Field +from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter from pipecat.frames.frames import ( Frame, FunctionCallInProgressFrame, @@ -85,6 +86,9 @@ class AnthropicLLMService(LLMService): use `AsyncAnthropicBedrock` and `AsyncAnthropicVertex` clients """ + # Overriding the default adapter to use the Anthropic one. + adapter_class = AnthropicLLMAdapter + class InputParams(BaseModel): enable_prompt_caching_beta: Optional[bool] = False max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1) @@ -123,8 +127,8 @@ class AnthropicLLMService(LLMService): def enable_prompt_caching_beta(self) -> bool: return self._enable_prompt_caching_beta - @staticmethod def create_context_aggregator( + self, context: OpenAILLMContext, *, user_kwargs: Mapping[str, Any] = {}, @@ -149,6 +153,8 @@ class AnthropicLLMService(LLMService): AnthropicContextAggregatorPair. """ + context.set_llm_adapter(self.get_llm_adapter()) + if isinstance(context, OpenAILLMContext): context = AnthropicLLMContext.from_openai_context(context) user = AnthropicUserContextAggregator(context, **user_kwargs) @@ -382,6 +388,7 @@ class AnthropicLLMContext(OpenAILLMContext): tools=openai_context.tools, tool_choice=openai_context.tool_choice, ) + self.set_llm_adapter(openai_context.get_llm_adapter()) self._restructure_from_openai_messages() return self diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 934117c52..ef49df329 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -9,12 +9,14 @@ import base64 import json from dataclasses import dataclass from enum import Enum -from typing import Any, Dict, List, Mapping, Optional +from typing import Any, Dict, List, Mapping, Optional, Union import websockets from loguru import logger from pydantic import BaseModel, Field +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, @@ -152,6 +154,9 @@ class InputParams(BaseModel): class GeminiMultimodalLiveLLMService(LLMService): + # Overriding the default adapter to use the Gemini one. + adapter_class = GeminiLLMAdapter + def __init__( self, *, @@ -162,7 +167,7 @@ class GeminiMultimodalLiveLLMService(LLMService): start_audio_paused: bool = False, start_video_paused: bool = False, system_instruction: Optional[str] = None, - tools: Optional[List[dict]] = None, + tools: Optional[Union[List[dict], ToolsSchema]] = None, transcribe_user_audio: bool = False, transcribe_model_audio: bool = False, params: InputParams = InputParams(), @@ -435,7 +440,7 @@ class GeminiMultimodalLiveLLMService(LLMService): ) if self._tools: logger.debug(f"Gemini is configuring to use tools{self._tools}") - config.setup.tools = self._tools + config.setup.tools = self.get_llm_adapter().from_standard_tools(self._tools) await self.send_client_event(config) except Exception as e: @@ -726,6 +731,8 @@ class GeminiMultimodalLiveLLMService(LLMService): encapsulated in an GeminiMultimodalLiveContextAggregatorPair. """ + context.set_llm_adapter(self.get_llm_adapter()) + GeminiMultimodalLiveContext.upgrade(context) user = GeminiMultimodalLiveUserContextAggregator(context, **user_kwargs) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index cbbc73b47..1d914a9bb 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -15,6 +15,8 @@ from google.api_core.exceptions import DeadlineExceeded from openai import AsyncStream from openai.types.chat import ChatCompletionChunk +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter + # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" @@ -950,6 +952,9 @@ class GoogleLLMService(LLMService): franca for all LLM services, so that it is easy to switch between different LLMs. """ + # Overriding the default adapter to use the Gemini one. + adapter_class = GeminiLLMAdapter + class InputParams(BaseModel): max_tokens: Optional[int] = Field(default=4096, ge=1) temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) @@ -1180,8 +1185,8 @@ class GoogleLLMService(LLMService): if context: await self._process_context(context) - @staticmethod def create_context_aggregator( + self, context: OpenAILLMContext, *, user_kwargs: Mapping[str, Any] = {}, @@ -1206,6 +1211,8 @@ class GoogleLLMService(LLMService): GoogleContextAggregatorPair. """ + context.set_llm_adapter(self.get_llm_adapter()) + if isinstance(context, OpenAILLMContext): context = GoogleLLMContext.upgrade_to_google(context) user = GoogleUserContextAggregator(context, **user_kwargs) diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 1f1661cf4..cf7d74f59 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -206,8 +206,8 @@ class GrokLLMService(OpenAILLMService): if tokens.completion_tokens > self._completion_tokens: self._completion_tokens = tokens.completion_tokens - @staticmethod def create_context_aggregator( + self, context: OpenAILLMContext, *, user_kwargs: Mapping[str, Any] = {}, @@ -232,6 +232,8 @@ class GrokLLMService(OpenAILLMService): GrokContextAggregatorPair. """ + context.set_llm_adapter(self.get_llm_adapter()) + user = OpenAIUserContextAggregator(context, **user_kwargs) assistant = GrokAssistantContextAggregator(context, **assistant_kwargs) return GrokContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 425882d6f..5a3a993aa 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -343,8 +343,8 @@ class OpenAILLMService(BaseOpenAILLMService): ): super().__init__(model=model, params=params, **kwargs) - @staticmethod def create_context_aggregator( + self, context: OpenAILLMContext, *, user_kwargs: Mapping[str, Any] = {}, @@ -369,6 +369,7 @@ class OpenAILLMService(BaseOpenAILLMService): OpenAIContextAggregatorPair. """ + context.set_llm_adapter(self.get_llm_adapter()) user = OpenAIUserContextAggregator(context, **user_kwargs) assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs) return OpenAIContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 44ce45dd7..00f8cd840 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -12,6 +12,8 @@ from typing import Any, Mapping from loguru import logger +from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter + try: import websockets except ModuleNotFoundError as e: @@ -76,6 +78,9 @@ class OpenAIUnhandledFunctionException(Exception): class OpenAIRealtimeBetaLLMService(LLMService): + # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one. + adapter_class = OpenAIRealtimeLLMAdapter + def __init__( self, *, @@ -596,6 +601,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): OpenAIContextAggregatorPair. """ + context.set_llm_adapter(self.get_llm_adapter()) + OpenAIRealtimeLLMContext.upgrade_to_realtime(context) user = OpenAIRealtimeUserContextAggregator(context, **user_kwargs) From 0e55db054e2ce9eba5c5192455376bc0a9413ab1 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:10:47 -0300 Subject: [PATCH 252/427] Created script to fix ruff format issues. --- scripts/fix-ruff.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 scripts/fix-ruff.sh diff --git a/scripts/fix-ruff.sh b/scripts/fix-ruff.sh new file mode 100755 index 000000000..892f6d405 --- /dev/null +++ b/scripts/fix-ruff.sh @@ -0,0 +1,4 @@ +ruff format src +ruff format examples +ruff format tests +ruff check --select I --fix \ No newline at end of file From 5c912927bb9510a0731fc6723d51a89d32c7d917 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:11:02 -0300 Subject: [PATCH 253/427] Unit tests for function calling adapters. --- tests/test_function_calling_adapters.py | 176 ++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 tests/test_function_calling_adapters.py diff --git a/tests/test_function_calling_adapters.py b/tests/test_function_calling_adapters.py new file mode 100644 index 000000000..5d6dafce3 --- /dev/null +++ b/tests/test_function_calling_adapters.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from openai.types.chat import ChatCompletionToolParam + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema +from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter +from pipecat.adapters.services.gemini_adapter import GeminiLLMAdapter +from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter +from pipecat.adapters.services.open_ai_realtime_adapter import OpenAIRealtimeLLMAdapter + + +class TestFunctionAdapters(unittest.TestCase): + def setUp(self) -> None: + """Sets up a common tools schema for all tests.""" + function_def = FunctionSchema( + name="get_weather", + description="Get the weather in a given location", + properties={ + "location": {"type": "string", "description": "The city, e.g. San Francisco"}, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + required=["location", "format"], + ) + self.tools_def = ToolsSchema(standard_tools=[function_def]) + + def test_openai_adapter(self): + """Test OpenAI adapter format transformation.""" + expected = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] + assert OpenAILLMAdapter().to_provider_tools_format(self.tools_def) == expected + + def test_anthropic_adapter(self): + """Test Anthropic adapter format transformation.""" + expected = [ + { + "name": "get_weather", + "description": "Get the weather in a given location", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + } + ] + assert AnthropicLLMAdapter().to_provider_tools_format(self.tools_def) == expected + + def test_gemini_adapter(self): + """Test Gemini adapter format transformation.""" + expected = [ + { + "function_declarations": [ + { + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + } + ] + } + ] + assert GeminiLLMAdapter().to_provider_tools_format(self.tools_def) == expected + + def test_openai_realtime_adapter(self): + """Test Anthropic adapter format transformation.""" + expected = [ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + } + ] + assert OpenAIRealtimeLLMAdapter().to_provider_tools_format(self.tools_def) == expected + + def test_gemini_adapter_with_custom_tools(self): + """Test Gemini adapter format transformation.""" + search_tool = {"google_search": {}} + expected = [ + { + "function_declarations": [ + { + "name": "get_weather", + "description": "Get the weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city, e.g. San Francisco", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use.", + }, + }, + "required": ["location", "format"], + }, + } + ] + }, + search_tool, + ] + tools_def = self.tools_def + tools_def.custom_tools = {AdapterType.GEMINI: [search_tool]} + assert GeminiLLMAdapter().to_provider_tools_format(tools_def) == expected From ebcde719a67d5f7dff58bcf23adb46595bc40777 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:11:16 -0300 Subject: [PATCH 254/427] Integration test for function calling. --- ...st_integration_unified_function_calling.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/integration/test_integration_unified_function_calling.py diff --git a/tests/integration/test_integration_unified_function_calling.py b/tests/integration/test_integration_unified_function_calling.py new file mode 100644 index 000000000..88407d703 --- /dev/null +++ b/tests/integration/test_integration_unified_function_calling.py @@ -0,0 +1,96 @@ +import os +from unittest.mock import AsyncMock + +import pytest +from dotenv import load_dotenv + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.frames.frames import ( + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import LLMService +from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.google import GoogleLLMService +from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService +from pipecat.utils.test_frame_processor import TestFrameProcessor + +load_dotenv(override=True) + + +def standard_tools() -> ToolsSchema: + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location"], + ) + tools_def = ToolsSchema(standard_tools=[weather_function]) + return tools_def + + +async def _test_llm_function_calling(llm: LLMService): + # Create an AsyncMock for the function + mock_fetch_weather = AsyncMock() + + llm.register_function(None, mock_fetch_weather) + t = TestFrameProcessor([LLMFullResponseStartFrame, LLMTextFrame, LLMFullResponseEndFrame]) + llm.link(t) + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation.", + }, + {"role": "user", "content": " How is the weather today in San Francisco, California?"}, + ] + context = OpenAILLMContext(messages, standard_tools()) + # This is done by default inside the create_context_aggregator + context.set_llm_adapter(llm.get_llm_adapter()) + + frame = OpenAILLMContextFrame(context) + + # This will fail if an exception is raised + await llm.process_frame(frame, FrameDirection.DOWNSTREAM) + + # Assert that the mock function was called + mock_fetch_weather.assert_called_once() + + +@pytest.mark.skipif(os.getenv("OPENAI_API_KEY") is None, reason="OPENAI_API_KEY is not set") +@pytest.mark.asyncio +async def test_unified_function_calling_openai(): + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # This will fail if an exception is raised + await _test_llm_function_calling(llm) + + +@pytest.mark.skipif(os.getenv("GOOGLE_API_KEY") is None, reason="GOOGLE_API_KEY is not set") +@pytest.mark.asyncio +async def test_unified_function_calling_gemini(): + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") + # This will fail if an exception is raised + await _test_llm_function_calling(llm) + + +@pytest.mark.skipif(os.getenv("ANTHROPIC_API_KEY") is None, reason="ANTHROPIC_API_KEY is not set") +@pytest.mark.asyncio +async def test_unified_function_calling_anthropic(): + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620" + ) + # This will fail if an exception is raised + await _test_llm_function_calling(llm) From a840b0e815bb481603cfdca340f712f0887e7cfe Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:11:52 -0300 Subject: [PATCH 255/427] Prevents pytest from collecting TestFrameProcessor. --- src/pipecat/utils/test_frame_processor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py index fde476007..b35864497 100644 --- a/src/pipecat/utils/test_frame_processor.py +++ b/src/pipecat/utils/test_frame_processor.py @@ -8,6 +8,8 @@ class TestException(Exception): class TestFrameProcessor(FrameProcessor): + __test__ = False # Prevents pytest from collecting this class as a test + def __init__(self, test_frames): self.test_frames = test_frames self._list_counter = 0 From 2a75373c04b35937dd049159926ac784a929bf71 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:12:30 -0300 Subject: [PATCH 256/427] Created examples for unified format function calling. --- .../base_function_calling.py | 134 ++++++++++++++++++ .../multimodal_base_function_calling.py | 126 ++++++++++++++++ .../unified-format-function-calling/runner.py | 64 +++++++++ .../standard-function-calling-anthropic.py | 29 ++++ .../standard-function-calling-azure.py | 31 ++++ .../standard-function-calling-cerebras.py | 27 ++++ .../standard-function-calling-deepseek.py | 27 ++++ .../standard-function-calling-fireworks.py | 29 ++++ ...dard-function-calling-gemini-multimodal.py | 38 +++++ .../standard-function-calling-gemini.py | 27 ++++ .../standard-function-calling-grok.py | 27 ++++ .../standard-function-calling-groq.py | 27 ++++ .../standard-function-calling-nim.py | 29 ++++ ...andard-function-calling-openai-realtime.py | 43 ++++++ .../standard-function-calling-openai.py | 27 ++++ .../standard-function-calling-openrouter.py | 29 ++++ .../standard-function-calling-together.py | 30 ++++ 17 files changed, 744 insertions(+) create mode 100644 examples/unified-format-function-calling/base_function_calling.py create mode 100644 examples/unified-format-function-calling/multimodal_base_function_calling.py create mode 100644 examples/unified-format-function-calling/runner.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-anthropic.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-azure.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-cerebras.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-deepseek.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-fireworks.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-gemini.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-grok.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-groq.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-nim.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-openai-realtime.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-openai.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-openrouter.py create mode 100644 examples/unified-format-function-calling/standard-function-calling-together.py diff --git a/examples/unified-format-function-calling/base_function_calling.py b/examples/unified-format-function-calling/base_function_calling.py new file mode 100644 index 000000000..798d94465 --- /dev/null +++ b/examples/unified-format-function-calling/base_function_calling.py @@ -0,0 +1,134 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.ai_services import LLMService +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +load_dotenv(override=True) + + +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +class WeatherBot: + """Generic base class for setting up and running an LLM-powered bot.""" + + def __init__(self, llm: LLMService): + """Initialize the base handler with a specific LLM.""" + self.llm = llm + + async def run(self): + """Set up and start the processing pipeline.""" + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + self.llm.register_function( + None, fetch_weather_from_api, start_callback=start_fetch_weather + ) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation.", + }, + {"role": "user", "content": " Start the conversation by introducing yourself."}, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = self.llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + self.llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + await runner.run(task) diff --git a/examples/unified-format-function-calling/multimodal_base_function_calling.py b/examples/unified-format-function-calling/multimodal_base_function_calling.py new file mode 100644 index 000000000..8f4a51b96 --- /dev/null +++ b/examples/unified-format-function-calling/multimodal_base_function_calling.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.ai_services import LLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +load_dotenv(override=True) + + +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +class MultimodalWeatherBot: + """Generic base class for setting up and running an LLM-powered bot.""" + + def __init__(self, llm: LLMService): + """Initialize the base handler with a specific LLM.""" + self.llm = llm + + @staticmethod + def tools() -> ToolsSchema: + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location"], + ) + return ToolsSchema(standard_tools=[weather_function]) + + async def run(self): + """Set up and start the processing pipeline.""" + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + self.llm.register_function( + None, fetch_weather_from_api, start_callback=start_fetch_weather + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation.", + }, + {"role": "user", "content": " Start the conversation by introducing yourself."}, + ] + + context = OpenAILLMContext(messages, MultimodalWeatherBot.tools()) + context_aggregator = self.llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + self.llm, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + await runner.run(task) diff --git a/examples/unified-format-function-calling/runner.py b/examples/unified-format-function-calling/runner.py new file mode 100644 index 000000000..04157d549 --- /dev/null +++ b/examples/unified-format-function-calling/runner.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os +from typing import Optional + +import aiohttp + +from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper + + +async def configure(aiohttp_session: aiohttp.ClientSession): + (url, token, _) = await configure_with_args(aiohttp_session) + return (url, token) + + +async def configure_with_args( + aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None +): + if not parser: + parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=False, help="URL of the Daily room to join" + ) + parser.add_argument( + "-k", + "--apikey", + type=str, + required=False, + help="Daily API Key (needed to create an owner token for the room)", + ) + + args, unknown = parser.parse_known_args() + + url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL") + key = args.apikey or os.getenv("DAILY_API_KEY") + + if not url: + raise Exception( + "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL." + ) + + if not key: + raise Exception( + "No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers." + ) + + daily_rest_helper = DailyRESTHelper( + daily_api_key=key, + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=aiohttp_session, + ) + + # Create a meeting token for the given room with an expiration 1 hour in + # the future. + expiry_time: float = 60 * 60 + + token = await daily_rest_helper.get_token(url, expiry_time) + + return (url, token, args) diff --git a/examples/unified-format-function-calling/standard-function-calling-anthropic.py b/examples/unified-format-function-calling/standard-function-calling-anthropic.py new file mode 100644 index 000000000..7ae39b99a --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-anthropic.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.anthropic import AnthropicLLMService + +load_dotenv(override=True) + + +class AnthropicWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620" + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(AnthropicWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-azure.py b/examples/unified-format-function-calling/standard-function-calling-azure.py new file mode 100644 index 000000000..c1b24ca2a --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-azure.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.azure import AzureLLMService + +load_dotenv(override=True) + + +class AzureWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL"), + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(AzureWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-cerebras.py b/examples/unified-format-function-calling/standard-function-calling-cerebras.py new file mode 100644 index 000000000..6888268aa --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-cerebras.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.cerebras import CerebrasLLMService + +load_dotenv(override=True) + + +class CerebrasWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(CerebrasWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-deepseek.py b/examples/unified-format-function-calling/standard-function-calling-deepseek.py new file mode 100644 index 000000000..7c8cd6ebb --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-deepseek.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.deepseek import DeepSeekLLMService + +load_dotenv(override=True) + + +class DeepSeekWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(DeepSeekWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-fireworks.py b/examples/unified-format-function-calling/standard-function-calling-fireworks.py new file mode 100644 index 000000000..1128c1ada --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-fireworks.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.fireworks import FireworksLLMService + +load_dotenv(override=True) + + +class FireworksWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = FireworksLLMService( + api_key=os.getenv("FIREWORKS_API_KEY"), + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(FireworksWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py b/examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py new file mode 100644 index 000000000..7a479b6b3 --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from multimodal_base_function_calling import MultimodalWeatherBot + +from pipecat.adapters.schemas.tools_schema import AdapterType +from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService + +load_dotenv(override=True) + + +class GeminiMultimodalWeatherBot(MultimodalWeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + search_tool = {"google_search": {}} + tools_def = MultimodalWeatherBot.tools() + tools_def.custom_tools = {AdapterType.GEMINI: [search_tool]} + + llm = GeminiMultimodalLiveLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + voice_id="Puck", + transcribe_user_audio=True, + transcribe_model_audio=True, + tools=tools_def, + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(GeminiMultimodalWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-gemini.py b/examples/unified-format-function-calling/standard-function-calling-gemini.py new file mode 100644 index 000000000..d164c9e67 --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-gemini.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.google import GoogleLLMService + +load_dotenv(override=True) + + +class GeminiWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(GeminiWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-grok.py b/examples/unified-format-function-calling/standard-function-calling-grok.py new file mode 100644 index 000000000..3c2570d8a --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-grok.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.grok import GrokLLMService + +load_dotenv(override=True) + + +class GrokWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(GrokWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-groq.py b/examples/unified-format-function-calling/standard-function-calling-groq.py new file mode 100644 index 000000000..70a6cef47 --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-groq.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.groq import GroqLLMService + +load_dotenv(override=True) + + +class GroqWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(GroqWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-nim.py b/examples/unified-format-function-calling/standard-function-calling-nim.py new file mode 100644 index 000000000..f0d1e892b --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-nim.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.nim import NimLLMService + +load_dotenv(override=True) + + +class NimWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = NimLLMService( + api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(NimWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-openai-realtime.py b/examples/unified-format-function-calling/standard-function-calling-openai-realtime.py new file mode 100644 index 000000000..203d0abc3 --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-openai-realtime.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from dotenv import load_dotenv +from multimodal_base_function_calling import MultimodalWeatherBot + +from pipecat.services.openai_realtime_beta import ( + InputAudioTranscription, + OpenAIRealtimeBetaLLMService, + SessionProperties, + TurnDetection, +) + +load_dotenv(override=True) + + +class OpenAiRealTimeWeatherBot(MultimodalWeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + session_properties = SessionProperties( + input_audio_transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + turn_detection=TurnDetection(silence_duration_ms=1000), + ) + + llm = OpenAIRealtimeBetaLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + session_properties=session_properties, + start_audio_paused=False, + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(OpenAiRealTimeWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-openai.py b/examples/unified-format-function-calling/standard-function-calling-openai.py new file mode 100644 index 000000000..7763ee505 --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-openai.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.openai import OpenAILLMService + +load_dotenv(override=True) + + +class OpenAiWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(OpenAiWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-openrouter.py b/examples/unified-format-function-calling/standard-function-calling-openrouter.py new file mode 100644 index 000000000..cb1ad3964 --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-openrouter.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.openrouter import OpenRouterLLMService + +load_dotenv(override=True) + + +class OpenRouterWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = OpenRouterLLMService( + api_key=os.getenv("OPENROUTER_API_KEY"), model="openai/gpt-4o-2024-11-20" + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(OpenRouterWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-together.py b/examples/unified-format-function-calling/standard-function-calling-together.py new file mode 100644 index 000000000..fe100c95c --- /dev/null +++ b/examples/unified-format-function-calling/standard-function-calling-together.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os + +from base_function_calling import WeatherBot +from dotenv import load_dotenv + +from pipecat.services.together import TogetherLLMService + +load_dotenv(override=True) + + +class TogetherWeatherBot(WeatherBot): + """Main class defining the LLM and passing it to the base handler.""" + + def __init__(self): + llm = TogetherLLMService( + api_key=os.getenv("TOGETHER_API_KEY"), + model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", + ) + super().__init__(llm) + + +if __name__ == "__main__": + asyncio.run(TogetherWeatherBot().run()) From 76d36a312b29c227b60f6b5e8c4813f3c6642e82 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 5 Mar 2025 14:18:37 -0300 Subject: [PATCH 257/427] Adding the unified format function calling to the changelog. --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e30fe4be..4d5e4f862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for a unified format for specifying function calling across all LLM services. + ```python + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + ``` + - Added `speech_threshold` parameter to `GladiaSTTService`. - Allow passing user (`user_kwargs`) and assistant (`assistant_kwargs`) context From bb29e50adb36c67efa1335959ff01c7295f844d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 4 Mar 2025 23:03:27 -0800 Subject: [PATCH 258/427] introduce BaseObject class --- CHANGELOG.md | 46 ++++++++++-------- src/pipecat/audio/vad/vad_analyzer.py | 4 +- src/pipecat/pipeline/base_task.py | 17 ++----- src/pipecat/pipeline/runner.py | 10 ++-- src/pipecat/pipeline/task.py | 18 +------ src/pipecat/pipeline/task_observer.py | 16 +------ src/pipecat/processors/frame_processor.py | 40 ++-------------- src/pipecat/transports/base_transport.py | 54 +++------------------ src/pipecat/utils/base_object.py | 58 +++++++++++++++++++++++ 9 files changed, 103 insertions(+), 160 deletions(-) create mode 100644 src/pipecat/utils/base_object.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d5e4f862..4ff473ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,26 +9,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added support for a unified format for specifying function calling across all LLM services. - ```python - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - ``` +- Added new base class `BaseObject` which is now the base class of + `FrameProcessor`, `PipelineRunner`, `PipelineTask` and `BaseTransport`. The + new `BaseObject` adds supports for event handlers. + +- Added support for a unified format for specifying function calling across all + LLM services. + +```python + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) +``` - Added `speech_threshold` parameter to `GladiaSTTService`. diff --git a/src/pipecat/audio/vad/vad_analyzer.py b/src/pipecat/audio/vad/vad_analyzer.py index b5d9b0ba2..3ca21a208 100644 --- a/src/pipecat/audio/vad/vad_analyzer.py +++ b/src/pipecat/audio/vad/vad_analyzer.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from abc import abstractmethod +from abc import ABC, abstractmethod from enum import Enum from typing import Optional @@ -33,7 +33,7 @@ class VADParams(BaseModel): min_volume: float = VAD_MIN_VOLUME -class VADAnalyzer: +class VADAnalyzer(ABC): def __init__(self, *, sample_rate: Optional[int] = None, params: VADParams): self._init_sample_rate = sample_rate self._sample_rate = 0 diff --git a/src/pipecat/pipeline/base_task.py b/src/pipecat/pipeline/base_task.py index b8622bac2..278709b7e 100644 --- a/src/pipecat/pipeline/base_task.py +++ b/src/pipecat/pipeline/base_task.py @@ -5,25 +5,14 @@ # import asyncio -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import AsyncIterable, Iterable from pipecat.frames.frames import Frame +from pipecat.utils.base_object import BaseObject -class BaseTask(ABC): - @property - @abstractmethod - def id(self) -> int: - """Returns the unique indetifier for this task.""" - pass - - @property - @abstractmethod - def name(self) -> str: - """Returns the name of this task.""" - pass - +class BaseTask(BaseObject): @abstractmethod def set_event_loop(self, loop: asyncio.AbstractEventLoop): """Sets the event loop that this task will run on.""" diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index acdbab562..3209fa92a 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -12,10 +12,10 @@ from typing import Optional from loguru import logger from pipecat.pipeline.task import PipelineTask -from pipecat.utils.utils import obj_count, obj_id +from pipecat.utils.base_object import BaseObject -class PipelineRunner: +class PipelineRunner(BaseObject): def __init__( self, *, @@ -24,8 +24,7 @@ class PipelineRunner: force_gc: bool = False, loop: Optional[asyncio.AbstractEventLoop] = None, ): - self.id: int = obj_id() - self.name: str = name or f"{self.__class__.__name__}#{obj_count(self)}" + super().__init__(name=name) self._tasks = {} self._sig_task = None @@ -74,6 +73,3 @@ class PipelineRunner: collected = gc.collect() logger.debug(f"Garbage collector: collected {collected} objects.") logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}") - - def __str__(self): - return self.name diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 9e6b313a6..1c9d2dff9 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -32,7 +32,6 @@ from pipecat.pipeline.base_task import BaseTask from pipecat.pipeline.task_observer import TaskObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.utils.asyncio import BaseTaskManager, TaskManager -from pipecat.utils.utils import obj_count, obj_id HEARTBEAT_SECONDS = 1.0 HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 5 @@ -138,9 +137,7 @@ class PipelineTask(BaseTask): task_manager: Optional[BaseTaskManager] = None, check_dangling_tasks: bool = True, ): - self._id: int = obj_id() - self._name: str = f"{self.__class__.__name__}#{obj_count(self)}" - + super().__init__() self._pipeline = pipeline self._clock = clock self._params = params @@ -180,16 +177,6 @@ class PipelineTask(BaseTask): self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) - @property - def id(self) -> int: - """Returns the unique indetifier for this task.""" - return self._id - - @property - def name(self) -> str: - """Returns the name of this task.""" - return self._name - @property def params(self) -> PipelineParams: """Returns the pipeline parameters of this task.""" @@ -434,6 +421,3 @@ class PipelineTask(BaseTask): tasks = [t.get_name() for t in self._task_manager.current_tasks()] if tasks: logger.warning(f"Dangling tasks detected: {tasks}") - - def __str__(self): - return self.name diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index 122038386..dd805032c 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -13,7 +13,6 @@ from pipecat.frames.frames import Frame from pipecat.observers.base_observer import BaseObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.utils.asyncio import BaseTaskManager -from pipecat.utils.utils import obj_count, obj_id @dataclass @@ -56,20 +55,10 @@ class TaskObserver(BaseObserver): """ def __init__(self, *, observers: List[BaseObserver] = [], task_manager: BaseTaskManager): - self._id: int = obj_id() - self._name: str = f"{self.__class__.__name__}#{obj_count(self)}" self._observers = observers self._task_manager = task_manager self._proxies: List[Proxy] = [] - @property - def id(self) -> int: - return self._id - - @property - def name(self) -> str: - return self._name - async def start(self): """Starts all proxy observer tasks.""" self._proxies = self._create_proxies(self._observers) @@ -100,7 +89,7 @@ class TaskObserver(BaseObserver): queue = asyncio.Queue() task = self._task_manager.create_task( self._proxy_task_handler(queue, observer), - f"{self}::{observer.__class__.__name__}::_proxy_task_handler", + f"TaskObserver::{observer.__class__.__name__}::_proxy_task_handler", ) proxy = Proxy(queue=queue, task=task, observer=observer) proxies.append(proxy) @@ -112,6 +101,3 @@ class TaskObserver(BaseObserver): await observer.on_push_frame( data.src, data.dst, data.frame, data.direction, data.timestamp ) - - def __str__(self): - return self.name diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 335585ec3..6a1669ff1 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -5,7 +5,6 @@ # import asyncio -import inspect from enum import Enum from typing import Awaitable, Callable, Coroutine, Optional @@ -24,7 +23,7 @@ from pipecat.frames.frames import ( from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics from pipecat.utils.asyncio import BaseTaskManager -from pipecat.utils.utils import obj_count, obj_id +from pipecat.utils.base_object import BaseObject class FrameDirection(Enum): @@ -32,7 +31,7 @@ class FrameDirection(Enum): UPSTREAM = 2 -class FrameProcessor: +class FrameProcessor(BaseObject): def __init__( self, *, @@ -40,14 +39,11 @@ class FrameProcessor: metrics: Optional[FrameProcessorMetrics] = None, **kwargs, ): - self._id: int = obj_id() - self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" + super().__init__(name=name) self._parent: Optional["FrameProcessor"] = None self._prev: Optional["FrameProcessor"] = None self._next: Optional["FrameProcessor"] = None - self._event_handlers: dict = {} - # Clock self._clock: Optional[BaseClock] = None @@ -254,23 +250,6 @@ class FrameProcessor: else: await self.__push_queue.put((frame, direction)) - def event_handler(self, event_name: str): - def decorator(handler): - self.add_event_handler(event_name, handler) - return handler - - return decorator - - def add_event_handler(self, event_name: str, handler): - if event_name not in self._event_handlers: - raise Exception(f"Event handler {event_name} not registered") - self._event_handlers[event_name].append(handler) - - def _register_event_handler(self, event_name: str): - if event_name in self._event_handlers: - raise Exception(f"Event handler {event_name} already registered") - self._event_handlers[event_name] = [] - async def __start(self, frame: StartFrame): self.__create_input_task() self.__create_push_task() @@ -385,16 +364,3 @@ class FrameProcessor: (frame, direction) = await self.__push_queue.get() await self.__internal_push_frame(frame, direction) self.__push_queue.task_done() - - async def _call_event_handler(self, event_name: str, *args, **kwargs): - try: - for handler in self._event_handlers[event_name]: - if inspect.iscoroutinefunction(handler): - await handler(self, *args, **kwargs) - else: - handler(self, *args, **kwargs) - except Exception as e: - logger.exception(f"Exception in event handler {event_name}: {e}") - - def __str__(self): - return self.name diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index c28579784..06f6cb920 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -4,18 +4,16 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import inspect -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Optional -from loguru import logger from pydantic import BaseModel, ConfigDict from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer from pipecat.audio.vad.vad_analyzer import VADAnalyzer from pipecat.processors.frame_processor import FrameProcessor -from pipecat.utils.utils import obj_count, obj_id +from pipecat.utils.base_object import BaseObject class TransportParams(BaseModel): @@ -43,7 +41,7 @@ class TransportParams(BaseModel): vad_analyzer: Optional[VADAnalyzer] = None -class BaseTransport(ABC): +class BaseTransport(BaseObject): def __init__( self, *, @@ -51,54 +49,14 @@ class BaseTransport(ABC): input_name: Optional[str] = None, output_name: Optional[str] = None, ): - self._id: int = obj_id() - self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" + super().__init__(name=name) self._input_name = input_name self._output_name = output_name - self._event_handlers: dict = {} - - @property - def id(self) -> int: - return self._id - - @property - def name(self) -> str: - return self._name @abstractmethod def input(self) -> FrameProcessor: - raise NotImplementedError + pass @abstractmethod def output(self) -> FrameProcessor: - raise NotImplementedError - - def event_handler(self, event_name: str): - def decorator(handler): - self.add_event_handler(event_name, handler) - return handler - - return decorator - - def add_event_handler(self, event_name: str, handler): - if event_name not in self._event_handlers: - raise Exception(f"Event handler {event_name} not registered") - self._event_handlers[event_name].append(handler) - - def _register_event_handler(self, event_name: str): - if event_name in self._event_handlers: - raise Exception(f"Event handler {event_name} already registered") - self._event_handlers[event_name] = [] - - async def _call_event_handler(self, event_name: str, *args, **kwargs): - try: - for handler in self._event_handlers[event_name]: - if inspect.iscoroutinefunction(handler): - await handler(self, *args, **kwargs) - else: - handler(self, *args, **kwargs) - except Exception as e: - logger.exception(f"Exception in event handler {event_name}: {e}") - - def __str__(self): - return self.name + pass diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py new file mode 100644 index 000000000..e51eac35d --- /dev/null +++ b/src/pipecat/utils/base_object.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import inspect +from abc import ABC +from typing import Optional + +from loguru import logger + +from pipecat.utils.utils import obj_count, obj_id + + +class BaseObject(ABC): + def __init__(self, *, name: Optional[str] = None): + self._id: int = obj_id() + self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" + self._event_handlers: dict = {} + + @property + def id(self) -> int: + return self._id + + @property + def name(self) -> str: + return self._name + + def event_handler(self, event_name: str): + def decorator(handler): + self.add_event_handler(event_name, handler) + return handler + + return decorator + + def add_event_handler(self, event_name: str, handler): + if event_name not in self._event_handlers: + raise Exception(f"Event handler {event_name} not registered") + self._event_handlers[event_name].append(handler) + + def _register_event_handler(self, event_name: str): + if event_name in self._event_handlers: + raise Exception(f"Event handler {event_name} already registered") + self._event_handlers[event_name] = [] + + async def _call_event_handler(self, event_name: str, *args, **kwargs): + try: + for handler in self._event_handlers[event_name]: + if inspect.iscoroutinefunction(handler): + await handler(self, *args, **kwargs) + else: + handler(self, *args, **kwargs) + except Exception as e: + logger.exception(f"Exception in event handler {event_name}: {e}") + + def __str__(self): + return self.name From 532423eb4cdde150daca8eb52eba39d69d6c866c Mon Sep 17 00:00:00 2001 From: Dominic Stewart <45786774+DominicStewart@users.noreply.github.com> Date: Wed, 5 Mar 2025 13:40:36 -0800 Subject: [PATCH 259/427] Updated example to switch pipelines per the original request (#1320) --- examples/phone-chatbot/bot_daily_gemini.py | 296 +++++++++++++++------ 1 file changed, 213 insertions(+), 83 deletions(-) diff --git a/examples/phone-chatbot/bot_daily_gemini.py b/examples/phone-chatbot/bot_daily_gemini.py index 8ada81a9c..4972cd3ee 100644 --- a/examples/phone-chatbot/bot_daily_gemini.py +++ b/examples/phone-chatbot/bot_daily_gemini.py @@ -14,8 +14,10 @@ from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( + EndFrame, EndTaskFrame, InputAudioRawFrame, + StopTaskFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, @@ -25,10 +27,15 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.ai_services import LLMService +from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.google import GoogleLLMService from pipecat.services.google.google import GoogleLLMContext -from pipecat.transports.services.daily import DailyDialinSettings, DailyParams, DailyTransport +from pipecat.transports.services.daily import ( + DailyDialinSettings, + DailyParams, + DailyTransport, +) load_dotenv(override=True) @@ -39,6 +46,8 @@ logger.add(sys.stderr, level="DEBUG") daily_api_key = os.getenv("DAILY_API_KEY", "") daily_api_url = os.getenv("DAILY_API_URL", "https://api.daily.co/v1") +system_message = None + class UserAudioCollector(FrameProcessor): """This FrameProcessor collects audio frames in a buffer, then adds them to the @@ -112,7 +121,13 @@ class FunctionHandlers: self.context_switcher = context_switcher async def voicemail_response( - self, function_name, tool_call_id, args, llm, context, result_callback + self, + function_name, + tool_call_id, + args, + llm: LLMService, + context, + result_callback, ): """Function the bot can call to leave a voicemail message.""" message = """You are Chatbot leaving a voicemail message. Say EXACTLY this message and nothing else: @@ -122,62 +137,48 @@ class FunctionHandlers: After saying this message, call the terminate_call function.""" await self.context_switcher.switch_context(system_instruction=message) - await result_callback("Leaving a voicemail message") async def human_conversation( - self, function_name, tool_call_id, args, llm, context, result_callback + self, + function_name, + tool_call_id, + args, + llm: LLMService, + context, + result_callback, ): """Function the bot can when it detects it's talking to a human.""" - message = """You are Chatbot talking to a human. Be friendly and helpful. - - Start with: "Hello! I'm a friendly chatbot. How can I help you today?" - - Keep your responses brief and to the point. Listen to what the person says. - - When the person indicates they're done with the conversation by saying something like: - - "Goodbye" - - "That's all" - - "I'm done" - - "Thank you, that's all I needed" - - THEN say: "Thank you for chatting. Goodbye!" and call the terminate_call function.""" - - await self.context_switcher.switch_context(system_instruction=message) - - await result_callback("Talking to the customer") + await llm.push_frame(StopTaskFrame(), FrameDirection.UPSTREAM) async def terminate_call( - function_name, tool_call_id, args, llm: LLMService, context, result_callback + function_name, + tool_call_id, + args, + llm: LLMService, + context, + result_callback, + call_state=None, ): """Function the bot can call to terminate the call upon completion of the call.""" - - await llm.queue_frame(EndTaskFrame(), FrameDirection.UPSTREAM) + if call_state: + call_state.bot_terminated_call = True + await llm.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) async def main( room_url: str, token: str, - callId: str, - callDomain: str, + callId: Optional[str], + callDomain: Optional[str], detect_voicemail: bool, dialout_number: Optional[str], ): - # dialin_settings are only needed if Daily's SIP URI is used - # If you are handling this via Twilio, Telnyx, set this to None - # and handle call-forwarding when on_dialin_ready fires. - - # We don't want to specify dial-in settings if we're not dialing in dialin_settings = None if callId and callDomain: dialin_settings = DailyDialinSettings(call_id=callId, call_domain=callDomain) - - transport = DailyTransport( - room_url, - token, - "Chatbot", - DailyParams( + transport_params = DailyParams( api_url=daily_api_url, api_key=daily_api_key, dialin_settings=dialin_settings, @@ -187,8 +188,30 @@ async def main( vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_audio_passthrough=True, - # transcription_enabled=True, - ), + ) + else: + transport_params = DailyParams( + api_url=daily_api_url, + api_key=daily_api_key, + audio_in_enabled=True, + audio_out_enabled=True, + camera_out_enabled=False, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ) + + class CallState: + participant_left_early = False + bot_terminated_call = False + + call_state = CallState() + + transport = DailyTransport( + room_url, + token, + "Chatbot", + transport_params, ) tts = ElevenLabsTTSService( @@ -196,6 +219,10 @@ async def main( voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + ### VOICEMAIL PIPELINE + tools = [ { "function_declarations": [ @@ -217,55 +244,67 @@ async def main( system_instruction = """You are Chatbot trying to determine if this is a voicemail system or a human. -If you hear any of these phrases (or very similar ones): -- "Please leave a message after the beep" -- "No one is available to take your call" -- "Record your message after the tone" -- "You have reached voicemail for..." -- "You have reached [phone number]" -- "[phone number] is unavailable" -- "The person you are trying to reach..." -- "The number you have dialed..." -- "Your call has been forwarded to an automated voice messaging system" + If you hear any of these phrases (or very similar ones): + - "Please leave a message after the beep" + - "No one is available to take your call" + - "Record your message after the tone" + - "You have reached voicemail for..." + - "You have reached [phone number]" + - "[phone number] is unavailable" + - "The person you are trying to reach..." + - "The number you have dialed..." + - "Your call has been forwarded to an automated voice messaging system" -Then call the function switch_to_voicemail_response. + Then call the function switch_to_voicemail_response. -If it sounds like a human (saying hello, asking questions, etc.), call the function switch_to_human_conversation. + If it sounds like a human (saying hello, asking questions, etc.), call the function switch_to_human_conversation. -DO NOT say anything until you've determined if this is a voicemail or human.""" + DO NOT say anything until you've determined if this is a voicemail or human.""" - llm = GoogleLLMService( + voicemail_detection_llm = GoogleLLMService( model="models/gemini-2.0-flash-lite", api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, tools=tools, ) - context = GoogleLLMContext() - context_aggregator = llm.create_context_aggregator(context) - audio_collector = UserAudioCollector(context, context_aggregator.user()) - - context_switcher = ContextSwitcher(llm, context_aggregator.user()) + voicemail_detection_context = GoogleLLMContext() + voicemail_detection_context_aggregator = voicemail_detection_llm.create_context_aggregator( + voicemail_detection_context + ) + context_switcher = ContextSwitcher( + voicemail_detection_llm, voicemail_detection_context_aggregator.user() + ) handlers = FunctionHandlers(context_switcher) - llm.register_function("switch_to_voicemail_response", handlers.voicemail_response) - llm.register_function("switch_to_human_conversation", handlers.human_conversation) - llm.register_function("terminate_call", terminate_call) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - audio_collector, # Collect audio frames - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses - ] + voicemail_detection_llm.register_function( + "switch_to_voicemail_response", handlers.voicemail_response + ) + voicemail_detection_llm.register_function( + "switch_to_human_conversation", handlers.human_conversation + ) + voicemail_detection_llm.register_function( + "terminate_call", + lambda *args, **kwargs: terminate_call(*args, **kwargs, call_state=call_state), ) - task = PipelineTask( - pipeline, + voicemail_detection_audio_collector = UserAudioCollector( + voicemail_detection_context, voicemail_detection_context_aggregator.user() + ) + + voicemail_detection_pipeline = Pipeline( + [ + transport.input(), # Transport user input + voicemail_detection_audio_collector, # Collect audio frames + voicemail_detection_context_aggregator.user(), # User responses + voicemail_detection_llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + voicemail_detection_context_aggregator.assistant(), # Assistant spoken responses + ] + ) + voicemail_detection_pipeline_task = PipelineTask( + voicemail_detection_pipeline, params=PipelineParams(allow_interruptions=True), ) @@ -300,25 +339,116 @@ DO NOT say anything until you've determined if this is a voicemail or human.""" # machine to say something like 'Leave a message after the beep', or for the user to say 'Hello?'. @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): + logger.debug("Detect voicemail; capturing participant transcription") await transport.capture_participant_transcription(participant["id"]) else: - logger.debug("no dialout number; assuming dialin") + logger.debug("+++++ No dialout number; assuming dialin") # Different handlers for dialin @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): + # This event is not firing for some reason await transport.capture_participant_transcription(participant["id"]) - # For the dialin case, we want the bot to answer the phone and greet the user. We - # can prompt the bot to speak by putting the context into the pipeline. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - await task.cancel() + dialin_instructions = """Always call the function switch_to_human_conversation""" + messages = [ + { + "role": "system", + "content": dialin_instructions, + } + ] + voicemail_detection_context_aggregator.user().set_messages(messages) + await voicemail_detection_pipeline_task.queue_frames( + [voicemail_detection_context_aggregator.user().get_context_frame()] + ) runner = PipelineRunner() - await runner.run(task) + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + call_state.participant_left_early = True + await voicemail_detection_pipeline_task.queue_frame(EndFrame()) + + print("!!! starting voicemail detection pipeline") + await runner.run(voicemail_detection_pipeline_task) + print("!!! Done with voicemail detection pipeline") + + if call_state.participant_left_early or call_state.bot_terminated_call: + if call_state.participant_left_early: + print("!!! Participant left early; terminating call") + elif call_state.bot_terminated_call: + print("!!! Bot terminated call; not proceeding to human conversation") + return + + ### HUMAN CONVERSATION PIPELINE + + human_conversation_system_instruction = """You are Chatbot talking to a human. Be friendly and helpful. + + Start with: "Hello! I'm a friendly chatbot. How can I help you today?" + + Keep your responses brief and to the point. Listen to what the person says. + + When the person indicates they're done with the conversation by saying something like: + - "Goodbye" + - "That's all" + - "I'm done" + - "Thank you, that's all I needed" + + THEN say: "Thank you for chatting. Goodbye!" and call the terminate_call function.""" + + human_conversation_llm = GoogleLLMService( + model="models/gemini-2.0-flash-001", + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=human_conversation_system_instruction, + tools=tools, + ) + human_conversation_context = GoogleLLMContext() + + human_conversation_context_aggregator = human_conversation_llm.create_context_aggregator( + human_conversation_context + ) + + human_conversation_llm.register_function( + "terminate_call", + lambda *args, **kwargs: terminate_call(*args, **kwargs, call_state=call_state), + ) + + human_conversation_pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + human_conversation_context_aggregator.user(), # User responses + human_conversation_llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + human_conversation_context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + human_conversation_pipeline_task = PipelineTask( + human_conversation_pipeline, + params=PipelineParams(allow_interruptions=True), + ) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await voicemail_detection_pipeline_task.queue_frame(EndFrame()) + await human_conversation_pipeline_task.queue_frame(EndFrame()) + + print("!!! starting human conversation pipeline") + human_conversation_context_aggregator.user().set_messages( + [ + { + "role": "system", + "content": human_conversation_system_instruction, + } + ] + ) + await human_conversation_pipeline_task.queue_frames( + [human_conversation_context_aggregator.user().get_context_frame()] + ) + await runner.run(human_conversation_pipeline_task) + + print("!!! Done with human conversation pipeline") if __name__ == "__main__": From 26000b616d7ca8f9ad2f97ff4ffaabde52505a4c Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 6 Mar 2025 10:15:04 -0300 Subject: [PATCH 260/427] Fixing the base_whisper services to implement set_language. --- src/pipecat/services/base_whisper.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/pipecat/services/base_whisper.py b/src/pipecat/services/base_whisper.py index 4de62f08c..146fd4f39 100644 --- a/src/pipecat/services/base_whisper.py +++ b/src/pipecat/services/base_whisper.py @@ -138,6 +138,15 @@ class BaseWhisperSTTService(SegmentedSTTService): def language_to_service_language(self, language: Language) -> Optional[str]: return language_to_whisper_language(language) + async def set_language(self, language: Language): + """Set the language for transcription. + + Args: + language: The Language enum value to use for transcription. + """ + logger.info(f"Switching STT language to: [{language}]") + self._language = language + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: try: await self.start_processing_metrics() From 2d114b15f9ff2787cb6eb4b0bea75ee762413f34 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 6 Mar 2025 10:34:25 -0300 Subject: [PATCH 261/427] Adding missing flush_audio method to AzureTTSService. --- src/pipecat/services/azure.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index 83e5bbe6d..60a3b6a8b 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -577,6 +577,10 @@ class AzureTTSService(AzureBaseTTSService): logger.error(f"Speech synthesis canceled: {evt.result.cancellation_details.reason}") self._audio_queue.put_nowait(None) + async def flush_audio(self): + logger.trace(f"{self}: flushing audio") + # TODO: check what we need to implement here ? + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"{self}: Generating TTS [{text}]") From 2df77430aa04e7e84d94b80f868ee73299818c2d Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 6 Mar 2025 10:35:26 -0300 Subject: [PATCH 262/427] Refactoring the 14 series examples to use the unified format for function calling. --- examples/foundational/14-function-calling.py | 43 +++++------- .../14a-function-calling-anthropic.py | 29 ++++---- .../14b-function-calling-anthropic-video.py | 50 ++++++------- .../14c-function-calling-together.py | 42 +++++------ .../14d-function-calling-video.py | 68 ++++++++---------- .../14e-function-calling-gemini.py | 70 ++++++++----------- .../foundational/14f-function-calling-groq.py | 42 +++++------ .../foundational/14g-function-calling-grok.py | 42 +++++------ .../14h-function-calling-azure.py | 42 +++++------ .../14i-function-calling-fireworks.py | 42 +++++------ .../foundational/14j-function-calling-nim.py | 42 +++++------ .../14k-function-calling-cerebras.py | 42 +++++------ .../14l-function-calling-deepseek.py | 42 +++++------ .../14m-function-calling-openrouter.py | 42 +++++------ ...o-function-calling-gemini-openai-format.py | 42 +++++------ 15 files changed, 295 insertions(+), 385 deletions(-) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 7c77378fc..707644f88 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -65,30 +66,24 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + messages = [ { "role": "system", diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 4723de004..ba93a3be4 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -11,6 +11,9 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -59,22 +62,18 @@ async def main(): ) llm.register_function("get_weather", get_weather) - tools = [ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - } - }, - "required": ["location"], + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } - ] + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) # todo: test with very short initial user message diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 69f1e5776..695278214 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -11,6 +11,9 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -72,36 +75,29 @@ async def main(): llm.register_function("get_weather", get_weather) llm.register_function("get_image", get_image) - tools = [ - { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - } - }, - "required": ["location"], + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, }, - { - "name": "get_image", - "description": "Get an image from the video stream.", - "input_schema": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question that the user is asking about the image.", - } - }, - "required": ["question"], - }, + required=["location"], + ) + get_image_function = FunctionSchema( + name="get_image", + description="Get an image from the video stream.", + properties= { + "question": { + "type": "string", + "description": "The question that the user is asking about the image.", + } }, - ] + required=["question"], + ) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) # todo: test with very short initial user message diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 9c981f7bc..4c4bec4da 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -69,30 +70,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index aa71d05ff..916606c50 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -66,47 +67,34 @@ async def main(): llm.register_function("get_weather", get_weather) llm.register_function("get_image", get_image) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ), - ChatCompletionToolParam( - type="function", - function={ - "name": "get_image", - "description": "Get an image from the video stream.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to ask the AI to generate an image of", - }, - }, - "required": ["question"], - }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", }, - ), - ] + }, + required=["location"], + ) + get_image_function = FunctionSchema( + name="get_image", + description="Get an image from the video stream.", + properties={ + "question": { + "type": "string", + "description": "The question that the user is asking about the image.", + } + }, + required=["question"], + ) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 477d6c5ca..30b74ba2c 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -11,6 +11,9 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -73,45 +76,34 @@ async def main(): llm.register_function("get_weather", get_weather, start_fetch_weather) llm.register_function("get_image", get_image) - tools = [ - { - "function_declarations": [ - { - "name": "get_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - { - "name": "get_image", - "description": "Get and image from the camera or video stream.", - "parameters": { - "type": "object", - "properties": { - "question": { - "type": "string", - "description": "The question to to use when running inference on the acquired image.", - }, - }, - "required": ["question"], - }, - }, - ] - } - ] + weather_function = FunctionSchema( + name="get_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + get_image_function = FunctionSchema( + name="get_image", + description="Get an image from the video stream.", + properties={ + "question": { + "type": "string", + "description": "The question that the user is asking about the image.", + } + }, + required=["question"], + ) + tools = ToolsSchema(standard_tools=[weather_function, get_image_function]) system_prompt = """\ You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index f2768a788..46caed370 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -68,30 +69,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 7318f455d..c2bbce221 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -66,30 +67,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index cfd56b59d..edd33b1f7 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -70,30 +71,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 53346686f..54ef75b54 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -69,30 +70,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 712ce0742..39c29f004 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -69,30 +70,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Returns the current weather at a location, if one is specified, and defaults to the user's location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to find the weather of, or if not provided, it's the default location.", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "Whether to use SI or USCS units (celsius or fahrenheit).", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 4a65adbef..b0b3ffb1e 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -66,30 +67,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather for a specific location. You MUST use this function whenever asked about weather.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Use fahrenheit for US locations, celsius for others.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index f67d583ee..e96db3254 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -66,30 +67,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather for a specific location. You MUST use this function whenever asked about weather.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Use fahrenheit for US locations, celsius for others.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 622337b07..745d130e7 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -70,30 +71,23 @@ async def main(): # sent to the same callback with an additional function_name parameter. llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "system", diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py index 4b04f9285..1434c9a77 100644 --- a/examples/foundational/14o-function-calling-gemini-openai-format.py +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -11,7 +11,8 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from openai.types.chat import ChatCompletionToolParam +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -68,30 +69,23 @@ async def main(): "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather ) - tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - ) - ] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) messages = [ { "role": "user", From 4b167a3c3defa5b59192e0682ff06fa79ffe9463 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 6 Mar 2025 10:38:45 -0300 Subject: [PATCH 263/427] Fixing the ruff format. --- examples/foundational/14-function-calling.py | 4 ++-- examples/foundational/14a-function-calling-anthropic.py | 3 +-- .../foundational/14b-function-calling-anthropic-video.py | 5 ++--- examples/foundational/14c-function-calling-together.py | 4 ++-- examples/foundational/14d-function-calling-video.py | 4 ++-- examples/foundational/14e-function-calling-gemini.py | 3 +-- examples/foundational/14f-function-calling-groq.py | 4 ++-- examples/foundational/14g-function-calling-grok.py | 4 ++-- examples/foundational/14h-function-calling-azure.py | 4 ++-- examples/foundational/14i-function-calling-fireworks.py | 4 ++-- examples/foundational/14j-function-calling-nim.py | 4 ++-- examples/foundational/14k-function-calling-cerebras.py | 4 ++-- examples/foundational/14l-function-calling-deepseek.py | 4 ++-- examples/foundational/14m-function-calling-openrouter.py | 4 ++-- .../14o-function-calling-gemini-openai-format.py | 4 ++-- 15 files changed, 28 insertions(+), 31 deletions(-) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 707644f88..043f278e2 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index ba93a3be4..14e788f99 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -11,11 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger +from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from runner import configure - from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 695278214..b31e59442 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -11,11 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger +from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from runner import configure - from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -89,7 +88,7 @@ async def main(): get_image_function = FunctionSchema( name="get_image", description="Get an image from the video stream.", - properties= { + properties={ "question": { "type": "string", "description": "The question that the user is asking about the image.", diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 4c4bec4da..7f47eb28e 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 916606c50..15344b7ac 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 30b74ba2c..146ed77d7 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -11,11 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger +from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from runner import configure - from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 46caed370..5bbddcc4d 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index c2bbce221..423919772 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index edd33b1f7..669055a55 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 54ef75b54..88168383f 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 39c29f004..5d4b123d0 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index b0b3ffb1e..b9f1c6b18 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index e96db3254..6add663d5 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index 745d130e7..e816f6322 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py index 1434c9a77..a1bb53b6b 100644 --- a/examples/foundational/14o-function-calling-gemini-openai-format.py +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -11,10 +11,10 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline From 21443b9a08eecdb796a9629c9105d75cb2ed8888 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 6 Mar 2025 11:59:08 -0300 Subject: [PATCH 264/427] Refactored gemini multimodal example to use the unified format for function calling. --- ...gemini-multimodal-live-function-calling.py | 49 +++++++++---------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 14f2d62c7..3aaad44ce 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -14,6 +14,8 @@ from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -41,32 +43,6 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "function_declarations": [ - { - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - ] - } -] - system_instruction = """ You are a helpful assistant who can answer questions and use tools. @@ -95,6 +71,27 @@ async def main(): ), ) + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + search_tool = {"google_search": {}} + tools = ToolsSchema( + standard_tools=[weather_function], custom_tools={AdapterType.GEMINI: [search_tool]} + ) + llm = GeminiMultimodalLiveLLMService( api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=system_instruction, From 55b0797fd5cabaa791b2c144d0229353ce23f9d4 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Thu, 6 Mar 2025 12:00:22 -0300 Subject: [PATCH 265/427] Removing the extra examples inside the unified-format-function-calling folder --- .../base_function_calling.py | 134 ------------------ .../multimodal_base_function_calling.py | 126 ---------------- .../unified-format-function-calling/runner.py | 64 --------- .../standard-function-calling-anthropic.py | 29 ---- .../standard-function-calling-azure.py | 31 ---- .../standard-function-calling-cerebras.py | 27 ---- .../standard-function-calling-deepseek.py | 27 ---- .../standard-function-calling-fireworks.py | 29 ---- ...dard-function-calling-gemini-multimodal.py | 38 ----- .../standard-function-calling-gemini.py | 27 ---- .../standard-function-calling-grok.py | 27 ---- .../standard-function-calling-groq.py | 27 ---- .../standard-function-calling-nim.py | 29 ---- ...andard-function-calling-openai-realtime.py | 43 ------ .../standard-function-calling-openai.py | 27 ---- .../standard-function-calling-openrouter.py | 29 ---- .../standard-function-calling-together.py | 30 ---- src/pipecat/services/azure.py | 1 - 18 files changed, 745 deletions(-) delete mode 100644 examples/unified-format-function-calling/base_function_calling.py delete mode 100644 examples/unified-format-function-calling/multimodal_base_function_calling.py delete mode 100644 examples/unified-format-function-calling/runner.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-anthropic.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-azure.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-cerebras.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-deepseek.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-fireworks.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-gemini.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-grok.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-groq.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-nim.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-openai-realtime.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-openai.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-openrouter.py delete mode 100644 examples/unified-format-function-calling/standard-function-calling-together.py diff --git a/examples/unified-format-function-calling/base_function_calling.py b/examples/unified-format-function-calling/base_function_calling.py deleted file mode 100644 index 798d94465..000000000 --- a/examples/unified-format-function-calling/base_function_calling.py +++ /dev/null @@ -1,134 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import os -import sys - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.ai_services import LLMService -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - -load_dotenv(override=True) - - -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - -async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): - await result_callback({"conditions": "nice", "temperature": "75"}) - - -class WeatherBot: - """Generic base class for setting up and running an LLM-powered bot.""" - - def __init__(self, llm: LLMService): - """Initialize the base handler with a specific LLM.""" - self.llm = llm - - async def run(self): - """Set up and start the processing pipeline.""" - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady - ) - - # Register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - self.llm.register_function( - None, fetch_weather_from_api, start_callback=start_fetch_weather - ) - - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location"], - ) - tools = ToolsSchema(standard_tools=[weather_function]) - - messages = [ - { - "role": "system", - "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation.", - }, - {"role": "user", "content": " Start the conversation by introducing yourself."}, - ] - - context = OpenAILLMContext(messages, tools) - context_aggregator = self.llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - self.llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - runner = PipelineRunner() - await runner.run(task) diff --git a/examples/unified-format-function-calling/multimodal_base_function_calling.py b/examples/unified-format-function-calling/multimodal_base_function_calling.py deleted file mode 100644 index 8f4a51b96..000000000 --- a/examples/unified-format-function-calling/multimodal_base_function_calling.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import sys -from typing import List - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.adapters.schemas.function_schema import FunctionSchema -from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.ai_services import LLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - -load_dotenv(override=True) - - -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - -async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): - await result_callback({"conditions": "nice", "temperature": "75"}) - - -class MultimodalWeatherBot: - """Generic base class for setting up and running an LLM-powered bot.""" - - def __init__(self, llm: LLMService): - """Initialize the base handler with a specific LLM.""" - self.llm = llm - - @staticmethod - def tools() -> ToolsSchema: - weather_function = FunctionSchema( - name="get_current_weather", - description="Get the current weather", - properties={ - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the user's location.", - }, - }, - required=["location"], - ) - return ToolsSchema(standard_tools=[weather_function]) - - async def run(self): - """Set up and start the processing pipeline.""" - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - token, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) - - # Register a function_name of None to get all functions - # sent to the same callback with an additional function_name parameter. - self.llm.register_function( - None, fetch_weather_from_api, start_callback=start_fetch_weather - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful assistant who can report the weather in any location in the universe. Respond concisely. Your response will be turned into speech so use only simple words and punctuation.", - }, - {"role": "user", "content": " Start the conversation by introducing yourself."}, - ] - - context = OpenAILLMContext(messages, MultimodalWeatherBot.tools()) - context_aggregator = self.llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - self.llm, - transport.output(), - context_aggregator.assistant(), - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - runner = PipelineRunner() - await runner.run(task) diff --git a/examples/unified-format-function-calling/runner.py b/examples/unified-format-function-calling/runner.py deleted file mode 100644 index 04157d549..000000000 --- a/examples/unified-format-function-calling/runner.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import argparse -import os -from typing import Optional - -import aiohttp - -from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper - - -async def configure(aiohttp_session: aiohttp.ClientSession): - (url, token, _) = await configure_with_args(aiohttp_session) - return (url, token) - - -async def configure_with_args( - aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None -): - if not parser: - parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=False, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=False, - help="Daily API Key (needed to create an owner token for the room)", - ) - - args, unknown = parser.parse_known_args() - - url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL") - key = args.apikey or os.getenv("DAILY_API_KEY") - - if not url: - raise Exception( - "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL." - ) - - if not key: - raise Exception( - "No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers." - ) - - daily_rest_helper = DailyRESTHelper( - daily_api_key=key, - daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), - aiohttp_session=aiohttp_session, - ) - - # Create a meeting token for the given room with an expiration 1 hour in - # the future. - expiry_time: float = 60 * 60 - - token = await daily_rest_helper.get_token(url, expiry_time) - - return (url, token, args) diff --git a/examples/unified-format-function-calling/standard-function-calling-anthropic.py b/examples/unified-format-function-calling/standard-function-calling-anthropic.py deleted file mode 100644 index 7ae39b99a..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-anthropic.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.anthropic import AnthropicLLMService - -load_dotenv(override=True) - - -class AnthropicWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620" - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(AnthropicWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-azure.py b/examples/unified-format-function-calling/standard-function-calling-azure.py deleted file mode 100644 index c1b24ca2a..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-azure.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.azure import AzureLLMService - -load_dotenv(override=True) - - -class AzureWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(AzureWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-cerebras.py b/examples/unified-format-function-calling/standard-function-calling-cerebras.py deleted file mode 100644 index 6888268aa..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-cerebras.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.cerebras import CerebrasLLMService - -load_dotenv(override=True) - - -class CerebrasWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(CerebrasWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-deepseek.py b/examples/unified-format-function-calling/standard-function-calling-deepseek.py deleted file mode 100644 index 7c8cd6ebb..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-deepseek.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.deepseek import DeepSeekLLMService - -load_dotenv(override=True) - - -class DeepSeekWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(DeepSeekWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-fireworks.py b/examples/unified-format-function-calling/standard-function-calling-fireworks.py deleted file mode 100644 index 1128c1ada..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-fireworks.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.fireworks import FireworksLLMService - -load_dotenv(override=True) - - -class FireworksWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = FireworksLLMService( - api_key=os.getenv("FIREWORKS_API_KEY"), - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(FireworksWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py b/examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py deleted file mode 100644 index 7a479b6b3..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-gemini-multimodal.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from dotenv import load_dotenv -from multimodal_base_function_calling import MultimodalWeatherBot - -from pipecat.adapters.schemas.tools_schema import AdapterType -from pipecat.services.gemini_multimodal_live import GeminiMultimodalLiveLLMService - -load_dotenv(override=True) - - -class GeminiMultimodalWeatherBot(MultimodalWeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - search_tool = {"google_search": {}} - tools_def = MultimodalWeatherBot.tools() - tools_def.custom_tools = {AdapterType.GEMINI: [search_tool]} - - llm = GeminiMultimodalLiveLLMService( - api_key=os.getenv("GOOGLE_API_KEY"), - voice_id="Puck", - transcribe_user_audio=True, - transcribe_model_audio=True, - tools=tools_def, - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(GeminiMultimodalWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-gemini.py b/examples/unified-format-function-calling/standard-function-calling-gemini.py deleted file mode 100644 index d164c9e67..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-gemini.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.google import GoogleLLMService - -load_dotenv(override=True) - - -class GeminiWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(GeminiWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-grok.py b/examples/unified-format-function-calling/standard-function-calling-grok.py deleted file mode 100644 index 3c2570d8a..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-grok.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.grok import GrokLLMService - -load_dotenv(override=True) - - -class GrokWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(GrokWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-groq.py b/examples/unified-format-function-calling/standard-function-calling-groq.py deleted file mode 100644 index 70a6cef47..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-groq.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.groq import GroqLLMService - -load_dotenv(override=True) - - -class GroqWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(GroqWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-nim.py b/examples/unified-format-function-calling/standard-function-calling-nim.py deleted file mode 100644 index f0d1e892b..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-nim.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.nim import NimLLMService - -load_dotenv(override=True) - - -class NimWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = NimLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(NimWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-openai-realtime.py b/examples/unified-format-function-calling/standard-function-calling-openai-realtime.py deleted file mode 100644 index 203d0abc3..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-openai-realtime.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from dotenv import load_dotenv -from multimodal_base_function_calling import MultimodalWeatherBot - -from pipecat.services.openai_realtime_beta import ( - InputAudioTranscription, - OpenAIRealtimeBetaLLMService, - SessionProperties, - TurnDetection, -) - -load_dotenv(override=True) - - -class OpenAiRealTimeWeatherBot(MultimodalWeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), - # Set openai TurnDetection parameters. Not setting this at all will turn it - # on by default - turn_detection=TurnDetection(silence_duration_ms=1000), - ) - - llm = OpenAIRealtimeBetaLLMService( - api_key=os.getenv("OPENAI_API_KEY"), - session_properties=session_properties, - start_audio_paused=False, - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(OpenAiRealTimeWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-openai.py b/examples/unified-format-function-calling/standard-function-calling-openai.py deleted file mode 100644 index 7763ee505..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-openai.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.openai import OpenAILLMService - -load_dotenv(override=True) - - -class OpenAiWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(OpenAiWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-openrouter.py b/examples/unified-format-function-calling/standard-function-calling-openrouter.py deleted file mode 100644 index cb1ad3964..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-openrouter.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.openrouter import OpenRouterLLMService - -load_dotenv(override=True) - - -class OpenRouterWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = OpenRouterLLMService( - api_key=os.getenv("OPENROUTER_API_KEY"), model="openai/gpt-4o-2024-11-20" - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(OpenRouterWeatherBot().run()) diff --git a/examples/unified-format-function-calling/standard-function-calling-together.py b/examples/unified-format-function-calling/standard-function-calling-together.py deleted file mode 100644 index fe100c95c..000000000 --- a/examples/unified-format-function-calling/standard-function-calling-together.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os - -from base_function_calling import WeatherBot -from dotenv import load_dotenv - -from pipecat.services.together import TogetherLLMService - -load_dotenv(override=True) - - -class TogetherWeatherBot(WeatherBot): - """Main class defining the LLM and passing it to the base handler.""" - - def __init__(self): - llm = TogetherLLMService( - api_key=os.getenv("TOGETHER_API_KEY"), - model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - ) - super().__init__(llm) - - -if __name__ == "__main__": - asyncio.run(TogetherWeatherBot().run()) diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index 60a3b6a8b..c59cd29c2 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -579,7 +579,6 @@ class AzureTTSService(AzureBaseTTSService): async def flush_audio(self): logger.trace(f"{self}: flushing audio") - # TODO: check what we need to implement here ? async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"{self}: Generating TTS [{text}]") From 06c742a2ad147536db88c6d25dd31c9da2e3358b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 5 Mar 2025 17:20:02 -0800 Subject: [PATCH 266/427] AudioBufferProcessor: add on_user_turn_audio_data and on_bot_turn_audio_data --- CHANGELOG.md | 4 + .../audio/audio_buffer_processor.py | 73 +++++++++++++++++-- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff473ea2..c631bb0e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to + `AudioBufferProcessor`. This gives the ability to grab the audio of only that + turn for both the user and the bot. + - Added new base class `BaseObject` which is now the base class of `FrameProcessor`, `PipelineRunner`, `PipelineTask` and `BaseTransport`. The new `BaseObject` adds supports for event handlers. diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 1863a0ee6..c1b2eb810 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -10,12 +10,16 @@ from typing import Optional from pipecat.audio.utils import create_default_resampler, interleave_stereo_audio, mix_audio from pipecat.frames.frames import ( AudioRawFrame, + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, CancelFrame, EndFrame, Frame, InputAudioRawFrame, OutputAudioRawFrame, StartFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -30,12 +34,15 @@ class AudioBufferProcessor(FrameProcessor): Events: on_audio_data: Triggered when buffer_size is reached, providing merged audio on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks + on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio + on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio Args: sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1 buffer_size (int): Size of buffer before triggering events. 0 for no buffering user_continuous_stream (bool): Whether user audio is continuous or speech-only + enable_turn_audio (bool): Whether turn audio event handlers should be triggered Audio handling: - Mono output (num_channels=1): User and bot audio are mixed @@ -56,18 +63,26 @@ class AudioBufferProcessor(FrameProcessor): num_channels: int = 1, buffer_size: int = 0, user_continuous_stream: bool = True, + enable_turn_audio: bool = False, **kwargs, ): super().__init__(**kwargs) self._init_sample_rate = sample_rate self._sample_rate = 0 + self._audio_buffer_size_1s = 0 self._num_channels = num_channels self._buffer_size = buffer_size self._user_continuous_stream = user_continuous_stream + self._enable_turn_audio = enable_turn_audio self._user_audio_buffer = bytearray() self._bot_audio_buffer = bytearray() + self._user_speaking = False + self._bot_speaking = False + self._user_turn_audio_buffer = bytearray() + self._bot_turn_audio_buffer = bytearray() + # Intermittent (non continous user stream variables) self._last_user_frame_at = 0 self._last_bot_frame_at = 0 @@ -78,6 +93,8 @@ class AudioBufferProcessor(FrameProcessor): self._register_event_handler("on_audio_data") self._register_event_handler("on_track_audio_data") + self._register_event_handler("on_user_turn_audio_data") + self._register_event_handler("on_bot_turn_audio_data") @property def sample_rate(self) -> int: @@ -150,13 +167,9 @@ class AudioBufferProcessor(FrameProcessor): self._update_sample_rate(frame) if self._recording: - if self._user_continuous_stream: - await self._handle_continuous_stream(frame) - else: - await self._handle_intermittent_stream(frame) - - if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size: - await self._call_on_audio_data_handler() + await self._process_recording(frame) + if self._enable_turn_audio: + await self._process_turn_recording(frame) if isinstance(frame, (CancelFrame, EndFrame)): await self.stop_recording() @@ -165,6 +178,50 @@ class AudioBufferProcessor(FrameProcessor): def _update_sample_rate(self, frame: StartFrame): self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate + self._audio_buffer_size_1s = self._sample_rate * 2 + + async def _process_recording(self, frame: Frame): + if self._user_continuous_stream: + await self._handle_continuous_stream(frame) + else: + await self._handle_intermittent_stream(frame) + + if self._buffer_size > 0 and len(self._user_audio_buffer) > self._buffer_size: + await self._call_on_audio_data_handler() + + async def _process_turn_recording(self, frame: Frame): + if isinstance(frame, UserStartedSpeakingFrame): + self._user_speaking = True + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._call_event_handler( + "on_user_turn_audio_data", self._user_turn_audio_buffer, self.sample_rate, 1 + ) + self._user_speaking = False + self._user_turn_audio_buffer = bytearray() + elif isinstance(frame, BotStartedSpeakingFrame): + self._bot_speaking = True + elif isinstance(frame, BotStoppedSpeakingFrame): + await self._call_event_handler( + "on_bot_turn_audio_data", self._bot_turn_audio_buffer, self.sample_rate, 1 + ) + self._bot_speaking = False + self._bot_turn_audio_buffer = bytearray() + + if isinstance(frame, InputAudioRawFrame): + resampled = await self._resample_audio(frame) + self._user_turn_audio_buffer += resampled + # In the case of the user, we need to keep a short buffer of audio + # since VAD notification of when the user starts speaking comes + # later. + if ( + not self._user_speaking + and len(self._user_turn_audio_buffer) > self._audio_buffer_size_1s + ): + discarded = len(self._user_turn_audio_buffer) - self._audio_buffer_size_1s + self._user_turn_audio_buffer = self._user_turn_audio_buffer[discarded:] + elif self._bot_speaking and isinstance(frame, OutputAudioRawFrame): + resampled = await self._resample_audio(frame) + self._bot_turn_audio_buffer += resampled async def _handle_continuous_stream(self, frame: Frame): if isinstance(frame, InputAudioRawFrame): @@ -233,6 +290,8 @@ class AudioBufferProcessor(FrameProcessor): def _reset_audio_buffers(self): self._user_audio_buffer = bytearray() self._bot_audio_buffer = bytearray() + self._user_turn_audio_buffer = bytearray() + self._bot_turn_audio_buffer = bytearray() async def _resample_audio(self, frame: AudioRawFrame) -> bytes: return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate) From d7e93551d24da068f43acb607cfa15648fc01322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 5 Mar 2025 17:20:19 -0800 Subject: [PATCH 267/427] examples(chatbot-audio-recording): add support for user/bot turn audio --- examples/chatbot-audio-recording/bot.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/chatbot-audio-recording/bot.py b/examples/chatbot-audio-recording/bot.py index d37d0414e..2dd87449c 100644 --- a/examples/chatbot-audio-recording/bot.py +++ b/examples/chatbot-audio-recording/bot.py @@ -33,9 +33,11 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def save_audio(audio: bytes, sample_rate: int, num_channels: int): +async def save_audio(audio: bytes, sample_rate: int, num_channels: int, name: str): if len(audio) > 0: - filename = f"conversation_recording{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav" + filename = ( + f"{name}_conversation_recording{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav" + ) with io.BytesIO() as buffer: with wave.open(buffer, "wb") as wf: wf.setsampwidth(2) @@ -110,7 +112,7 @@ async def main(): # NOTE: Watch out! This will save all the conversation in memory. You # can pass `buffer_size` to get periodic callbacks. - audiobuffer = AudioBufferProcessor() + audiobuffer = AudioBufferProcessor(enable_turn_audio=True) pipeline = Pipeline( [ @@ -128,7 +130,15 @@ async def main(): @audiobuffer.event_handler("on_audio_data") async def on_audio_data(buffer, audio, sample_rate, num_channels): - await save_audio(audio, sample_rate, num_channels) + await save_audio(audio, sample_rate, num_channels, "full") + + @audiobuffer.event_handler("on_user_turn_audio_data") + async def on_user_turn_audio_data(buffer, audio, sample_rate, num_channels): + await save_audio(audio, sample_rate, num_channels, "user") + + @audiobuffer.event_handler("on_bot_turn_audio_data") + async def on_bot_turn_audio_data(buffer, audio, sample_rate, num_channels): + await save_audio(audio, sample_rate, num_channels, "bot") @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): From a91c26785f7ec407d053e867f65c14a2446256d8 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 6 Mar 2025 18:31:48 -0500 Subject: [PATCH 268/427] Store recording in a folder --- examples/chatbot-audio-recording/bot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/chatbot-audio-recording/bot.py b/examples/chatbot-audio-recording/bot.py index 2dd87449c..2162ed5e2 100644 --- a/examples/chatbot-audio-recording/bot.py +++ b/examples/chatbot-audio-recording/bot.py @@ -32,11 +32,15 @@ load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") +# Create the recordings directory if it doesn't exist +os.makedirs("recordings", exist_ok=True) + async def save_audio(audio: bytes, sample_rate: int, num_channels: int, name: str): if len(audio) > 0: - filename = ( - f"{name}_conversation_recording{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav" + filename = os.path.join( + "recordings", + f"{name}_conversation_recording{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}.wav", ) with io.BytesIO() as buffer: with wave.open(buffer, "wb") as wf: From e1c8a09b607bfca2a946a693261390afa58a8a77 Mon Sep 17 00:00:00 2001 From: Kunal Shah Date: Mon, 10 Mar 2025 14:43:58 -0700 Subject: [PATCH 269/427] [Cartesia] Update the default alias for Cartesia TTS Service --- examples/translation-chatbot/bot.py | 2 +- src/pipecat/services/cartesia.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index dc1d05f3d..3443f3370 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -144,7 +144,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="34dbb662-8e98-413c-a1ef-1a3407675fe7", # Spanish Narrator Man - model="sonic-multilingual", + model="sonic-2", ) in_language = "English" diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 9bbde11bb..8b7f57c63 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -84,7 +84,7 @@ class CartesiaTTSService(AudioContextWordTTSService): voice_id: str, cartesia_version: str = "2024-06-10", url: str = "wss://api.cartesia.ai/tts/websocket", - model: str = "sonic", + model: str = "sonic-2", sample_rate: Optional[int] = None, encoding: str = "pcm_s16le", container: str = "raw", @@ -311,7 +311,7 @@ class CartesiaHttpTTSService(TTSService): *, api_key: str, voice_id: str, - model: str = "sonic", + model: str = "sonic-2", base_url: str = "https://api.cartesia.ai", sample_rate: Optional[int] = None, encoding: str = "pcm_s16le", From 40562402a2e3f33d7d6fbeede4528d23fa580da2 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 10 Mar 2025 21:10:11 -0400 Subject: [PATCH 270/427] Changelog entry for Cartesia model update --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c631bb0e2..2dafdfa26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. +### Changed + +- Updated the default mode for `CartesiaTTSService` and + `CartesiaHttpTTSService` to `sonic-2`. + ## [0.0.58] - 2025-02-26 ### Added From a62741df94a54f00e364cb3e0a8d0b7aec3bb998 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Mar 2025 07:56:27 -0400 Subject: [PATCH 271/427] Add support for Chirp voices in GoogleTTSService --- CHANGELOG.md | 2 ++ src/pipecat/services/google/google.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dafdfa26..9c4ee5eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for Chirp voices in `GoogleTTSService`. + - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that turn for both the user and the bot. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 1d914a9bb..009fd772f 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1448,10 +1448,13 @@ class GoogleTTSService(TTSService): try: await self.start_ttfb_metrics() + # Check if the voice is a Chirp voice (including Chirp 3) or Journey voice + is_chirp_voice = "chirp" in self._voice_id.lower() is_journey_voice = "journey" in self._voice_id.lower() # Create synthesis input based on voice_id - if is_journey_voice: + if is_chirp_voice or is_journey_voice: + # Chirp and Journey voices don't support SSML, use plain text synthesis_input = texttospeech_v1.SynthesisInput(text=text) else: ssml = self._construct_ssml(text) From 740ba4e759387a11384a02728e1cbe2a6fefd50d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 11 Mar 2025 14:29:03 -0700 Subject: [PATCH 272/427] ai_services: fix abstractmethod issues --- src/pipecat/services/ai_services.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 65c9b5d92..8533a23fc 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -270,10 +270,6 @@ class TTSService(AIService): def set_voice(self, voice: str): self._voice_id = voice - @abstractmethod - async def flush_audio(self): - pass - # Converts the text to audio. @abstractmethod async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: @@ -539,6 +535,9 @@ class WebsocketTTSService(TTSService, WebsocketService): TTSService.__init__(self, **kwargs) WebsocketService.__init__(self) + async def flush_audio(self): + pass + class InterruptibleTTSService(WebsocketTTSService): """This is a base class for websocket-based TTS services that don't support @@ -762,11 +761,9 @@ class STTService(AIService): def sample_rate(self) -> int: return self._sample_rate - @abstractmethod async def set_model(self, model: str): self.set_model_name(model) - @abstractmethod async def set_language(self, language: Language): pass From ecc44111283940bad5216547c60e2bf2b4e76e36 Mon Sep 17 00:00:00 2001 From: Lucas Rothman Date: Tue, 11 Mar 2025 16:02:33 -0700 Subject: [PATCH 273/427] Tavus support for custom output rate --- src/pipecat/services/tavus.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index b0ca699cb..abb8b921d 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -37,6 +37,7 @@ class TavusVideoService(AIService): replica_id: str, persona_id: str = "pipecat0", # Use `pipecat0` so that your TTS voice is used in place of the Tavus persona session: aiohttp.ClientSession, + output_sample_rate: int = 16000, **kwargs, ) -> None: super().__init__(**kwargs) @@ -44,6 +45,7 @@ class TavusVideoService(AIService): self._replica_id = replica_id self._persona_id = persona_id self._session = session + self._output_sample_rate = output_sample_rate self._conversation_id: str @@ -94,7 +96,7 @@ class TavusVideoService(AIService): async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None: """Encodes audio to base64 and sends it to Tavus""" if not done: - audio = await self._resampler.resample(audio, in_rate, 16000) + audio = await self._resampler.resample(audio, in_rate, self._output_sample_rate) audio_base64 = base64.b64encode(audio).decode("utf-8") logger.trace(f"{self}: sending {len(audio)} bytes") await self._send_audio_message(audio_base64, done=done) @@ -108,7 +110,7 @@ class TavusVideoService(AIService): elif isinstance(frame, TTSAudioRawFrame): await self._encode_audio_and_send(frame.audio, frame.sample_rate, done=False) elif isinstance(frame, TTSStoppedFrame): - await self._encode_audio_and_send(b"\x00", 16000, done=True) + await self._encode_audio_and_send(b"\x00", self._output_sample_rate, done=True) await self.stop_ttfb_metrics() await self.stop_processing_metrics() elif isinstance(frame, StartInterruptionFrame): @@ -137,6 +139,7 @@ class TavusVideoService(AIService): "inference_id": self._current_idx_str, "audio": audio_base64, "done": done, + "sample_rate": self._output_sample_rate, }, } ) From e6f269a903626e60bd5126efa4a71834a2ee7291 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 15:50:48 -0500 Subject: [PATCH 274/427] Add flush_audio to FishTTSService --- CHANGELOG.md | 2 ++ src/pipecat/services/fish.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c4ee5eaf..b1de5e50a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for Chirp voices in `GoogleTTSService`. +- Added a `flush_audio()` method to `FishTTSService`. + - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that turn for both the user and the bot. diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 0b2729958..96968d6e9 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -148,6 +148,14 @@ class FishAudioTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"Error closing websocket: {e}") + async def flush_audio(self): + """Flush any buffered audio by sending a flush event to Fish Audio.""" + logger.trace(f"{self}: Flushing audio buffers") + if not self._websocket: + return + flush_message = {"event": "flush"} + await self._get_websocket().send(ormsgpack.packb(flush_message)) + def _get_websocket(self): if self._websocket: return self._websocket From cfca7269f40e417dc7d1f611ecba7d0282c838b9 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Mar 2025 21:53:03 -0400 Subject: [PATCH 275/427] Update the Cartesia voice in all demos with one built for sonic-2 --- examples/bot-ready-signalling/server/signalling_bot.py | 2 +- examples/deployment/modal-example/bot.py | 2 +- examples/foundational/01-say-one-thing.py | 2 +- examples/foundational/01a-local-audio.py | 2 +- examples/foundational/01b-livekit-audio.py | 2 +- examples/foundational/02-llm-say-one-thing.py | 2 +- examples/foundational/05-sync-speech-and-image.py | 2 +- examples/foundational/05a-local-sync-speech-and-image.py | 2 +- examples/foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-vad.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-anthropic.py | 2 +- examples/foundational/07b-interruptible-langchain.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07l-interruptible-together.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/10-wake-phrase.py | 2 +- examples/foundational/11-sound-effects.py | 2 +- examples/foundational/12-describe-video.py | 2 +- examples/foundational/12a-describe-video-gemini-flash.py | 2 +- examples/foundational/12b-describe-video-gpt-4o.py | 2 +- examples/foundational/12c-describe-video-anthropic.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14a-function-calling-anthropic.py | 2 +- examples/foundational/14b-function-calling-anthropic-video.py | 2 +- examples/foundational/14c-function-calling-together.py | 2 +- examples/foundational/14d-function-calling-video.py | 2 +- examples/foundational/14e-function-calling-gemini.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14k-function-calling-cerebras.py | 2 +- examples/foundational/14l-function-calling-deepseek.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/15-switch-voices.py | 2 +- examples/foundational/15a-switch-languages.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/20d-persistent-context-gemini.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- examples/foundational/22b-natural-conversation-proposal.py | 2 +- examples/foundational/22c-natural-conversation-mixed-llms.py | 2 +- examples/foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- examples/foundational/26d-gemini-multimodal-live-text.py | 2 +- examples/foundational/28a-transcription-processor-openai.py | 2 +- examples/foundational/28b-transcript-processor-anthropic.py | 2 +- examples/foundational/28c-transcription-processor-gemini.py | 2 +- examples/foundational/29-livekit-audio-chat.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/32-gemini-grounding-metadata.py | 2 +- examples/foundational/34-audio-recording.py | 2 +- examples/moondream-chatbot/bot.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- examples/patient-intake/bot.py | 2 +- examples/telnyx-chatbot/bot.py | 2 +- examples/twilio-chatbot/bot.py | 2 +- examples/websocket-server/bot.py | 2 +- 65 files changed, 65 insertions(+), 65 deletions(-) diff --git a/examples/bot-ready-signalling/server/signalling_bot.py b/examples/bot-ready-signalling/server/signalling_bot.py index c6e47b812..31c940181 100644 --- a/examples/bot-ready-signalling/server/signalling_bot.py +++ b/examples/bot-ready-signalling/server/signalling_bot.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) runner = PipelineRunner() diff --git a/examples/deployment/modal-example/bot.py b/examples/deployment/modal-example/bot.py index 02bcff379..97ec11905 100644 --- a/examples/deployment/modal-example/bot.py +++ b/examples/deployment/modal-example/bot.py @@ -34,7 +34,7 @@ async def main(room_url: str, token: str): ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY", ""), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + api_key=os.getenv("CARTESIA_API_KEY", ""), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index dbbb94cb4..05db53c66 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -36,7 +36,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) runner = PipelineRunner() diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index 20aaedfae..bb5cc22bc 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -29,7 +29,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) pipeline = Pipeline([tts, transport.output()]) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index ed610e027..46c7f8b4c 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -83,7 +83,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) runner = PipelineRunner() diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index d19496fae..b11e1d302 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -37,7 +37,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 4b343d2af..8b0021c6c 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -87,7 +87,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 7bd689fe6..6b24629ce 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index ee9ae056a..1a4f4d5a6 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -74,7 +74,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 077f9b7dd..0230051a7 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -93,7 +93,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index d6d3aca7c..ee1b647ab 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 13e728bfc..e0f7fcf95 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index f855f81e6..84c3838f3 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index efa631b34..6f61eafc5 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) prompt = ChatPromptTemplate.from_messages( diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 95d786297..831919412 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) timestamp = int(time.time()) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index b1e92e787..08fc3a1c9 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index 54ebd4c2c..f02748c15 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index bffa35322..3d6b7d6f9 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index e1232a155..de6b85b25 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -213,7 +213,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 1bdeba15f..b6cfb05a0 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 2f62d5d71..d50d9eaa2 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -100,7 +100,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index 2ae3ed1ad..e99599ddc 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index 876aa7f07..c830a791c 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index f2944b140..ab3e32837 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index a8f3f0455..f5be8e434 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 043f278e2..c592006fe 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -58,7 +58,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 14e788f99..1605b1720 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -53,7 +53,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index b31e59442..cd7f0fb97 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -62,7 +62,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 7f47eb28e..6294617cd 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 15344b7ac..714f54c5d 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -60,7 +60,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 146ed77d7..99d90aa76 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -68,7 +68,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 5bbddcc4d..4d915f5d6 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -61,7 +61,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 423919772..979c5c252 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 669055a55..ad8b64d34 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AzureLLMService( diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 88168383f..719624f08 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = FireworksLLMService( diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 5d4b123d0..612972846 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady # text_filter=MarkdownTextFilter(), ) diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index b9f1c6b18..252d29289 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 6add663d5..e468f1ef0 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index aee185f75..6a823b0ba 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -55,7 +55,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 49280449f..466e54545 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -78,7 +78,7 @@ async def main(): british_lady = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) barbershop_man = CartesiaTTSService( diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 217502ea2..beb9929e0 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -71,7 +71,7 @@ async def main(): english_tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) spanish_tts = CartesiaTTSService( diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 00c969df4..0d60acd72 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -48,7 +48,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 8da683a47..67f703e06 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -184,7 +184,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 64829450d..29eb85bc7 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -179,7 +179,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 74c4e69be..f5746716c 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -234,7 +234,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 9282f4e4b..d4580c712 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -56,7 +56,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 1eb2dd956..17c396bdb 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -229,7 +229,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 362c7554d..8485be2f3 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -433,7 +433,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 0041ff530..503b0f5c9 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -644,7 +644,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # This is the LLM that will transcribe user speech. diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index 1b443190d..f1653b655 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index e3ff21d95..fb73b70f2 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -294,7 +294,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) conversation_llm = GoogleLLMService( diff --git a/examples/foundational/26d-gemini-multimodal-live-text.py b/examples/foundational/26d-gemini-multimodal-live-text.py index 2e0e27a13..147b882eb 100644 --- a/examples/foundational/26d-gemini-multimodal-live-text.py +++ b/examples/foundational/26d-gemini-multimodal-live-text.py @@ -78,7 +78,7 @@ async def main(): # ) tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121" ) messages = [ diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index f341cc0dd..884197ff9 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index 05956fdf7..acad368d3 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index 3760a227f..62426fa85 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -134,7 +134,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = GoogleLLMService( diff --git a/examples/foundational/29-livekit-audio-chat.py b/examples/foundational/29-livekit-audio-chat.py index 2ad02c296..1965477e2 100644 --- a/examples/foundational/29-livekit-audio-chat.py +++ b/examples/foundational/29-livekit-audio-chat.py @@ -131,7 +131,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index 49885d82c..f44d47e9f 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -89,7 +89,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 516549694..0432bb830 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -81,7 +81,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # Initialize the Gemini Multimodal Live model diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 1d0431b83..94877c722 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -107,7 +107,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4") diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 5fb86ac79..d44b3a34e 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -154,7 +154,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index d1cde2f22..f6787e933 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -96,7 +96,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady text_filter=MarkdownTextFilter(), ) diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 75279fce3..5ac2ebe93 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -303,7 +303,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) # tts = CartesiaTTSService( diff --git a/examples/telnyx-chatbot/bot.py b/examples/telnyx-chatbot/bot.py index 8243e05a4..a31e4ac91 100644 --- a/examples/telnyx-chatbot/bot.py +++ b/examples/telnyx-chatbot/bot.py @@ -54,7 +54,7 @@ async def run_bot( tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index afddc3fe9..be3a0c40d 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -74,7 +74,7 @@ async def run_bot(websocket_client: WebSocket, stream_sid: str, testing: bool): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady push_silence_after_stop=testing, ) diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index d44c77df8..8f2e98e7d 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady ) messages = [ From 3522bbb533d9ec49e510b329215f185b0a5a0494 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 11 Mar 2025 21:55:18 -0400 Subject: [PATCH 276/427] tmp --- examples/bot-ready-signalling/server/signalling_bot.py | 2 +- examples/foundational/01-say-one-thing.py | 2 +- examples/foundational/01a-local-audio.py | 2 +- examples/foundational/01b-livekit-audio.py | 2 +- examples/foundational/02-llm-say-one-thing.py | 2 +- examples/foundational/05-sync-speech-and-image.py | 2 +- examples/foundational/05a-local-sync-speech-and-image.py | 2 +- examples/foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-vad.py | 2 +- examples/foundational/07-interruptible.py | 2 +- examples/foundational/07a-interruptible-anthropic.py | 2 +- examples/foundational/07b-interruptible-langchain.py | 2 +- examples/foundational/07h-interruptible-openpipe.py | 2 +- examples/foundational/07j-interruptible-gladia.py | 2 +- examples/foundational/07l-interruptible-together.py | 2 +- examples/foundational/07o-interruptible-assemblyai.py | 2 +- examples/foundational/07s-interruptible-google-audio-in.py | 2 +- examples/foundational/10-wake-phrase.py | 2 +- examples/foundational/11-sound-effects.py | 2 +- examples/foundational/12-describe-video.py | 2 +- examples/foundational/12a-describe-video-gemini-flash.py | 2 +- examples/foundational/12b-describe-video-gpt-4o.py | 2 +- examples/foundational/12c-describe-video-anthropic.py | 2 +- examples/foundational/14-function-calling.py | 2 +- examples/foundational/14a-function-calling-anthropic.py | 2 +- .../foundational/14b-function-calling-anthropic-video.py | 2 +- examples/foundational/14c-function-calling-together.py | 2 +- examples/foundational/14d-function-calling-video.py | 2 +- examples/foundational/14e-function-calling-gemini.py | 2 +- examples/foundational/14f-function-calling-groq.py | 2 +- examples/foundational/14g-function-calling-grok.py | 2 +- examples/foundational/14h-function-calling-azure.py | 2 +- examples/foundational/14i-function-calling-fireworks.py | 2 +- examples/foundational/14j-function-calling-nim.py | 2 +- examples/foundational/14k-function-calling-cerebras.py | 2 +- examples/foundational/14l-function-calling-deepseek.py | 2 +- examples/foundational/14n-function-calling-perplexity.py | 2 +- examples/foundational/15-switch-voices.py | 7 +++++-- examples/foundational/15a-switch-languages.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/20a-persistent-context-openai.py | 2 +- examples/foundational/20c-persistent-context-anthropic.py | 2 +- examples/foundational/20d-persistent-context-gemini.py | 2 +- examples/foundational/22-natural-conversation.py | 2 +- examples/foundational/22b-natural-conversation-proposal.py | 2 +- .../foundational/22c-natural-conversation-mixed-llms.py | 2 +- .../foundational/22d-natural-conversation-gemini-audio.py | 2 +- examples/foundational/23-bot-background-sound.py | 2 +- examples/foundational/25-google-audio-in.py | 2 +- .../foundational/28a-transcription-processor-openai.py | 2 +- .../foundational/28b-transcript-processor-anthropic.py | 2 +- .../foundational/28c-transcription-processor-gemini.py | 2 +- examples/foundational/29-livekit-audio-chat.py | 2 +- examples/foundational/30-observer.py | 2 +- examples/foundational/32-gemini-grounding-metadata.py | 2 +- examples/moondream-chatbot/bot.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- examples/patient-intake/bot.py | 2 +- examples/telnyx-chatbot/bot.py | 2 +- examples/twilio-chatbot/bot.py | 2 +- examples/websocket-server/bot.py | 2 +- 62 files changed, 66 insertions(+), 63 deletions(-) diff --git a/examples/bot-ready-signalling/server/signalling_bot.py b/examples/bot-ready-signalling/server/signalling_bot.py index 31c940181..3b98700aa 100644 --- a/examples/bot-ready-signalling/server/signalling_bot.py +++ b/examples/bot-ready-signalling/server/signalling_bot.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) runner = PipelineRunner() diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index 05db53c66..945afefda 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -36,7 +36,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) runner = PipelineRunner() diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index bb5cc22bc..c5ed3610e 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -29,7 +29,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) pipeline = Pipeline([tts, transport.output()]) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index 46c7f8b4c..5529d4184 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -83,7 +83,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) runner = PipelineRunner() diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index b11e1d302..f6f8030f1 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -37,7 +37,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 8b0021c6c..f6d415e64 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -87,7 +87,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 6b24629ce..c09631061 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaHttpTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) imagegen = FalImageGenService( diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 1a4f4d5a6..d59d143f8 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -74,7 +74,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 0230051a7..ab1c9399d 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -93,7 +93,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index ee1b647ab..2afc36702 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index e0f7fcf95..5eab40454 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index 84c3838f3..85f5ac68e 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 6f61eafc5..dd7b0b69c 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -64,7 +64,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) prompt = ChatPromptTemplate.from_messages( diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index 831919412..bded77139 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) timestamp = int(time.time()) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index 08fc3a1c9..7dcd44a7a 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index f02748c15..d17a4fe7f 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -46,7 +46,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 3d6b7d6f9..3c30cabc9 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -51,7 +51,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index de6b85b25..76711d730 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -213,7 +213,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index b6cfb05a0..0fbd47f4a 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -47,7 +47,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index d50d9eaa2..ed189551a 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -100,7 +100,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index e99599ddc..5eeb69718 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index c830a791c..59a9f40db 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -77,7 +77,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index ab3e32837..fb3aa039e 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index f5be8e434..bd364a636 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -76,7 +76,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) @transport.event_handler("on_first_participant_joined") diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index c592006fe..1d4e37344 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -58,7 +58,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 1605b1720..13505550b 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -53,7 +53,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index cd7f0fb97..8c49900fa 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -62,7 +62,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index 6294617cd..dd3eb714f 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = TogetherLLMService( diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 714f54c5d..6e290d55f 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -60,7 +60,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 99d90aa76..bf022a65a 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -68,7 +68,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index 4d915f5d6..c402ac91a 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -61,7 +61,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 979c5c252..e6b822e39 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index ad8b64d34..aefa1a474 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AzureLLMService( diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 719624f08..50291615b 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = FireworksLLMService( diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 612972846..d703d637a 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady # text_filter=MarkdownTextFilter(), ) diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 252d29289..9b3641307 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index e468f1ef0..0360f0b84 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") diff --git a/examples/foundational/14n-function-calling-perplexity.py b/examples/foundational/14n-function-calling-perplexity.py index 6a823b0ba..9d2439fb0 100644 --- a/examples/foundational/14n-function-calling-perplexity.py +++ b/examples/foundational/14n-function-calling-perplexity.py @@ -55,7 +55,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = PerplexityLLMService(api_key=os.getenv("PERPLEXITY_API_KEY"), model="sonar") diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 466e54545..15bd2f954 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -78,7 +78,7 @@ async def main(): british_lady = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) barbershop_man = CartesiaTTSService( @@ -125,7 +125,10 @@ async def main(): llm, # LLM ParallelPipeline( # TTS (one of the following vocies) [FunctionFilter(news_lady_filter), news_lady], # News Lady voice - [FunctionFilter(british_lady_filter), british_lady], # British Lady voice + [ + FunctionFilter(british_lady_filter), + british_lady, + ], # British Reading Lady voice [FunctionFilter(barbershop_man_filter), barbershop_man], # Barbershop Man voice ), transport.output(), # Transport bot output diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index beb9929e0..4e05efcb0 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -71,7 +71,7 @@ async def main(): english_tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) spanish_tts = CartesiaTTSService( diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 0d60acd72..0a41f9d84 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -48,7 +48,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index 67f703e06..0be0536cd 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -184,7 +184,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 29eb85bc7..65d45c3d6 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -179,7 +179,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index f5746716c..740bdc68f 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -234,7 +234,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index d4580c712..d1f84bb90 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -56,7 +56,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 17c396bdb..c97289217 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -229,7 +229,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 8485be2f3..d342a2765 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -433,7 +433,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will be used to detect if the user has finished a diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 503b0f5c9..9e3b8c9a3 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -644,7 +644,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # This is the LLM that will transcribe user speech. diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index f1653b655..92d6cdc42 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -59,7 +59,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index fb73b70f2..830cfb769 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -294,7 +294,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) conversation_llm = GoogleLLMService( diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index 884197ff9..a3707e347 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService( diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index acad368d3..c9f2672e0 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -113,7 +113,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = AnthropicLLMService( diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index 62426fa85..558edc76d 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -134,7 +134,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = GoogleLLMService( diff --git a/examples/foundational/29-livekit-audio-chat.py b/examples/foundational/29-livekit-audio-chat.py index 1965477e2..e5afb5ffb 100644 --- a/examples/foundational/29-livekit-audio-chat.py +++ b/examples/foundational/29-livekit-audio-chat.py @@ -131,7 +131,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ diff --git a/examples/foundational/30-observer.py b/examples/foundational/30-observer.py index f44d47e9f..129407628 100644 --- a/examples/foundational/30-observer.py +++ b/examples/foundational/30-observer.py @@ -89,7 +89,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/foundational/32-gemini-grounding-metadata.py b/examples/foundational/32-gemini-grounding-metadata.py index 0432bb830..5f0a497a5 100644 --- a/examples/foundational/32-gemini-grounding-metadata.py +++ b/examples/foundational/32-gemini-grounding-metadata.py @@ -81,7 +81,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # Initialize the Gemini Multimodal Live model diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index d44b3a34e..5dd88f4d1 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -154,7 +154,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index f6787e933..2a389094e 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -96,7 +96,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady text_filter=MarkdownTextFilter(), ) diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 5ac2ebe93..b6f7fb941 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -303,7 +303,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) # tts = CartesiaTTSService( diff --git a/examples/telnyx-chatbot/bot.py b/examples/telnyx-chatbot/bot.py index a31e4ac91..94f44e2b1 100644 --- a/examples/telnyx-chatbot/bot.py +++ b/examples/telnyx-chatbot/bot.py @@ -54,7 +54,7 @@ async def run_bot( tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index be3a0c40d..7b05e6dfe 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -74,7 +74,7 @@ async def run_bot(websocket_client: WebSocket, stream_sid: str, testing: bool): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady push_silence_after_stop=testing, ) diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index 8f2e98e7d..489605aef 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Lady + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) messages = [ From 5f97f6ff94417733f812caf9300f1ab4232b873e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 16:08:30 -0500 Subject: [PATCH 277/427] Add flush_audio() to LmntTTSService --- CHANGELOG.md | 2 +- src/pipecat/services/lmnt.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1de5e50a..cbafd4de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support for Chirp voices in `GoogleTTSService`. -- Added a `flush_audio()` method to `FishTTSService`. +- Added a `flush_audio()` method to `FishTTSService` and `LmntTTSService`. - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 31ddaf123..cfdf8e6cd 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -170,6 +170,11 @@ class LmntTTSService(InterruptibleTTSService): return self._websocket raise Exception("Websocket not connected") + async def flush_audio(self): + if not self._websocket: + return + await self._get_websocket().send(json.dumps({"flush": True})) + async def _receive_messages(self): """Receive messages from LMNT websocket.""" async for message in self._get_websocket(): From 4a363bebf0f86b6d773741066f25c662fa80634e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 16:04:41 -0500 Subject: [PATCH 278/427] Add a set_language convenience method for GoogleSTTService --- CHANGELOG.md | 4 ++++ src/pipecat/services/google/google.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbafd4de7..4e730b70d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a `flush_audio()` method to `FishTTSService` and `LmntTTSService`. +- Added a `set_language` convenience method for `GoogleSTTService`, allowing + you to set a single language. This is in addition to the `set_languages` + method which allows you to set a list of languages. + - Added `on_user_turn_audio_data` and `on_bot_turn_audio_data` to `AudioBufferProcessor`. This gives the ability to grab the audio of only that turn for both the user and the bot. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 009fd772f..9d4a0a8a8 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -58,7 +58,6 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import ImageGenService, LLMService, STTService, TTSService from pipecat.services.google.frames import LLMSearchResponseFrame from pipecat.services.openai import ( - BaseOpenAILLMService, OpenAIAssistantContextAggregator, OpenAILLMService, OpenAIUnhandledFunctionException, @@ -1730,6 +1729,17 @@ class GoogleSTTService(STTService): await self._disconnect() await self._connect() + async def set_language(self, language: Language): + """Update the service's recognition language. + + A convenience method for setting a single language. + + Args: + language: New language for recognition. + """ + logger.debug(f"Switching STT language to: {language}") + await self.set_languages([language]) + async def set_languages(self, languages: List[Language]): """Update the service's recognition languages. From d9ef19233a59e0cd050d399827a03233bf03ae00 Mon Sep 17 00:00:00 2001 From: Michael Louis Date: Wed, 12 Mar 2025 10:30:23 -0400 Subject: [PATCH 279/427] Added foundational example for ultravox --- .../07u-interruptible-ultravox.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 examples/foundational/07u-interruptible-ultravox.py diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py new file mode 100644 index 000000000..195eaf197 --- /dev/null +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -0,0 +1,93 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.services.ultravox import UltravoxSTTService +from pipecat.services.cartesia import CartesiaTTSService + +load_dotenv(override=True) + +# NOTE: This example requires GPU resources to run efficiently. +# The Ultravox model is compute-intensive and performs best with GPU acceleration. +# This can be deployed on cloud GPU providers like Cerebrium.ai for optimal performance. + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +#Want to initialize the ultravox processor since it takes time to load the model and dont +#want to load it every time the pipeline is run +ultravox_processor = UltravoxSTTService( + model_size="fixie-ai/ultravox-v0_4_1-llama-3_1-8b", + hf_token=os.getenv("HF_TOKEN"), +) + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=False, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), + vad_audio_passthrough=True, + ), + ) + + tts = CartesiaTTSService( + api_key=os.environ.get("CARTESIA_API_KEY"), + voice_id='97f4b8fb-f2fe-444b-bb9a-c109783a857a', + + ) + + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + ultravox_processor, + tts, # TTS + transport.output(), # Transport bot output + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + ), + ) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) + + From 23a4933af9d58828efd62b867f63b6045b9f632c Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Wed, 12 Mar 2025 17:15:31 +0000 Subject: [PATCH 280/427] Initial implementation of Neuphonic service. A TTS provider. --- dot-env.template | 3 + pyproject.toml | 1 + src/pipecat/services/neuphonic.py | 364 ++++++++++++++++++++++++++++++ 3 files changed, 368 insertions(+) create mode 100644 src/pipecat/services/neuphonic.py diff --git a/dot-env.template b/dot-env.template index e31681235..331eba79c 100644 --- a/dot-env.template +++ b/dot-env.template @@ -29,6 +29,9 @@ DAILY_SAMPLE_ROOM_URL=https://... ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=... +# Neuphonic +NEUPHONIC_API_TOKEN=... + # Fal FAL_KEY=... diff --git a/pyproject.toml b/pyproject.toml index dd58b1bfe..b5a870941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] +neuphonic = [ "pyneuphonic~=1.5.10", "websockets>=12.0,<14.0" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py new file mode 100644 index 000000000..b14c325f8 --- /dev/null +++ b/src/pipecat/services/neuphonic.py @@ -0,0 +1,364 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import base64 +import json +from typing import Any, AsyncGenerator, Mapping, Optional + +from loguru import logger +from pydantic import BaseModel + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + LLMFullResponseEndFrame, + StartFrame, + StartInterruptionFrame, + TTSAudioRawFrame, + TTSSpeakFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import TTSService, WordTTSService +from pipecat.services.websocket_service import WebsocketService +from pipecat.transcriptions.language import Language + +# See .env.example for Neuphonic configuration needed +try: + import websockets + from pyneuphonic import Neuphonic, TTSConfig +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_TOKEN` environment variable." + ) + raise Exception(f"Missing module: {e}") + +# Models that support language codes +NEUPHONIC_MULTILINGUAL_MODELS = { + "neu_fast", + "neu_hq", +} + + +def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: + BASE_LANGUAGES = { + Language.DE: "de", + Language.EN: "en", + Language.ES: "es", + Language.NL: "nl", + Language.AR: "ar", + } + + result = BASE_LANGUAGES.get(language) + + # If not found in base languages, try to find the base language from a variant + if not result: + # Convert enum value to string and get the base language part (e.g. es-ES -> es) + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + # Look up the base code in our supported languages + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + +class NeuphonicTTSService(WordTTSService, WebsocketService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + speed: Optional[float] = 1.0 + + def __init__( + self, + *, + api_key: str, + voice_id: Optional[str] = None, + model: str = "neu_hq", + url: str = "wss://api.neuphonic.com", + sample_rate: Optional[int] = 22050, + encoding: str = "pcm_linear", + params: InputParams = InputParams(), + **kwargs, + ): + WordTTSService.__init__( + self, + aggregate_sentences=True, + push_text_frames=False, + push_stop_frames=True, + stop_frame_timeout_s=2.0, + sample_rate=sample_rate, + **kwargs, + ) + WebsocketService.__init__(self) + + self._api_key = api_key + self._url = url + self._settings = { + "lang_code": self.language_to_service_language(params.language), + "speed": params.speed, + "encoding": encoding, + "model": model, + "sampling_rate": sample_rate, + } + self.set_model_name(model) + self.set_voice(voice_id) + + # Indicates if we have sent TTSStartedFrame. It will reset to False when + # there's an interruption or TTSStoppedFrame. + self._started = False + self._cumulative_time = 0 + + def can_generate_metrics(self) -> bool: + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + return language_to_neuphonic_lang_code(language) + + async def set_model(self, model: str): + await super().set_model(model) + logger.info(f"Switching TTS model to: [{model}]") + await self._disconnect() + await self._connect() + + async def _update_settings(self, settings: Mapping[str, Any]): + if "voice_id" in settings: + self.set_voice(settings["voice_id"]) + + await super()._update_settings(settings) + await self._disconnect() + await self._connect() + logger.info(f"Switching TTS to settings: [{self._settings}]") + + async def start(self, frame: StartFrame): + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._disconnect() + + async def flush_audio(self): + if self._websocket: + msg = {"text": ""} + await self._websocket.send(json.dumps(msg)) + + async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + await super().push_frame(frame, direction) + if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): + self._started = False + if isinstance(frame, TTSStoppedFrame): + await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # If we received a TTSSpeakFrame and the LLM response included text (it + # might be that it's only a function calling response) we pause + # processing more frames until we receive a BotStoppedSpeakingFrame. + if isinstance(frame, TTSSpeakFrame): + await self.pause_processing_frames() + elif isinstance(frame, LLMFullResponseEndFrame) and self._started: + await self.pause_processing_frames() + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.resume_processing_frames() + + async def _connect(self): + await self._connect_websocket() + + self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._keepalive_task = self.create_task(self._keepalive_task_handler()) + + async def _disconnect(self): + if self._receive_task: + await self.cancel_task(self._receive_task) + self._receive_task = None + + if self._keepalive_task: + await self.cancel_task(self._keepalive_task) + self._keepalive_task = None + + await self._disconnect_websocket() + + async def _connect_websocket(self): + try: + logger.debug("Connecting to Neuphonic") + + tts_config = { + **self._settings, + "voice_id": self._voice_id, + "model": self.model_name, + } + + query_params = [f"api_key={self._api_key}"] + for key, value in tts_config.items(): + if value is not None: + query_params.append(f"{key}={value}") + + url = f"{self._url}/speak/{self._settings['lang_code']}?{'&'.join(query_params)}" + + self._websocket = await websockets.connect(url) + + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._websocket = None + + async def _disconnect_websocket(self): + try: + await self.stop_all_metrics() + + if self._websocket: + logger.debug("Disconnecting from Neuphonic") + await self._websocket.close() + self._websocket = None + + self._started = False + except Exception as e: + logger.error(f"{self} error closing websocket: {e}") + + async def _receive_messages(self): + async for message in self._websocket: + if isinstance(message, str): + msg = json.loads(message) + if msg.get("data", {}).get("audio") is not None: + await self.stop_ttfb_metrics() + self.start_word_timestamps() + + audio = base64.b64decode(msg["data"]["audio"]) + frame = TTSAudioRawFrame(audio, self.sample_rate, 1) + await self.push_frame(frame) + + async def _keepalive_task_handler(self): + while True: + await asyncio.sleep(10) + await self._send_text("") + + async def _send_text(self, text: str): + if self._websocket: + msg = {"text": text} + logger.debug(f"Sending text to websocket: {msg}") + await self._websocket.send(json.dumps(msg)) + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"Generating TTS: [{text}]") + + try: + if not self._websocket: + await self._connect() + + try: + if not self._started: + await self.start_ttfb_metrics() + yield TTSStartedFrame() + self._started = True + self._cumulative_time = 0 + + await self._send_text(text) + await self.start_tts_usage_metrics(text) + except Exception as e: + logger.error(f"{self} error sending message: {e}") + yield TTSStoppedFrame() + await self._disconnect() + await self._connect() + return + yield None + except Exception as e: + logger.error(f"{self} exception: {e}") + + +class NeuphonicHttpTTSService(TTSService): + """Neuphonic Text-to-Speech service using HTTP streaming. + + Args: + api_key: Neuphonic API key + voice_id: ID of the voice to use + model: Neuphonic model to use (default: "neu_hq") + url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com") + sample_rate: Sample rate for audio output (default: 22050Hz) + encoding: Audio encoding format (default: "pcm_linear") + params: Additional parameters for TTS generation including language and speed + **kwargs: Additional keyword arguments passed to the parent class + """ + + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + speed: Optional[float] = 1.0 + + def __init__( + self, + *, + api_key: str, + voice_id: Optional[str] = None, + model: str = "neu_hq", + url: str = "https://api.neuphonic.com", + sample_rate: Optional[int] = 22050, + encoding: str = "pcm_linear", + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + + self._api_key = api_key + self._url = url + self._settings = { + "lang_code": self.language_to_service_language(params.language), + "speed": params.speed, + "encoding": encoding, + "model": model, + "sampling_rate": sample_rate, + } + self.set_model_name(model) + self.set_voice(voice_id) + + def can_generate_metrics(self) -> bool: + return True + + async def start(self, frame: StartFrame): + await super().start(frame) + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Neuphonic streaming API. + + Args: + text: The text to convert to speech + Yields: + Frames containing audio data and status information + """ + logger.debug(f"Generating TTS: [{text}]") + + client = Neuphonic(api_key=self._api_key, base_url=self._url.replace("https://", "")) + + sse = client.tts.AsyncSSEClient() + + try: + await self.start_ttfb_metrics() + response = sse.send( + text, TTSConfig(**self._settings, model=self.model_name, voice_id=self._voice_id) + ) + + await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + + async for message in response: + if response.status_code != 200: + logger.error(f"{self} error: {message.errors}") + yield ErrorFrame(error=f"Neuphonic API error: {message.errors}") + + await self.stop_ttfb_metrics() + yield TTSAudioRawFrame(message.data.audio, self.sample_rate, 1) + except Exception as e: + logger.error(f"Error in run_tts: {e}") + yield ErrorFrame(error=str(e)) + finally: + yield TTSStoppedFrame() From f84348296816920f5fef87aadbd228186249e02d Mon Sep 17 00:00:00 2001 From: macaki Date: Wed, 12 Mar 2025 11:26:43 -0600 Subject: [PATCH 281/427] [rime client] Sending over trailing space to help indicate end of utterance after a punctuation. --- src/pipecat/services/rime.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 60083f55d..935d87de2 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -249,7 +249,9 @@ class RimeTTSService(AudioContextWordTTSService): async def flush_audio(self): if not self._context_id or not self._websocket: return + logger.trace(f"{self}: flushing audio") + await self._get_websocket().send(json.dumps({"text": " "})) self._context_id = None async def _receive_messages(self): From ead555eb4bca2c6db01beefc9ec2a097c9186109 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Wed, 12 Mar 2025 17:39:04 +0000 Subject: [PATCH 282/427] Corrected versions on pyproject.toml. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5a870941..e698ab134 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -neuphonic = [ "pyneuphonic~=1.5.10", "websockets>=12.0,<14.0" ] +neuphonic = [ "pyneuphonic~=1.5.11", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] From effb5f6cd8fbd012a14a8d92cceda27ff0f2e35d Mon Sep 17 00:00:00 2001 From: macaki Date: Wed, 12 Mar 2025 11:57:25 -0600 Subject: [PATCH 283/427] added changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e730b70d..1ca6ed7e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. +### Fixed + +- Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. + ## [0.0.58] - 2025-02-26 ### Added From 0b9c4b2255e81753e4a37ef405950f6d00d0894e Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Wed, 12 Mar 2025 18:04:48 +0000 Subject: [PATCH 284/427] Fixed a couple of small bugs. --- src/pipecat/services/neuphonic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index b14c325f8..f86280850 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -105,7 +105,6 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): "lang_code": self.language_to_service_language(params.language), "speed": params.speed, "encoding": encoding, - "model": model, "sampling_rate": sample_rate, } self.set_model_name(model) @@ -315,7 +314,6 @@ class NeuphonicHttpTTSService(TTSService): "lang_code": self.language_to_service_language(params.language), "speed": params.speed, "encoding": encoding, - "model": model, "sampling_rate": sample_rate, } self.set_model_name(model) @@ -327,6 +325,9 @@ class NeuphonicHttpTTSService(TTSService): async def start(self, frame: StartFrame): await super().start(frame) + async def flush_audio(self): + pass + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: """Generate speech from text using Neuphonic streaming API. @@ -351,7 +352,7 @@ class NeuphonicHttpTTSService(TTSService): yield TTSStartedFrame() async for message in response: - if response.status_code != 200: + if message.status_code != 200: logger.error(f"{self} error: {message.errors}") yield ErrorFrame(error=f"Neuphonic API error: {message.errors}") From 5c9e33bc7a1ab24feb4ab3f0a265a15a05748ec8 Mon Sep 17 00:00:00 2001 From: macaki Date: Wed, 12 Mar 2025 12:20:18 -0600 Subject: [PATCH 285/427] formatting --- src/pipecat/services/rime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 935d87de2..b2610b06c 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -249,7 +249,7 @@ class RimeTTSService(AudioContextWordTTSService): async def flush_audio(self): if not self._context_id or not self._websocket: return - + logger.trace(f"{self}: flushing audio") await self._get_websocket().send(json.dumps({"text": " "})) self._context_id = None From 08fb931ef6a295bd4b32a86b1e279f6f079374a8 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Thu, 13 Mar 2025 12:10:03 +0000 Subject: [PATCH 286/427] Swapped NEUPHONIC_API_TOKEN for NEUPHONIC_API_KEY. --- dot-env.template | 2 +- src/pipecat/services/neuphonic.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dot-env.template b/dot-env.template index 331eba79c..2da20fc0b 100644 --- a/dot-env.template +++ b/dot-env.template @@ -30,7 +30,7 @@ ELEVENLABS_API_KEY=... ELEVENLABS_VOICE_ID=... # Neuphonic -NEUPHONIC_API_TOKEN=... +NEUPHONIC_API_KEY=... # Fal FAL_KEY=... diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index f86280850..757d60c3e 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -38,7 +38,7 @@ try: except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( - "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_TOKEN` environment variable." + "In order to use Neuphonic, you need to `pip install pipecat-ai[neuphonic]`. Also, set `NEUPHONIC_API_KEY` environment variable." ) raise Exception(f"Missing module: {e}") From 1bf964a667f2a756ef82a22088864adf205c00e8 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Thu, 13 Mar 2025 14:42:42 +0000 Subject: [PATCH 287/427] Added two examples on how to use Neuphonic as a TTS (07u). --- .../07u-interruptible-neuphonic-http.py | 102 ++++++++++++++++++ .../07u-interruptible-neuphonic.py | 102 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 examples/foundational/07u-interruptible-neuphonic-http.py create mode 100644 examples/foundational/07u-interruptible-neuphonic.py diff --git a/examples/foundational/07u-interruptible-neuphonic-http.py b/examples/foundational/07u-interruptible-neuphonic-http.py new file mode 100644 index 000000000..8377c3baf --- /dev/null +++ b/examples/foundational/07u-interruptible-neuphonic-http.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.neuphonic import NeuphonicHttpTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = NeuphonicHttpTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/07u-interruptible-neuphonic.py b/examples/foundational/07u-interruptible-neuphonic.py new file mode 100644 index 000000000..b5d666511 --- /dev/null +++ b/examples/foundational/07u-interruptible-neuphonic.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.neuphonic import NeuphonicTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = NeuphonicTTSService( + api_key=os.getenv("NEUPHONIC_API_KEY"), + voice_id="fc854436-2dac-4d21-aa69-ae17b54e98eb", # Emily + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 25dcf7def6b4ab4eb4529e017d27e22ecb2e99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 12 Mar 2025 19:33:40 -0700 Subject: [PATCH 288/427] PipelineTask: add on_frame_reached_upstream/on_frame_reached_downstream --- CHANGELOG.md | 7 ++++++- src/pipecat/pipeline/task.py | 22 ++++++++++++++++++++++ tests/test_pipeline.py | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ca6ed7e6..776041481 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `on_frame_reached_upstream` and `on_frame_reached_downstream` event + handlers to `PipelineTask`. Those events will be called when a frame reaches + the beginning or end of the pipeline respectively. + - Added support for Chirp voices in `GoogleTTSService`. - Added a `flush_audio()` method to `FishTTSService` and `LmntTTSService`. @@ -80,7 +84,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. +- Fixed an issue in `RimeTTSService` where the last line of text sent didn't + result in an audio output being generated. ## [0.0.58] - 2025-02-26 diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 1c9d2dff9..f20ef9d1f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -119,12 +119,25 @@ class PipelineTaskSink(FrameProcessor): class PipelineTask(BaseTask): """Manages the execution of a pipeline, handling frame processing and task lifecycle. + It has a couple of event handlers `on_frame_reached_upstream` and + `on_frame_reached_downstream` that are called when upstream frames or + downstream frames reach both ends of pipeline. + + @task.event_handler("on_frame_reached_upstream") + async def on_frame_reached_upstream(task, frame): + ... + + @task.event_handler("on_frame_reached_downstream") + async def on_frame_reached_downstream(task, frame): + ... + Args: pipeline: The pipeline to execute. params: Configuration parameters for the pipeline. observers: List of observers for monitoring pipeline execution. clock: Clock implementation for timing operations. check_dangling_tasks: Whether to check for processors' tasks finishing properly. + """ def __init__( @@ -177,6 +190,9 @@ class PipelineTask(BaseTask): self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) + self._register_event_handler("on_frame_reached_upstream") + self._register_event_handler("on_frame_reached_downstream") + @property def params(self) -> PipelineParams: """Returns the pipeline parameters of this task.""" @@ -356,6 +372,9 @@ class PipelineTask(BaseTask): """ while True: frame = await self._up_queue.get() + + await self._call_event_handler("on_frame_reached_upstream", frame) + if isinstance(frame, EndTaskFrame): # Tell the task we should end nicely. await self.queue_frame(EndFrame()) @@ -383,6 +402,9 @@ class PipelineTask(BaseTask): """ while True: frame = await self._down_queue.get() + + await self._call_event_handler("on_frame_reached_downstream", frame) + if isinstance(frame, (EndFrame, StopFrame)): self._pipeline_end_event.set() elif isinstance(frame, HeartbeatFrame): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 07abef48e..a375812b2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -12,7 +12,7 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.filters.identity_filter import IdentityFilter -from pipecat.processors.frame_processor import FrameProcessor +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.tests.utils import HeartbeatsObserver, run_test @@ -94,6 +94,40 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.run() assert task.has_finished() + async def test_task_event_handlers(self): + upstream_received = False + downstream_received = False + + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline) + task.set_event_loop(asyncio.get_event_loop()) + + @task.event_handler("on_frame_reached_upstream") + async def on_frame_reached_upstream(task, frame): + nonlocal upstream_received + if isinstance(frame, TextFrame) and frame.text == "Hello Upstream!": + upstream_received = True + + @task.event_handler("on_frame_reached_downstream") + async def on_frame_reached_downstream(task, frame): + nonlocal downstream_received + if isinstance(frame, TextFrame) and frame.text == "Hello Downstream!": + downstream_received = True + await identity.push_frame( + TextFrame(text="Hello Upstream!"), FrameDirection.UPSTREAM + ) + + await task.queue_frame(TextFrame(text="Hello Downstream!")) + + try: + await asyncio.wait_for(task.run(), timeout=1.0) + except asyncio.TimeoutError: + pass + + assert upstream_received + assert downstream_received + async def test_task_heartbeats(self): heartbeats_counter = 0 From 87004937be423c741741acef2a2085cb467418f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 10:47:47 -0700 Subject: [PATCH 289/427] services(ultravox): CHANGELOG, formatting and minor changes --- CHANGELOG.md | 3 + README.md | 4 +- .../07u-interruptible-ultravox.py | 15 +- pyproject.toml | 1 + src/pipecat/services/ultravox.py | 208 +++++++++--------- 5 files changed, 120 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 776041481..deda5de9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `UltravoxSTTService`. + (see https://github.com/fixie-ai/ultravox) + - Added `on_frame_reached_upstream` and `on_frame_reached_downstream` event handlers to `PipelineTask`. Those events will be called when a frame reaches the beginning or end of the pipeline respectively. diff --git a/README.md b/README.md index ec09c8f72..599412673 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ pip install "pipecat-ai[option,...]" ### Available services | Category | Services | Install Command Example | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------| +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | | Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index 195eaf197..3ae4540f0 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -17,9 +17,9 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.services.ultravox import UltravoxSTTService from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.ultravox import UltravoxSTTService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -30,13 +30,14 @@ load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") -#Want to initialize the ultravox processor since it takes time to load the model and dont -#want to load it every time the pipeline is run +# Want to initialize the ultravox processor since it takes time to load the model and dont +# want to load it every time the pipeline is run ultravox_processor = UltravoxSTTService( model_size="fixie-ai/ultravox-v0_4_1-llama-3_1-8b", hf_token=os.getenv("HF_TOKEN"), ) + async def main(): async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) @@ -56,11 +57,9 @@ async def main(): tts = CartesiaTTSService( api_key=os.environ.get("CARTESIA_API_KEY"), - voice_id='97f4b8fb-f2fe-444b-bb9a-c109783a857a', - + voice_id="97f4b8fb-f2fe-444b-bb9a-c109783a857a", ) - pipeline = Pipeline( [ transport.input(), # Transport user input @@ -89,5 +88,3 @@ async def main(): if __name__ == "__main__": asyncio.run(main()) - - diff --git a/pyproject.toml b/pyproject.toml index dd58b1bfe..eab7b60f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] together = [] +ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] whisper = [ "faster-whisper~=1.1.1" ] openrouter = [] diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index f39dc348f..40029e673 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -1,132 +1,140 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + """This module implements Ultravox speech-to-text with a locally-loaded model.""" import json -import time import os +import time +from typing import AsyncGenerator, List, Optional + import numpy as np -from enum import Enum -from typing import AsyncGenerator, Optional, List -from loguru import logger -from pydantic import BaseModel from huggingface_hub import login +from loguru import logger from pipecat.frames.frames import ( - Frame, AudioRawFrame, - TranscriptionFrame, - TextFrame, - StartFrame, - EndFrame, CancelFrame, + EndFrame, + ErrorFrame, + Frame, + StartFrame, + TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, - ErrorFrame ) -from pipecat.services.ai_services import AIService from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import AIService from pipecat.utils.time import time_now_iso8601 try: - from vllm import SamplingParams, AsyncLLMEngine - from vllm.engine.arg_utils import AsyncEngineArgs from transformers import AutoTokenizer + from vllm import AsyncLLMEngine, SamplingParams + from vllm.engine.arg_utils import AsyncEngineArgs except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use Ultravox, you need to `pip install pipecat-ai[ultravox]`.") raise Exception(f"Missing module: {e}") + class AudioBuffer: """Buffer to collect audio frames before processing. - + Attributes: frames: List of AudioRawFrames to process started_at: Timestamp when speech started is_processing: Flag to prevent concurrent processing """ + def __init__(self): self.frames: List[AudioRawFrame] = [] self.started_at: Optional[float] = None self.is_processing: bool = False + class UltravoxModel: """Model wrapper for the Ultravox multimodal model. - + This class handles loading and running the Ultravox model for speech-to-text. - + Args: model_name: The name or path of the Ultravox model to load - + Attributes: model_name: The name of the loaded model engine: The vLLM engine for model inference tokenizer: The tokenizer for the model stop_token_ids: Optional token IDs to stop generation """ + def __init__(self, model_name: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b"): self.model_name = model_name self._initialize_engine() self._initialize_tokenizer() self.stop_token_ids = None - + def _initialize_engine(self): """Initialize the vLLM engine for inference.""" engine_args = AsyncEngineArgs( model=self.model_name, gpu_memory_utilization=0.9, max_model_len=8192, - trust_remote_code=True + trust_remote_code=True, ) self.engine = AsyncLLMEngine.from_engine_args(engine_args) - + def _initialize_tokenizer(self): """Initialize the tokenizer for the model.""" self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) - + def format_prompt(self, messages: list): """Format chat messages into a prompt for the model. - + Args: messages: List of message dictionaries with 'role' and 'content' - + Returns: str: Formatted prompt string """ return self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True + messages, tokenize=False, add_generation_prompt=True ) - async def generate(self, messages: list, temperature: float = 0.7, max_tokens: int = 100, audio: np.ndarray = None): + async def generate( + self, + messages: list, + temperature: float = 0.7, + max_tokens: int = 100, + audio: np.ndarray = None, + ): """Generate text from audio input using the model. - + Args: messages: List of message dictionaries temperature: Sampling temperature max_tokens: Maximum tokens to generate audio: Audio data as numpy array - + Yields: str: JSON chunks of the generated response """ sampling_params = SamplingParams( - temperature=temperature, - max_tokens=max_tokens, - stop_token_ids=self.stop_token_ids + temperature=temperature, max_tokens=max_tokens, stop_token_ids=self.stop_token_ids ) - - mm_data = { - "audio": audio - } + + mm_data = {"audio": audio} inputs = {"prompt": self.format_prompt(messages), "multi_modal_data": mm_data} results_generator = self.engine.generate(inputs, sampling_params, str(time.time())) - + previous_text = "" first_chunk = True async for output in results_generator: prompt_output = output.outputs - new_text = prompt_output[0].text[len(previous_text):] + new_text = prompt_output[0].text[len(previous_text) :] previous_text = prompt_output[0].text # Construct OpenAI-compatible chunk @@ -160,19 +168,20 @@ class UltravoxModel: yield json.dumps(chunk) + class UltravoxSTTService(AIService): """Service to transcribe audio using the Ultravox multimodal model. - + This service collects audio frames and processes them with Ultravox to generate text transcriptions. - + Args: model_size: The Ultravox model to use (ModelSize enum or string) hf_token: Hugging Face token for model access temperature: Sampling temperature for generation max_tokens: Maximum tokens to generate **kwargs: Additional arguments passed to AIService - + Attributes: model: The UltravoxModel instance buffer: Buffer to collect audio frames @@ -180,17 +189,18 @@ class UltravoxSTTService(AIService): max_tokens: Maximum tokens to generate _connection_active: Flag indicating if service is active """ + def __init__( self, *, - model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b", + model_size: str = "fixie-ai/ultravox-v0_4_1-llama-3_1-8b", hf_token: Optional[str] = None, temperature: float = 0.7, max_tokens: int = 100, **kwargs, ): super().__init__(**kwargs) - + # Authenticate with Hugging Face if token provided if hf_token: login(token=hf_token) @@ -198,22 +208,22 @@ class UltravoxSTTService(AIService): login(token=os.environ.get("HF_TOKEN")) else: logger.warning("No Hugging Face token provided. Model may not load correctly.") - + # Initialize model model_name = model_size if isinstance(model_size, str) else model_size.value - self.model = UltravoxModel(model_name=model_name) - + self._model = UltravoxModel(model_name=model_name) + # Initialize service state - self.buffer = AudioBuffer() - self.temperature = temperature - self.max_tokens = max_tokens + self._buffer = AudioBuffer() + self._temperature = temperature + self._max_tokens = max_tokens self._connection_active = False - + logger.info(f"Initialized UltravoxSTTService with model: {model_name}") - + def can_generate_metrics(self) -> bool: """Indicates whether this service can generate metrics. - + Returns: bool: True, as this service supports metric generation. """ @@ -221,7 +231,7 @@ class UltravoxSTTService(AIService): async def start(self, frame: StartFrame): """Handle service start. - + Args: frame: StartFrame that triggered this method """ @@ -231,7 +241,7 @@ class UltravoxSTTService(AIService): async def stop(self, frame: EndFrame): """Handle service stop. - + Args: frame: EndFrame that triggered this method """ @@ -241,20 +251,20 @@ class UltravoxSTTService(AIService): async def cancel(self, frame: CancelFrame): """Handle service cancellation. - + Args: frame: CancelFrame that triggered this method """ await super().cancel(frame) self._connection_active = False - self.buffer = AudioBuffer() + self._buffer = AudioBuffer() logger.info("UltravoxSTTService cancelled") async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames. - + This method collects audio frames and processes them when speech ends. - + Args: frame: The frame to process direction: Direction of the frame (input/output) @@ -263,44 +273,44 @@ class UltravoxSTTService(AIService): if isinstance(frame, UserStartedSpeakingFrame): logger.info("Speech started") - self.buffer = AudioBuffer() - self.buffer.started_at = time.time() - - elif isinstance(frame, AudioRawFrame) and self.buffer.started_at is not None: - self.buffer.frames.append(frame) - + self._buffer = AudioBuffer() + self._buffer.started_at = time.time() + + elif isinstance(frame, AudioRawFrame) and self._buffer.started_at is not None: + self._buffer.frames.append(frame) + elif isinstance(frame, UserStoppedSpeakingFrame): - if self.buffer.frames and not self.buffer.is_processing: + if self._buffer.frames and not self._buffer.is_processing: logger.info("Speech ended, processing buffer...") await self.process_generator(self._process_audio_buffer()) return # Return early to avoid pushing None frame - + # Only push the original frame if we haven't processed audio if frame is not None: await self.push_frame(frame, direction) async def _process_audio_buffer(self) -> AsyncGenerator[Frame, None]: """Process collected audio frames with Ultravox. - + This method concatenates audio frames, processes them with the model, and yields the resulting text frames. - + Yields: Frame: TextFrame containing the transcribed text """ try: - self.buffer.is_processing = True - + self._buffer.is_processing = True + # Check if we have valid frames before processing - if not self.buffer.frames: + if not self._buffer.frames: logger.warning("No audio frames to process") yield ErrorFrame("No audio frames to process") return - + # Process audio frames audio_arrays = [] - for f in self.buffer.frames: - if hasattr(f, 'audio') and f.audio: + for f in self._buffer.frames: + if hasattr(f, "audio") and f.audio: # Handle bytes data - these are int16 PCM samples if isinstance(f.audio, bytes): try: @@ -319,75 +329,73 @@ class UltravoxSTTService(AIService): audio_arrays.append(f.audio.astype(np.int16)) else: audio_arrays.append(f.audio) - + # Only proceed if we have valid audio arrays if not audio_arrays: logger.warning("No valid audio data found in frames") yield ErrorFrame("No valid audio data found in frames") return - + # Concatenate audio frames - all should be int16 now audio_data = np.concatenate(audio_arrays) - + # Generate text using the model - if self.model: + if self._model: try: logger.info("Generating text from audio using model...") full_response = "" - + # Start metrics tracking await self.start_ttfb_metrics() await self.start_processing_metrics() - - async for response in self.model.generate( - messages=[{ - 'role': 'user', - 'content': "<|audio|>\n" - }], - temperature=self.temperature, - max_tokens=self.max_tokens, - audio=audio_data + + async for response in self._model.generate( + messages=[{"role": "user", "content": "<|audio|>\n"}], + temperature=self._temperature, + max_tokens=self._max_tokens, + audio=audio_data, ): # Stop TTFB metrics after first response await self.stop_ttfb_metrics() - + chunk = json.loads(response) if "choices" in chunk and len(chunk["choices"]) > 0: delta = chunk["choices"][0]["delta"] if "content" in delta: new_text = delta["content"] full_response += new_text - + # Stop processing metrics after completion await self.stop_processing_metrics() - + logger.info(f"Generated text: {full_response}") - + # Create a transcription frame with the generated text transcription = full_response.strip() if transcription: yield TranscriptionFrame( + user_id="", text=transcription, - interim_text="", - timestamp=time_now_iso8601() + timestamp=time_now_iso8601(), ) else: logger.warning("Empty transcription result") yield ErrorFrame("Empty transcription result") - + except Exception as e: logger.error(f"Error generating text from model: {e}") yield ErrorFrame(f"Error generating text: {str(e)}") else: logger.warning("No model available for text generation") yield ErrorFrame("No model available for text generation") - + except Exception as e: logger.error(f"Error processing audio buffer: {e}") import traceback + logger.error(traceback.format_exc()) yield ErrorFrame(f"Error processing audio: {str(e)}") finally: - self.buffer.is_processing = False - self.buffer.frames = [] - self.buffer.started_at = None + self._buffer.is_processing = False + self._buffer.frames = [] + self._buffer.started_at = None From c9a31ea5135b0f381ba1995ff2e735ebab18b939 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Thu, 13 Mar 2025 14:35:47 -0700 Subject: [PATCH 290/427] fix for 26-gemini-multimodal-live.py --- examples/foundational/26-gemini-multimodal-live.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 71d41304b..c26d6ad2f 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -77,7 +77,7 @@ async def main(): LLMMessagesAppendFrame( messages=[ { - "role": "assistant", + "role": "user", "content": "Greet the user.", } ] From 6eb3a8409fe9b8bba378041e956bbe654e768dc0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 18:42:19 -0400 Subject: [PATCH 291/427] README: Add Parakeet and FastPitch --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 599412673..822c75aee 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ pip install "pipecat-ai[option,...]" ### Available services | Category | Services | Install Command Example | -|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------| -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | From 1b3b4ee04a837eb8a38506d28af6b50176319e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 18:39:11 -0700 Subject: [PATCH 292/427] PipelineTask: only call event handlers if a filter is matched --- CHANGELOG.md | 5 +++- src/pipecat/pipeline/task.py | 45 ++++++++++++++++++++++++++++++++---- tests/test_pipeline.py | 2 ++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deda5de9d..4feca2771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `on_frame_reached_upstream` and `on_frame_reached_downstream` event handlers to `PipelineTask`. Those events will be called when a frame reaches - the beginning or end of the pipeline respectively. + the beginning or end of the pipeline respectively. Note that by default, the + event handlers will not be called unless a filter is set with + `PipelineTask.set_reached_upstream_filter()` or + `PipelineTask.set_reached_downstream_filter()`. - Added support for Chirp voices in `GoogleTTSService`. diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index f20ef9d1f..c10b33189 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -5,7 +5,7 @@ # import asyncio -from typing import Any, AsyncIterable, Dict, Iterable, List, Optional +from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type from loguru import logger from pydantic import BaseModel, ConfigDict @@ -121,7 +121,9 @@ class PipelineTask(BaseTask): It has a couple of event handlers `on_frame_reached_upstream` and `on_frame_reached_downstream` that are called when upstream frames or - downstream frames reach both ends of pipeline. + downstream frames reach both ends of pipeline. By default, the events + handlers will not be called unless some filters are set using + `set_reached_upstream_filter` and `set_reached_downstream_filter`. @task.event_handler("on_frame_reached_upstream") async def on_frame_reached_upstream(task, frame): @@ -180,16 +182,35 @@ class PipelineTask(BaseTask): # StopFrame) has been received in the down queue. self._pipeline_end_event = asyncio.Event() + # This is a source processor that we connect to the provided + # pipeline. This source processor allows up to receive and react to + # upstream frames. self._source = PipelineTaskSource(self._up_queue) self._source.link(pipeline) + # This is a sink processor that we connect to the provided + # pipeline. This sink processor allows up to receive and react to + # downstream frames. self._sink = PipelineTaskSink(self._down_queue) pipeline.link(self._sink) + # This task maneger will handle all the asyncio tasks created by this + # PipelineTask and its frame processors. self._task_manager = task_manager or TaskManager() + # The task observer acts as a proxy to the provided observers. This way, + # we only need to pass a single observer (using the StartFrame) which + # then just acts as a proxy. self._observer = TaskObserver(observers=observers, task_manager=self._task_manager) + # These events can be used to check which frames make it to the source + # or sink processors. Instead of calling the event handlers for every + # frame the user needs to specify which events they are interested + # in. This is mainly for efficiency reason because each event handler + # creates a task and most likely you only care about one or two frame + # types. + self._reached_upstream_types: Tuple[Type[Frame], ...] = () + self._reached_downstream_types: Tuple[Type[Frame], ...] = () self._register_event_handler("on_frame_reached_upstream") self._register_event_handler("on_frame_reached_downstream") @@ -201,6 +222,20 @@ class PipelineTask(BaseTask): def set_event_loop(self, loop: asyncio.AbstractEventLoop): self._task_manager.set_event_loop(loop) + def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]): + """Sets which frames will be checked before calling the + on_frame_reached_upstream event handler. + + """ + self._reached_upstream_types = types + + def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]): + """Sets which frames will be checked before calling the + on_frame_reached_downstream event handler. + + """ + self._reached_downstream_types = types + def has_finished(self) -> bool: """Indicates whether the tasks has finished. That is, all processors have stopped. @@ -373,7 +408,8 @@ class PipelineTask(BaseTask): while True: frame = await self._up_queue.get() - await self._call_event_handler("on_frame_reached_upstream", frame) + if isinstance(frame, self._reached_upstream_types): + await self._call_event_handler("on_frame_reached_upstream", frame) if isinstance(frame, EndTaskFrame): # Tell the task we should end nicely. @@ -403,7 +439,8 @@ class PipelineTask(BaseTask): while True: frame = await self._down_queue.get() - await self._call_event_handler("on_frame_reached_downstream", frame) + if isinstance(frame, self._reached_downstream_types): + await self._call_event_handler("on_frame_reached_downstream", frame) if isinstance(frame, (EndFrame, StopFrame)): self._pipeline_end_event.set() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a375812b2..c3811a672 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -102,6 +102,8 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): pipeline = Pipeline([identity]) task = PipelineTask(pipeline) task.set_event_loop(asyncio.get_event_loop()) + task.set_reached_upstream_filter((TextFrame,)) + task.set_reached_downstream_filter((TextFrame,)) @task.event_handler("on_frame_reached_upstream") async def on_frame_reached_upstream(task, frame): From 7dec8431e16a6675d869c057ef7e4181a23a2ea3 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Fri, 14 Mar 2025 10:52:13 +0000 Subject: [PATCH 293/427] Review comments by aconchillo. --- pyproject.toml | 2 +- src/pipecat/services/neuphonic.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e698ab134..1b324c3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -neuphonic = [ "pyneuphonic~=1.5.11", "websockets~=13.1" ] +neuphonic = [ "pyneuphonic~=1.5.12", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index 757d60c3e..c0d35fb40 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -27,8 +27,7 @@ from pipecat.frames.frames import ( TTSStoppedFrame, ) from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.ai_services import TTSService, WordTTSService -from pipecat.services.websocket_service import WebsocketService +from pipecat.services.ai_services import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language # See .env.example for Neuphonic configuration needed @@ -71,7 +70,7 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: return result -class NeuphonicTTSService(WordTTSService, WebsocketService): +class NeuphonicTTSService(InterruptibleTTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -88,7 +87,7 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): params: InputParams = InputParams(), **kwargs, ): - WordTTSService.__init__( + super().__init__( self, aggregate_sentences=True, push_text_frames=False, @@ -97,7 +96,6 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): sample_rate=sample_rate, **kwargs, ) - WebsocketService.__init__(self) self._api_key = api_key self._url = url @@ -232,7 +230,6 @@ class NeuphonicTTSService(WordTTSService, WebsocketService): msg = json.loads(message) if msg.get("data", {}).get("audio") is not None: await self.stop_ttfb_metrics() - self.start_word_timestamps() audio = base64.b64decode(msg["data"]["audio"]) frame = TTSAudioRawFrame(audio, self.sample_rate, 1) From 11b13d053be82eca507f79284692cb8fb1d4ecb2 Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Fri, 14 Mar 2025 11:17:22 +0000 Subject: [PATCH 294/427] Fixed a bug from previous commit. Removed the concept of model from Neuphonic. --- pyproject.toml | 2 +- src/pipecat/services/neuphonic.py | 25 +------------------------ 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b324c3a5..f87aa7572 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ aws = [ "boto3~=1.35.99" ] azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] -neuphonic = [ "pyneuphonic~=1.5.12", "websockets~=13.1" ] +neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index c0d35fb40..a2ea98f42 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -41,12 +41,6 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") -# Models that support language codes -NEUPHONIC_MULTILINGUAL_MODELS = { - "neu_fast", - "neu_hq", -} - def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: BASE_LANGUAGES = { @@ -80,7 +74,6 @@ class NeuphonicTTSService(InterruptibleTTSService): *, api_key: str, voice_id: Optional[str] = None, - model: str = "neu_hq", url: str = "wss://api.neuphonic.com", sample_rate: Optional[int] = 22050, encoding: str = "pcm_linear", @@ -88,7 +81,6 @@ class NeuphonicTTSService(InterruptibleTTSService): **kwargs, ): super().__init__( - self, aggregate_sentences=True, push_text_frames=False, push_stop_frames=True, @@ -105,7 +97,6 @@ class NeuphonicTTSService(InterruptibleTTSService): "encoding": encoding, "sampling_rate": sample_rate, } - self.set_model_name(model) self.set_voice(voice_id) # Indicates if we have sent TTSStartedFrame. It will reset to False when @@ -119,12 +110,6 @@ class NeuphonicTTSService(InterruptibleTTSService): def language_to_service_language(self, language: Language) -> Optional[str]: return language_to_neuphonic_lang_code(language) - async def set_model(self, model: str): - await super().set_model(model) - logger.info(f"Switching TTS model to: [{model}]") - await self._disconnect() - await self._connect() - async def _update_settings(self, settings: Mapping[str, Any]): if "voice_id" in settings: self.set_voice(settings["voice_id"]) @@ -155,8 +140,6 @@ class NeuphonicTTSService(InterruptibleTTSService): await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False - if isinstance(frame, TTSStoppedFrame): - await self.add_word_timestamps([("LLMFullResponseEndFrame", 0), ("Reset", 0)]) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -195,7 +178,6 @@ class NeuphonicTTSService(InterruptibleTTSService): tts_config = { **self._settings, "voice_id": self._voice_id, - "model": self.model_name, } query_params = [f"api_key={self._api_key}"] @@ -279,7 +261,6 @@ class NeuphonicHttpTTSService(TTSService): Args: api_key: Neuphonic API key voice_id: ID of the voice to use - model: Neuphonic model to use (default: "neu_hq") url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com") sample_rate: Sample rate for audio output (default: 22050Hz) encoding: Audio encoding format (default: "pcm_linear") @@ -296,7 +277,6 @@ class NeuphonicHttpTTSService(TTSService): *, api_key: str, voice_id: Optional[str] = None, - model: str = "neu_hq", url: str = "https://api.neuphonic.com", sample_rate: Optional[int] = 22050, encoding: str = "pcm_linear", @@ -313,7 +293,6 @@ class NeuphonicHttpTTSService(TTSService): "encoding": encoding, "sampling_rate": sample_rate, } - self.set_model_name(model) self.set_voice(voice_id) def can_generate_metrics(self) -> bool: @@ -341,9 +320,7 @@ class NeuphonicHttpTTSService(TTSService): try: await self.start_ttfb_metrics() - response = sse.send( - text, TTSConfig(**self._settings, model=self.model_name, voice_id=self._voice_id) - ) + response = sse.send(text, TTSConfig(**self._settings, voice_id=self._voice_id)) await self.start_tts_usage_metrics(text) yield TTSStartedFrame() From f8610a69a5a9af67d8228de09630650bb9ca07f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 19:41:33 -0700 Subject: [PATCH 295/427] introduce text aggregators --- CHANGELOG.md | 5 ++ src/pipecat/services/ai_services.py | 20 +++---- .../utils/text/base_text_aggregator.py | 57 +++++++++++++++++++ .../utils/text/simple_text_aggregator.py | 42 ++++++++++++++ tests/test_simple_text_aggregator.py | 29 ++++++++++ 5 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 src/pipecat/utils/text/base_text_aggregator.py create mode 100644 src/pipecat/utils/text/simple_text_aggregator.py create mode 100644 tests/test_simple_text_aggregator.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4feca2771..a48fa9c3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `BaseTextAggregator`. Text aggregators are used by the TTS service + to aggregate LLM tokens and decide when the aggregated text should be pushed + to the TTS service. It also allows for the text to be manipulated while it's + being aggregated. + - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 8533a23fc..1701f9829 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -45,8 +45,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language -from pipecat.utils.string import match_endofsentence +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.base_text_filter import BaseTextFilter +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator from pipecat.utils.time import seconds_to_nanoseconds @@ -237,6 +238,9 @@ class TTSService(AIService): pause_frame_processing: bool = False, # TTS output sample rate sample_rate: Optional[int] = None, + # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. + text_aggregator: Optional[BaseTextAggregator] = None, + # Text filter executed after text has been aggregated. text_filter: Optional[BaseTextFilter] = None, **kwargs, ): @@ -252,12 +256,12 @@ class TTSService(AIService): self._sample_rate = 0 self._voice_id: str = "" self._settings: Dict[str, Any] = {} + self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() self._text_filter: Optional[BaseTextFilter] = text_filter self._stop_frame_task: Optional[asyncio.Task] = None self._stop_frame_queue: asyncio.Queue = asyncio.Queue() - self._current_sentence: str = "" self._processing_text: bool = False @property @@ -336,8 +340,8 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._current_sentence - self._current_sentence = "" + sentence = self._text_aggregator.text + self._text_aggregator.reset() self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): @@ -382,8 +386,8 @@ class TTSService(AIService): await self._stop_frame_queue.put(frame) async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): - self._current_sentence = "" self._processing_text = False + self._text_aggregator.handle_interruption() if self._text_filter: self._text_filter.handle_interruption() @@ -400,11 +404,7 @@ class TTSService(AIService): if not self._aggregate_sentences: text = frame.text else: - self._current_sentence += frame.text - eos_end_marker = match_endofsentence(self._current_sentence) - if eos_end_marker: - text = self._current_sentence[:eos_end_marker] - self._current_sentence = self._current_sentence[eos_end_marker:] + text = self._text_aggregator.aggregate(frame.text) if text: await self._push_tts_frames(text) diff --git a/src/pipecat/utils/text/base_text_aggregator.py b/src/pipecat/utils/text/base_text_aggregator.py new file mode 100644 index 000000000..452e5598e --- /dev/null +++ b/src/pipecat/utils/text/base_text_aggregator.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from abc import ABC, abstractmethod +from typing import Optional + + +class BaseTextAggregator(ABC): + """This is the base class for text aggregators. Text aggregators are usually + used by the TTS service to aggregate LLM tokens and decide when the + aggregated text should be pushed to the TTS service. + + Text aggregators can also be used to manipulate text while it's being + aggregated (e.g. reasoning blocks can be removed). + + """ + + @property + @abstractmethod + def text(self) -> str: + """Returns the currently aggregated text.""" + pass + + @abstractmethod + def aggregate(self, text: str) -> Optional[str]: + """Aggregates the specified text with the currently accumulated text. + + This method should be implemented to define how the new text contributes + to the aggregation process. It returns the updated aggregated text if + it's ready to be processed, or None otherwise. + + Args: + text (str): The text to be aggregated. + + Returns: + Optional[str]: The updated aggregated text or None if aggregated + text is not ready. + + """ + pass + + @abstractmethod + def handle_interruption(self): + """Handles interruptions. When an interruption occurs it is possible + that we might want to discard the aggregated text or do some internal + modifications to the aggregated text. + + """ + pass + + @abstractmethod + def reset(self): + """Clears the internally aggregated text.""" + pass diff --git a/src/pipecat/utils/text/simple_text_aggregator.py b/src/pipecat/utils/text/simple_text_aggregator.py new file mode 100644 index 000000000..9022fc25a --- /dev/null +++ b/src/pipecat/utils/text/simple_text_aggregator.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Optional + +from pipecat.utils.string import match_endofsentence +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator + + +class SimpleTextAggregator(BaseTextAggregator): + """This is a simple text aggregator. It aggregates text until an end of + sentence is found. + + """ + + def __init__(self): + self._text = "" + + @property + def text(self) -> str: + return self._text + + def aggregate(self, text: str) -> Optional[str]: + result: Optional[str] = None + + self._text += text + + eos_end_marker = match_endofsentence(self._text) + if eos_end_marker: + result = self._text[:eos_end_marker] + self._text = self._text[eos_end_marker:] + + return result + + def handle_interruption(self): + self._text = "" + + def reset(self): + self._text = "" diff --git a/tests/test_simple_text_aggregator.py b/tests/test_simple_text_aggregator.py new file mode 100644 index 000000000..10c4c6a88 --- /dev/null +++ b/tests/test_simple_text_aggregator.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator + + +class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.aggregator = SimpleTextAggregator() + + async def test_reset_aggregations(self): + assert self.aggregator.aggregate("Hello ") == None + assert self.aggregator.text == "Hello " + self.aggregator.reset() + assert self.aggregator.text == "" + + async def test_simple_sentence(self): + assert self.aggregator.aggregate("Hello ") == None + assert self.aggregator.aggregate("Pipecat!") == "Hello Pipecat!" + assert self.aggregator.text == "" + + async def test_multiple_sentences(self): + assert self.aggregator.aggregate("Hello Pipecat! How are ") == "Hello Pipecat!" + assert self.aggregator.aggregate("you?") == " How are you?" From b632d71465a8821e541b3fcc0ba1deaf13488b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 19:41:47 -0700 Subject: [PATCH 296/427] TTSService: flush_audio() should be in the base class --- src/pipecat/services/ai_services.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 1701f9829..3fe33d69e 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -285,6 +285,9 @@ class TTSService(AIService): async def update_setting(self, key: str, value: Any): pass + async def flush_audio(self): + pass + async def start(self, frame: StartFrame): await super().start(frame) self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate @@ -535,9 +538,6 @@ class WebsocketTTSService(TTSService, WebsocketService): TTSService.__init__(self, **kwargs) WebsocketService.__init__(self) - async def flush_audio(self): - pass - class InterruptibleTTSService(WebsocketTTSService): """This is a base class for websocket-based TTS services that don't support From 16d7df1c9f6bd8e9128346dd1091943c1ce464e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 18:53:24 -0700 Subject: [PATCH 297/427] removed most deprecations --- CHANGELOG.md | 16 + src/pipecat/audio/utils.py | 17 -- .../processors/filters/stt_mute_filter.py | 13 +- src/pipecat/processors/frameworks/rtvi.py | 273 ------------------ src/pipecat/services/aws.py | 13 - src/pipecat/transports/services/daily.py | 15 +- src/pipecat/vad/__init__.py | 0 src/pipecat/vad/silero.py | 16 - src/pipecat/vad/vad_analyzer.py | 15 - 9 files changed, 18 insertions(+), 360 deletions(-) delete mode 100644 src/pipecat/vad/__init__.py delete mode 100644 src/pipecat/vad/silero.py delete mode 100644 src/pipecat/vad/vad_analyzer.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a48fa9c3b..3d56a323f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. +### Removed + +- Removed deprecated `audio.resample_audio()`, use `create_default_resampler()` + instead. + +- Removed deprecated`stt_service` parameter from `STTMuteFilter`. + +- Removed deprecated RTVI processors, use an `RTVIObserver` instead. + +- Removed deprecated `AWSTTSService`, use `PollyTTSService` instead. + +- Removed deprecated field `tier` from `DailyTranscriptionSettings`, use `model` + instead. + +- Removed deprecated `pipecat.vad` package, use `pipecat.audio.vad` instead. + ### Fixed - Fixed an issue in `RimeTTSService` where the last line of text sent didn't diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 7c4c25fcc..1f7db648f 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -18,23 +18,6 @@ def create_default_resampler(**kwargs) -> BaseAudioResampler: return SOXRAudioResampler(**kwargs) -def resample_audio(audio: bytes, original_rate: int, target_rate: int) -> bytes: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'resample_audio()' is deprecated, use 'create_default_resampler()' instead.", - DeprecationWarning, - ) - - if original_rate == target_rate: - return audio - audio_data = np.frombuffer(audio, dtype=np.int16) - resampled_audio = soxr.resample(audio_data, original_rate, target_rate) - return resampled_audio.astype(np.int16).tobytes() - - def mix_audio(audio1: bytes, audio2: bytes) -> bytes: data1 = np.frombuffer(audio1, dtype=np.int16) data2 = np.frombuffer(audio2, dtype=np.int16) diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index c74e495ab..cce087f22 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -92,20 +92,9 @@ class STTMuteFilter(FrameProcessor): **kwargs: Additional arguments passed to parent class """ - def __init__( - self, *, config: STTMuteConfig, stt_service: Optional[FrameProcessor] = None, **kwargs - ): + def __init__(self, *, config: STTMuteConfig, **kwargs): super().__init__(**kwargs) self._config = config - if stt_service is not None: - import warnings - - warnings.warn( - "The stt_service parameter is deprecated and will be removed in a future version. " - "STTMuteFilter now manages mute state internally.", - DeprecationWarning, - stacklevel=2, - ) self._first_speech_handled = False self._bot_is_speaking = False self._function_call_in_progress = False diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index dac903862..29747a582 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -391,267 +391,6 @@ class RTVIServerMessageFrame(SystemFrame): return f"{self.name}(data: {self.data})" -class RTVIFrameProcessor(FrameProcessor): - def __init__(self, direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs): - super().__init__(**kwargs) - self._direction = direction - - async def _push_transport_message_urgent(self, model: BaseModel, exclude_none: bool = True): - frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) - await self.push_frame(frame, self._direction) - - -class RTVISpeakingProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVISpeakingProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): - await self._handle_interruptions(frame) - elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)): - await self._handle_bot_speaking(frame) - - async def _handle_interruptions(self, frame: Frame): - message = None - if isinstance(frame, UserStartedSpeakingFrame): - message = RTVIUserStartedSpeakingMessage() - elif isinstance(frame, UserStoppedSpeakingFrame): - message = RTVIUserStoppedSpeakingMessage() - - if message: - await self._push_transport_message_urgent(message) - - async def _handle_bot_speaking(self, frame: Frame): - message = None - if isinstance(frame, BotStartedSpeakingFrame): - message = RTVIBotStartedSpeakingMessage() - elif isinstance(frame, BotStoppedSpeakingFrame): - message = RTVIBotStoppedSpeakingMessage() - - if message: - await self._push_transport_message_urgent(message) - - -class RTVIUserTranscriptionProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIUserTranscriptionProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)): - await self._handle_user_transcriptions(frame) - - async def _handle_user_transcriptions(self, frame: Frame): - message = None - if isinstance(frame, TranscriptionFrame): - message = RTVIUserTranscriptionMessage( - data=RTVIUserTranscriptionMessageData( - text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=True - ) - ) - elif isinstance(frame, InterimTranscriptionFrame): - message = RTVIUserTranscriptionMessage( - data=RTVIUserTranscriptionMessageData( - text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=False - ) - ) - - if message: - await self._push_transport_message_urgent(message) - - -class RTVIUserLLMTextProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIUserLLMTextProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, OpenAILLMContextFrame): - await self._handle_context(frame) - - async def _handle_context(self, frame: OpenAILLMContextFrame): - messages = frame.context.messages - if len(messages) > 0: - message = messages[-1] - if message["role"] == "user": - content = message["content"] - if isinstance(content, list): - text = " ".join(item["text"] for item in content if "text" in item) - else: - text = content - rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text)) - await self._push_transport_message_urgent(rtvi_message) - - -class RTVIBotTranscriptionProcessor(RTVIFrameProcessor): - def __init__(self): - super().__init__() - self._aggregation = "" - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIBotTranscriptionProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, UserStartedSpeakingFrame): - await self._push_aggregation() - elif isinstance(frame, LLMTextFrame): - self._aggregation += frame.text - if match_endofsentence(self._aggregation): - await self._push_aggregation() - - async def _push_aggregation(self): - if len(self._aggregation) > 0: - message = RTVIBotTranscriptionMessage(data=RTVITextMessageData(text=self._aggregation)) - await self._push_transport_message_urgent(message) - self._aggregation = "" - - -class RTVIBotLLMProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIBotLLMProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, LLMFullResponseStartFrame): - await self._push_transport_message_urgent(RTVIBotLLMStartedMessage()) - elif isinstance(frame, LLMFullResponseEndFrame): - await self._push_transport_message_urgent(RTVIBotLLMStoppedMessage()) - elif isinstance(frame, LLMTextFrame): - message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) - await self._push_transport_message_urgent(message) - - -class RTVIBotTTSProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIBotTTSProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, TTSStartedFrame): - await self._push_transport_message_urgent(RTVIBotTTSStartedMessage()) - elif isinstance(frame, TTSStoppedFrame): - await self._push_transport_message_urgent(RTVIBotTTSStoppedMessage()) - elif isinstance(frame, TTSTextFrame): - message = RTVIBotTTSTextMessage(data=RTVITextMessageData(text=frame.text)) - await self._push_transport_message_urgent(message) - - -class RTVIMetricsProcessor(RTVIFrameProcessor): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVIMetricsProcessor' is deprecated, use an 'RTVIObserver' instead.", - DeprecationWarning, - ) - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - await self.push_frame(frame, direction) - - if isinstance(frame, MetricsFrame): - await self._handle_metrics(frame) - - async def _handle_metrics(self, frame: MetricsFrame): - metrics = {} - for d in frame.data: - if isinstance(d, TTFBMetricsData): - if "ttfb" not in metrics: - metrics["ttfb"] = [] - metrics["ttfb"].append(d.model_dump(exclude_none=True)) - elif isinstance(d, ProcessingMetricsData): - if "processing" not in metrics: - metrics["processing"] = [] - metrics["processing"].append(d.model_dump(exclude_none=True)) - elif isinstance(d, LLMUsageMetricsData): - if "tokens" not in metrics: - metrics["tokens"] = [] - metrics["tokens"].append(d.value.model_dump(exclude_none=True)) - elif isinstance(d, TTSUsageMetricsData): - if "characters" not in metrics: - metrics["characters"] = [] - metrics["characters"].append(d.model_dump(exclude_none=True)) - - message = RTVIMetricsMessage(data=metrics) - await self._push_transport_message_urgent(message) - - class RTVIObserver(BaseObserver): """Pipeline frame observer for RTVI server message handling. @@ -876,18 +615,6 @@ class RTVIProcessor(FrameProcessor): self._input_transport = input_transport self._input_transport.enable_audio_in_stream_on_start(False) - def observer(self) -> RTVIObserver: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'RTVI.observer()' is deprecated, instantiate an 'RTVIObserver' directly instead.", - DeprecationWarning, - ) - - return RTVIObserver(self) - def register_action(self, action: RTVIAction): id = self._action_id(action.service, action.action) self._registered_actions[id] = action diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index b83df2f77..4c0417d72 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -248,16 +248,3 @@ class PollyTTSService(TTSService): finally: yield TTSStoppedFrame() - - -class AWSTTSService(PollyTTSService): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "'AWSTTSService' is deprecated, use 'PollyTTSService' instead.", DeprecationWarning - ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7735ae91f..873b13cd1 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -6,7 +6,6 @@ import asyncio import time -import warnings from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from typing import Any, Awaitable, Callable, Mapping, Optional @@ -18,7 +17,7 @@ from daily import ( VirtualSpeakerDevice, ) from loguru import logger -from pydantic import BaseModel, model_validator +from pydantic import BaseModel from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams from pipecat.frames.frames import ( @@ -124,7 +123,6 @@ class DailyTranscriptionSettings(BaseModel): Attributes: language: ISO language code for transcription (e.g. "en"). - tier: Deprecated. Use model instead. model: Transcription model to use (e.g. "nova-2-general"). profanity_filter: Whether to filter profanity from transcripts. redact: Whether to redact sensitive information. @@ -135,7 +133,6 @@ class DailyTranscriptionSettings(BaseModel): """ language: str = "en" - tier: Optional[str] = None model: str = "nova-2-general" profanity_filter: bool = True redact: bool = False @@ -144,16 +141,6 @@ class DailyTranscriptionSettings(BaseModel): includeRawResponse: bool = True extra: Mapping[str, Any] = {"interim_results": True} - @model_validator(mode="before") - def check_deprecated_fields(cls, values): - with warnings.catch_warnings(): - warnings.simplefilter("always") - if "tier" in values: - warnings.warn( - "Field 'tier' is deprecated, use 'model' instead.", DeprecationWarning - ) - return values - class DailyParams(TransportParams): """Configuration parameters for Daily transport. diff --git a/src/pipecat/vad/__init__.py b/src/pipecat/vad/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/pipecat/vad/silero.py b/src/pipecat/vad/silero.py deleted file mode 100644 index 285e2e0e8..000000000 --- a/src/pipecat/vad/silero.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Package `pipecat.vad` is deprecated, use `pipecat.audio.vad` instead", DeprecationWarning - ) - -from ..audio.vad.silero import SileroVADAnalyzer -from ..processors.audio.vad.silero import SileroVAD diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py deleted file mode 100644 index 7c8fc4c31..000000000 --- a/src/pipecat/vad/vad_analyzer.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("always") - warnings.warn( - "Package `pipecat.vad` is deprecated, use `pipecat.audio.vad` instead", DeprecationWarning - ) - -from ..audio.vad.vad_analyzer import VADAnalyzer, VADParams, VADState From 24220f38f0880ea6f6e955d590acf5ae5bc675c9 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 7 Mar 2025 16:43:10 -0500 Subject: [PATCH 298/427] Add a Pipecat Cloud deployment example --- CHANGELOG.md | 4 + .../pipecat-cloud-example/.gitignore | 94 +++++++++ .../pipecat-cloud-example/Dockerfile | 7 + .../pipecat-cloud-example/README.md | 191 ++++++++++++++++++ .../deployment/pipecat-cloud-example/bot.py | 167 +++++++++++++++ .../pipecat-cloud-example/env.example | 2 + .../pipecat-cloud-example/local_runner.py | 46 +++++ .../pipecat-cloud-example/pcc-deploy.toml | 6 + .../pipecat-cloud-example/requirements.txt | 2 + 9 files changed, 519 insertions(+) create mode 100644 examples/deployment/pipecat-cloud-example/.gitignore create mode 100644 examples/deployment/pipecat-cloud-example/Dockerfile create mode 100644 examples/deployment/pipecat-cloud-example/README.md create mode 100644 examples/deployment/pipecat-cloud-example/bot.py create mode 100644 examples/deployment/pipecat-cloud-example/env.example create mode 100644 examples/deployment/pipecat-cloud-example/local_runner.py create mode 100644 examples/deployment/pipecat-cloud-example/pcc-deploy.toml create mode 100644 examples/deployment/pipecat-cloud-example/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d56a323f..e47f1010a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. +### Other + +- Added a Pipecat Cloud deployment example to the `examples` directory. + ## [0.0.58] - 2025-02-26 ### Added diff --git a/examples/deployment/pipecat-cloud-example/.gitignore b/examples/deployment/pipecat-cloud-example/.gitignore new file mode 100644 index 000000000..8a5b4de3d --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/.gitignore @@ -0,0 +1,94 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +dist/ +*.egg-info/ +*.egg +.installed.cfg +.eggs/ +downloads/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +MANIFEST + +# Virtual Environments +venv/ +env/ +.env +.venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.idea/ +.vscode/ +.spyderproject +.spyproject +.ropeproject + +# Testing and Coverage +.coverage +.coverage.* +htmlcov/ +.pytest_cache/ +.tox/ +.nox/ +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +cover/ + +# Logs and Databases +*.log +*.db +db.sqlite3 +db.sqlite3-journal +pip-log.txt + +# System Files +.DS_Store +Thumbs.db +desktop.ini +*.swp +*.swo +*.bak +*.tmp +*~ + +# Build and Documentation +docs/_build/ +.pybuilder/ +target/ +instance/ +.webassets-cache +.pdm.toml +.pdm-python +.pdm-build/ +__pypackages__/ + +# Other +*.mo +*.pot +*.sage.py +.mypy_cache/ +.dmypy.json +dmypy.json +.pyre/ +.pytype/ +cython_debug/ +.ipynb_checkpoints + +# Pipecat cloud +.pcc-deploy.toml \ No newline at end of file diff --git a/examples/deployment/pipecat-cloud-example/Dockerfile b/examples/deployment/pipecat-cloud-example/Dockerfile new file mode 100644 index 000000000..290899e5c --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/Dockerfile @@ -0,0 +1,7 @@ +FROM dailyco/pipecat-base:latest + +COPY ./requirements.txt requirements.txt + +RUN pip install --no-cache-dir --upgrade -r requirements.txt + +COPY ./bot.py bot.py \ No newline at end of file diff --git a/examples/deployment/pipecat-cloud-example/README.md b/examples/deployment/pipecat-cloud-example/README.md new file mode 100644 index 000000000..1950a0f94 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/README.md @@ -0,0 +1,191 @@ +# Pipecat Cloud Starter Project + +[![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.daily.co) [![Discord](https://img.shields.io/discord/1217145424381743145)](https://discord.gg/dailyco) + +A template voice agent for [Pipecat Cloud](https://www.daily.co/products/pipecat-cloud/) that demonstrates building and deploying a conversational AI agent. + +## Prerequisites + +- Python 3.10+ +- Linux, MacOS, or Windows Subsystem for Linux (WSL) +- [Docker](https://www.docker.com) and a Docker repository (e.g., [Docker Hub](https://hub.docker.com)) +- A Docker Hub account (or other container registry account) +- [Pipecat Cloud](https://pipecat.daily.co) account + +> **Note**: If you haven't installed Docker yet, follow the official installation guides for your platform ([Linux](https://docs.docker.com/engine/install/), [Mac](https://docs.docker.com/desktop/setup/install/mac-install/), [Windows](https://docs.docker.com/desktop/setup/install/windows-install/)). For Docker Hub, [create a free account](https://hub.docker.com/signup) and log in via terminal with `docker login`. + +## Getting Started + +### 1. Set up Python environment + +We recommend using a virtual environment to manage your Python dependencies. + +```bash +# Create a virtual environment +python -m venv venv + +# Activate it +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt +pip install pipecatcloud +``` + +### 2. Get the starter project + +Clone the starter project from GitHub: + +```bash +git clone https://github.com/daily-co/pipecat-cloud-starter +cd pipecat-cloud-starter +``` + +or use the Pipecat Cloud CLI to initialize a new project: + +```bash +mkdir pipecat-cloud-starter && cd pipecat-cloud-starter +pcc init +``` + +### 3. Authenticate with Pipecat Cloud + +```bash +pcc auth login +``` + +### 4. Acquire required API keys + +This starter requires the following API keys: + +- **OpenAI API Key**: Get from [platform.openai.com/api-keys](https://platform.openai.com/api-keys) +- **Cartesia API Key**: Get from [play.cartesia.ai/keys](https://play.cartesia.ai/keys) +- **Daily API Key**: Automatically provided through your Pipecat Cloud account + +### 5. Configure to run locally (optional) + +You can test your agent locally before deploying to Pipecat Cloud: + +- `DAILY_API_KEY` value can be found at [https://pipecat.daily.co](https://pipecat.daily.co) Under the `Settings` menu of your agent, in the `Daily` tab. + +```bash +# Set environment variables with your API keys +export CARTESIA_API_KEY="your_cartesia_key" +export DAILY_API_KEY="your_daily_key" +export OPENAI_API_KEY="your_openai_key" +LOCAL_RUN=1 python bot.py +``` + +## Deploy & Run + +### 1. Build and push your Docker image + +```bash +# Build the image (targeting ARM architecture for cloud deployment) +docker build --platform=linux/arm64 -t my-first-agent:latest . + +# Tag with your Docker username and version +docker tag my-first-agent:latest your-username/my-first-agent:0.1 + +# Push to Docker Hub +docker push your-username/my-first-agent:0.1 +``` + +### 2. Create a secret set for your API keys + +The starter project requires API keys for OpenAI and Cartesia: + +```bash +# Copy the example env file +cp env.example .env + +# Edit .env to add your API keys: +# CARTESIA_API_KEY=your_cartesia_key +# OPENAI_API_KEY=your_openai_key + +# Create a secret set from your .env file +pcc secrets set my-first-agent-secrets --file .env +``` + +Alternatively, you can create secrets directly via CLI: + +```bash +pcc secrets set my-first-agent-secrets \ + CARTESIA_API_KEY=your_cartesia_key \ + OPENAI_API_KEY=your_openai_key +``` + +### 3. Deploy to Pipecat Cloud + +```bash +pcc deploy my-first-agent your-username/my-first-agent:0.1 +``` + +> **Note (Optional)**: For a more maintainable approach, you can use the included `pcc-deploy.toml` file: +> +> ```toml +> agent_name = "my-first-agent" +> image = "your-username/my-first-agent:0.1" +> secret_set = "my-first-agent-secrets" +> +> [scaling] +> min_instances = 0 +> ``` +> +> Then simply run `pcc deploy` without additional arguments. + +> **Note**: If your repository is private, you'll need to add credentials: +> +> ```bash +> # Create pull secret (you'll be prompted for credentials) +> pcc secrets image-pull-secret pull-secret https://index.docker.io/v1/ +> +> # Deploy with credentials +> pcc deploy my-first-agent your-username/my-first-agent:0.1 --credentials pull-secret +> ``` + +### 4. Check deployment and scaling (optional) + +By default, your agent will use "scale-to-zero" configuration, which means it may have a cold start of around 10 seconds when first used. By default, idle instances are maintained for 5 minutes before being terminated when using scale-to-zero. + +For more responsive testing, you can scale your deployment to keep a minimum of one instance warm: + +```bash +# Ensure at least one warm instance is always available +pcc deploy my-first-agent your-username/my-first-agent:0.1 --min-instances 1 + +# Check the status of your deployment +pcc agent status my-first-agent +``` + +By default, idle instances are maintained for 5 minutes before being terminated when using scale-to-zero. + +### 5. Create an API key + +```bash +# Create a public API key for accessing your agent +pcc organizations keys create + +# Set it as the default key to use with your agent +pcc organizations keys use +``` + +### 6. Start your agent + +```bash +# Start a session with your agent in a Daily room +pcc agent start my-first-agent --use-daily +``` + +This will return a URL, which you can use to connect to your running agent. + +## Documentation + +For more details on Pipecat Cloud and its capabilities: + +- [Pipecat Cloud Documentation](https://docs.pipecat.daily.co) +- [Pipecat Project Documentation](https://docs.pipecat.ai) + +## Support + +Join our [Discord community](https://discord.gg/dailyco) for help and discussions. diff --git a/examples/deployment/pipecat-cloud-example/bot.py b/examples/deployment/pipecat-cloud-example/bot.py new file mode 100644 index 000000000..fdb3eb712 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/bot.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +# Check if we're in local development mode +LOCAL_RUN = os.getenv("LOCAL_RUN") +if LOCAL_RUN: + import asyncio + import webbrowser + + try: + from local_runner import configure + except ImportError: + logger.error("Could not import local_runner module. Local development mode may not work.") + +# Load environment variables +load_dotenv(override=True) + + +async def main(room_url: str, token: str, session_logger=None): + """Main pipeline setup and execution function. + + Args: + room_url: The Daily room URL + token: The Daily room token + session_logger: Optional logger instance + """ + log = session_logger or logger + + log.debug("Starting bot in room: {}", room_url) + + async with aiohttp.ClientSession() as session: + transport = DailyTransport( + room_url, + token, + "bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + log.info("First participant joined: {}", participant["id"]) + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append( + { + "role": "system", + "content": "Please start with 'Hello World' and introduce yourself to the user.", + } + ) + await task.queue_frames([LLMMessagesFrame(messages)]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + log.info("Participant left: {}", participant) + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +async def bot(config, room_url: str, token: str, session_id=None, session_logger=None): + """Main bot entry point compatible with the FastAPI route handler. + + Args: + config: The configuration object from the request body + room_url: The Daily room URL + token: The Daily room token + session_id: The session ID for logging + session_logger: The session-specific logger + """ + log = session_logger or logger + log.info(f"Bot process initialized {room_url} {token}") + log.info(f"Bot config {config}") + + try: + await main(room_url, token, session_logger) + log.info("Bot process completed") + except Exception as e: + log.exception(f"Error in bot process: {str(e)}") + raise + + +# Local development functions +async def local_main(): + """Function for local development testing.""" + try: + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + logger.warning("_") + logger.warning("_") + logger.warning(f"Talk to your voice agent here: {room_url}") + logger.warning("_") + logger.warning("_") + webbrowser.open(room_url) + await main(room_url, token) + except Exception as e: + logger.exception(f"Error in local development mode: {e}") + + +# Local development entry point +if LOCAL_RUN and __name__ == "__main__": + try: + asyncio.run(local_main()) + except Exception as e: + logger.exception(f"Failed to run in local mode: {e}") diff --git a/examples/deployment/pipecat-cloud-example/env.example b/examples/deployment/pipecat-cloud-example/env.example new file mode 100644 index 000000000..1cbb5c4f6 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/env.example @@ -0,0 +1,2 @@ +CARTESIA_API_KEY= +OPENAI_API_KEY= \ No newline at end of file diff --git a/examples/deployment/pipecat-cloud-example/local_runner.py b/examples/deployment/pipecat-cloud-example/local_runner.py new file mode 100644 index 000000000..432592534 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/local_runner.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import os + +import aiohttp + +from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams + + +async def configure(aiohttp_session: aiohttp.ClientSession): + (url, token) = await configure_with_args(aiohttp_session) + return (url, token) + + +async def configure_with_args(aiohttp_session: aiohttp.ClientSession = None): + key = os.getenv("DAILY_API_KEY") + if not key: + raise Exception( + "No Daily API key specified. set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers." + ) + + daily_rest_helper = DailyRESTHelper( + daily_api_key=key, + daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"), + aiohttp_session=aiohttp_session, + ) + + room = await daily_rest_helper.create_room( + DailyRoomParams(properties={"enable_prejoin_ui": False}) + ) + if not room.url: + raise HTTPException(status_code=500, detail="Failed to create room") + + url = room.url + + # Create a meeting token for the given room with an expiration 1 hour in + # the future. + expiry_time: float = 60 * 60 + + token = await daily_rest_helper.get_token(url, expiry_time) + + return (url, token) diff --git a/examples/deployment/pipecat-cloud-example/pcc-deploy.toml b/examples/deployment/pipecat-cloud-example/pcc-deploy.toml new file mode 100644 index 000000000..063ed5929 --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/pcc-deploy.toml @@ -0,0 +1,6 @@ +agent_name = "my-first-agent" +image = "your-username/my-first-agent:0.1" +secret_set = "my-first-agent-secrets" + +[scaling] + min_instances = 0 diff --git a/examples/deployment/pipecat-cloud-example/requirements.txt b/examples/deployment/pipecat-cloud-example/requirements.txt new file mode 100644 index 000000000..f5abaf91f --- /dev/null +++ b/examples/deployment/pipecat-cloud-example/requirements.txt @@ -0,0 +1,2 @@ +pipecat-ai[cartesia,daily,openai,silero]>=0.0.58 +python-dotenv~=1.0.1 \ No newline at end of file From d3cd1a6c597104685dad56d2d807fea07e799201 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Mar 2025 20:37:55 -0400 Subject: [PATCH 299/427] Update with latest starter --- .../pipecat-cloud-example/README.md | 55 ++++--- .../deployment/pipecat-cloud-example/bot.py | 144 +++++++++--------- .../pipecat-cloud-example/requirements.txt | 3 +- 3 files changed, 101 insertions(+), 101 deletions(-) diff --git a/examples/deployment/pipecat-cloud-example/README.md b/examples/deployment/pipecat-cloud-example/README.md index 1950a0f94..7d5e4bd6f 100644 --- a/examples/deployment/pipecat-cloud-example/README.md +++ b/examples/deployment/pipecat-cloud-example/README.md @@ -4,6 +4,8 @@ A template voice agent for [Pipecat Cloud](https://www.daily.co/products/pipecat-cloud/) that demonstrates building and deploying a conversational AI agent. +> **For a detailed step-by-step guide, see our [Quickstart Documentation](https://docs.pipecat.daily.co/quickstart).** + ## Prerequisites - Python 3.10+ @@ -14,25 +16,9 @@ A template voice agent for [Pipecat Cloud](https://www.daily.co/products/pipecat > **Note**: If you haven't installed Docker yet, follow the official installation guides for your platform ([Linux](https://docs.docker.com/engine/install/), [Mac](https://docs.docker.com/desktop/setup/install/mac-install/), [Windows](https://docs.docker.com/desktop/setup/install/windows-install/)). For Docker Hub, [create a free account](https://hub.docker.com/signup) and log in via terminal with `docker login`. -## Getting Started +## Get Started -### 1. Set up Python environment - -We recommend using a virtual environment to manage your Python dependencies. - -```bash -# Create a virtual environment -python -m venv venv - -# Activate it -source venv/bin/activate # On Windows: venv\Scripts\activate - -# Install dependencies -pip install -r requirements.txt -pip install pipecatcloud -``` - -### 2. Get the starter project +### 1. Get the starter project Clone the starter project from GitHub: @@ -41,11 +27,19 @@ git clone https://github.com/daily-co/pipecat-cloud-starter cd pipecat-cloud-starter ``` -or use the Pipecat Cloud CLI to initialize a new project: +### 2. Set up your Python environment + +We recommend using a virtual environment to manage your Python dependencies. ```bash -mkdir pipecat-cloud-starter && cd pipecat-cloud-starter -pcc init +# Create a virtual environment +python -m venv .venv + +# Activate it +source .venv/bin/activate # On Windows: .venv\Scripts\activate + +# Install the Pipecat Cloud CLI +pip install pipecatcloud ``` ### 3. Authenticate with Pipecat Cloud @@ -66,13 +60,24 @@ This starter requires the following API keys: You can test your agent locally before deploying to Pipecat Cloud: -- `DAILY_API_KEY` value can be found at [https://pipecat.daily.co](https://pipecat.daily.co) Under the `Settings` menu of your agent, in the `Daily` tab. - ```bash # Set environment variables with your API keys export CARTESIA_API_KEY="your_cartesia_key" export DAILY_API_KEY="your_daily_key" export OPENAI_API_KEY="your_openai_key" +``` + +> Your `DAILY_API_KEY` can be found at [https://pipecat.daily.co](https://pipecat.daily.co) under the `Settings` in the `Daily (WebRTC)` tab. + +First install requirements: + +```bash +pip install -r requirements.txt +``` + +Then, launch the bot.py script locally: + +```bash LOCAL_RUN=1 python bot.py ``` @@ -118,7 +123,7 @@ pcc secrets set my-first-agent-secrets \ ### 3. Deploy to Pipecat Cloud ```bash -pcc deploy my-first-agent your-username/my-first-agent:0.1 +pcc deploy my-first-agent your-username/my-first-agent:0.1 --secrets my-first-agent-secrets ``` > **Note (Optional)**: For a more maintainable approach, you can use the included `pcc-deploy.toml` file: @@ -137,7 +142,7 @@ pcc deploy my-first-agent your-username/my-first-agent:0.1 > **Note**: If your repository is private, you'll need to add credentials: > > ```bash -> # Create pull secret (you'll be prompted for credentials) +> # Create pull secret (you’ll be prompted for credentials) > pcc secrets image-pull-secret pull-secret https://index.docker.io/v1/ > > # Deploy with credentials diff --git a/examples/deployment/pipecat-cloud-example/bot.py b/examples/deployment/pipecat-cloud-example/bot.py index fdb3eb712..89d4973b7 100644 --- a/examples/deployment/pipecat-cloud-example/bot.py +++ b/examples/deployment/pipecat-cloud-example/bot.py @@ -9,6 +9,7 @@ import os import aiohttp from dotenv import load_dotenv from loguru import logger +from pipecatcloud.agent import DailySessionArguments from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -35,110 +36,103 @@ if LOCAL_RUN: load_dotenv(override=True) -async def main(room_url: str, token: str, session_logger=None): +async def main(room_url: str, token: str): """Main pipeline setup and execution function. Args: room_url: The Daily room URL token: The Daily room token - session_logger: Optional logger instance """ - log = session_logger or logger + logger.debug("Starting bot in room: {}", room_url) - log.debug("Starting bot in room: {}", room_url) + transport = DailyTransport( + room_url, + token, + "bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) - async with aiohttp.ClientSession() as session: - transport = DailyTransport( - room_url, - token, - "bot", - DailyParams( - audio_out_enabled=True, - transcription_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - ), - ) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" + ) - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22" - ) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] - messages = [ + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info("First participant joined: {}", participant["id"]) + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append( { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", - }, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - pipeline = Pipeline( - [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - ] + "content": "Please start with 'Hello World' and introduce yourself to the user.", + } ) + await task.queue_frames([LLMMessagesFrame(messages)]) - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - report_only_initial_ttfb=True, - ), - ) + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info("Participant left: {}", participant) + await task.cancel() - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - log.info("First participant joined: {}", participant["id"]) - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - messages.append( - { - "role": "system", - "content": "Please start with 'Hello World' and introduce yourself to the user.", - } - ) - await task.queue_frames([LLMMessagesFrame(messages)]) + runner = PipelineRunner() - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - log.info("Participant left: {}", participant) - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) + await runner.run(task) -async def bot(config, room_url: str, token: str, session_id=None, session_logger=None): +async def bot(args: DailySessionArguments): """Main bot entry point compatible with the FastAPI route handler. Args: - config: The configuration object from the request body room_url: The Daily room URL token: The Daily room token + body: The configuration object from the request body session_id: The session ID for logging - session_logger: The session-specific logger """ - log = session_logger or logger - log.info(f"Bot process initialized {room_url} {token}") - log.info(f"Bot config {config}") + logger.info(f"Bot process initialized {args.room_url} {args.token}") try: - await main(room_url, token, session_logger) - log.info("Bot process completed") + await main(args.room_url, args.token) + logger.info("Bot process completed") except Exception as e: - log.exception(f"Error in bot process: {str(e)}") + logger.exception(f"Error in bot process: {str(e)}") raise diff --git a/examples/deployment/pipecat-cloud-example/requirements.txt b/examples/deployment/pipecat-cloud-example/requirements.txt index f5abaf91f..2e9fd4e9d 100644 --- a/examples/deployment/pipecat-cloud-example/requirements.txt +++ b/examples/deployment/pipecat-cloud-example/requirements.txt @@ -1,2 +1,3 @@ +pipecatcloud pipecat-ai[cartesia,daily,openai,silero]>=0.0.58 -python-dotenv~=1.0.1 \ No newline at end of file +python-dotenv~=1.0.1 From fa7da8f5f6b8acaaf6daeee205e9606cbc936bbb Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Sat, 1 Mar 2025 00:20:16 +0530 Subject: [PATCH 300/427] adding vertex llm --- CHANGELOG.md | 3 ++ src/pipecat/services/google/google.py | 73 +++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..b6001f82e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. +- Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI + Gemini models. + ### Changed - Updated the default mode for `CartesiaTTSService` and diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 9d4a0a8a8..44a677136 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -71,6 +71,7 @@ try: import google.generativeai as gai from google import genai from google.api_core.client_options import ClientOptions + from google.auth.transport.requests import Request from google.cloud import speech_v2, texttospeech_v1 from google.cloud.speech_v2.types import cloud_speech from google.genai import types @@ -1333,6 +1334,78 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): ) +class GoogleVertexAIService(OpenAILLMService): + """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. + Reference: + https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + """ + + class InputParams(OpenAILLMService.InputParams): + """Input parameters specific to Vertex AI.""" + + project_id: str + location: str + + def __init__( + self, + *, + credentials: Optional[str] = None, + credentials_path: Optional[str] = None, + model: str = "google/gemini-1.5-flash", + params: InputParams = OpenAILLMService.InputParams(), + **kwargs, + ): + """Initializes the VertexLLMService. + Args: + credentials (Optional[str]): JSON string of service account credentials. + credentials_path (Optional[str]): Path to the service account JSON file. + model (str): Model identifier. Defaults to "google/gemini-1.5-flash". + params (InputParams): Vertex AI input parameters. + **kwargs: Additional arguments for OpenAILLMService. + """ + base_url = self._get_base_url(params) + self._api_key = self._get_api_token(credentials, credentials_path) + + super().__init__(api_key=self._api_key, base_url=base_url, model=model, **kwargs) + + @staticmethod + def _get_base_url(params: InputParams) -> str: + """Constructs the base URL for Vertex AI API.""" + return ( + f"https://{params.location}-aiplatform.googleapis.com/v1/" + f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" + ) + + @staticmethod + def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: + """Retrieves an authentication token using Google service account credentials. + Args: + credentials (Optional[str]): JSON string of service account credentials. + credentials_path (Optional[str]): Path to the service account JSON file. + Returns: + str: OAuth token for API authentication. + """ + creds: Optional[service_account.Credentials] = None + + if credentials: + # Parse and load credentials from JSON string + creds = service_account.Credentials.from_service_account_info( + json.loads(credentials), scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + elif credentials_path: + # Load credentials from JSON file + creds = service_account.Credentials.from_service_account_file( + credentials_path, scopes=["https://www.googleapis.com/auth/cloud-platform"] + ) + + if not creds: + raise ValueError("No valid credentials provided.") + + creds.refresh(Request()) # Ensure token is up-to-date, lifetime is 1 hour. + + return creds.token + + class GoogleTTSService(TTSService): class InputParams(BaseModel): pitch: Optional[str] = None From 5f000efc611a3e7514df53e190abd6414296122d Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Sat, 15 Mar 2025 10:36:26 +0530 Subject: [PATCH 301/427] adding example --- CHANGELOG.md | 3 +- .../14p-function-calling-gemini-vertex-ai.py | 137 ++++++++++++++++++ src/pipecat/services/google/google.py | 7 +- 3 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 examples/foundational/14p-function-calling-gemini-vertex-ai.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b6001f82e..b3ee2646b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,7 +89,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 foundational example `19a-azure-realtime-beta.py`. - Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI - Gemini models. + Gemini models. Added foundational example + `14p-function-calling-gemini-vertex-ai.py`. ### Changed diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py new file mode 100644 index 000000000..68608e932 --- /dev/null +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.google import GoogleVertexAIService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + """Push a frame to the LLM; this is handy when the LLM response might take a while.""" + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = ElevenLabsTTSService( + api_key=os.getenv("ELEVENLABS_API_KEY", ""), + voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + ) + + llm = GoogleVertexAIService( + # credentials="", + params=GoogleVertexAIService.InputParams( + project_id="", + ) + ) + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function( + "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather + ) + + weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the user's location.", + }, + }, + required=["location", "format"], + ) + tools = ToolsSchema(standard_tools=[weather_function]) + + messages = [ + { + "role": "user", + "content": "Start a conversation with 'Hey there' to get the current weather.", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 44a677136..f741154b9 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1343,15 +1343,16 @@ class GoogleVertexAIService(OpenAILLMService): class InputParams(OpenAILLMService.InputParams): """Input parameters specific to Vertex AI.""" + # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations + location: str = "us-east4" project_id: str - location: str def __init__( self, *, credentials: Optional[str] = None, credentials_path: Optional[str] = None, - model: str = "google/gemini-1.5-flash", + model: str = "google/gemini-2.0-flash-001", params: InputParams = OpenAILLMService.InputParams(), **kwargs, ): @@ -1359,7 +1360,7 @@ class GoogleVertexAIService(OpenAILLMService): Args: credentials (Optional[str]): JSON string of service account credentials. credentials_path (Optional[str]): Path to the service account JSON file. - model (str): Model identifier. Defaults to "google/gemini-1.5-flash". + model (str): Model identifier. Defaults to "google/gemini-2.0-flash-001". params (InputParams): Vertex AI input parameters. **kwargs: Additional arguments for OpenAILLMService. """ From c1382b06911d32982807059fbb9a3ecef25a376f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 15 Mar 2025 20:30:35 -0400 Subject: [PATCH 302/427] Update the 34-audio-recording.py example to include an STT processor --- CHANGELOG.md | 2 ++ examples/foundational/34-audio-recording.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..d0e9f8713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Update the `34-audio-recording.py` example to include an STT processor. + - Added a Pipecat Cloud deployment example to the `examples` directory. ## [0.0.58] - 2025-02-26 diff --git a/examples/foundational/34-audio-recording.py b/examples/foundational/34-audio-recording.py index 94877c722..24f009570 100644 --- a/examples/foundational/34-audio-recording.py +++ b/examples/foundational/34-audio-recording.py @@ -32,6 +32,7 @@ Requirements: OPENAI_API_KEY=your_openai_key CARTESIA_API_KEY=your_cartesia_key DAILY_API_KEY=your_daily_key + DEEPGRAM_API_KEY=your_deepgram_key The recordings will be saved in a 'recordings' directory with timestamps: recordings/ @@ -65,6 +66,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -98,13 +100,14 @@ async def main(): DailyParams( # audio_in_enabled=True, audio_out_enabled=True, - transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, # Enable audio passthrough for recording + vad_audio_passthrough=True, ), ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"), audio_passthrough=True) + tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", @@ -128,6 +131,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), + stt, context_aggregator.user(), llm, tts, From 02dbef8f5a4453212ceeb74e51bc9603cff2b522 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 11:28:51 -0400 Subject: [PATCH 303/427] Add Neuphonic TTS to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 822c75aee..f75e9d386 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ pip install "pipecat-ai[option,...]" | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | | Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | From 3458f1b6dede365574cc91f11f00a964b08b96b1 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 11:43:40 -0400 Subject: [PATCH 304/427] Add Google Imagen to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f75e9d386..9522ce175 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ pip install "pipecat-ai[option,...]" | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), [FastAPI Websocket](https://docs.pipecat.ai/server/services/transport/fastapi-websocket), [WebSocket Server](https://docs.pipecat.ai/server/services/transport/websocket-server), Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Vision & Image | [fal](https://docs.pipecat.ai/server/services/image-generation/fal), [Google Imagen](https://docs.pipecat.ai/server/services/image-generation/fal), [Moondream](https://docs.pipecat.ai/server/services/vision/moondream) | `pip install "pipecat-ai[moondream]"` | | Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | | Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | From 1ad8e28025dbe04272608cc00636f2ff76d90bd8 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:21:36 -0400 Subject: [PATCH 305/427] Update TranscriptProcessor to more robustly handle different TTSTextFrame outputs --- .../processors/transcript_processor.py | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 614fe176c..6a7793335 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -90,9 +90,50 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): self._aggregation_start_time: Optional[str] = None async def _emit_aggregated_text(self): - """Emit aggregated text as a transcript message.""" + """Emit aggregated text as a transcript message. + + This method intelligently joins text fragments to create natural spacing, + handling both word-by-word and pre-spaced text fragments appropriately. + + The implementation handles two common patterns from TTS services: + + 1. Word-by-word fragments without spacing: + ``` + TTSTextFrame: ['Hello.'] + TTSTextFrame: ['How'] + TTSTextFrame: ['can'] + TTSTextFrame: ['I'] + TTSTextFrame: ['assist'] + TTSTextFrame: ['you'] + TTSTextFrame: ['today?'] + ``` + Result: "Hello. How can I assist you today?" + + 2. Pre-spaced fragments: + ``` + TTSTextFrame: ['Hello'] + TTSTextFrame: [' there'] + TTSTextFrame: ['!'] + TTSTextFrame: [' How'] + TTSTextFrame: ["'s"] + TTSTextFrame: [' it'] + TTSTextFrame: [' going'] + TTSTextFrame: ['?'] + ``` + Result: "Hello there! How's it going?" + """ if self._current_text_parts and self._aggregation_start_time: - content = " ".join(self._current_text_parts).strip() + # Build content with intelligent spacing + content = "" + for i, part in enumerate(self._current_text_parts): + # Add a space only when the current part doesn't start with + # whitespace or punctuation/special characters + if i > 0 and not part.startswith((" ", ".", ",", "!", "?", ";", ":", "'", '"')): + content += " " + content += part + + content = content.strip() + if content: logger.debug(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( From 5b6b700214b308319730263dd6854a6e99b22efe Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:22:13 -0400 Subject: [PATCH 306/427] OpenAIRealtimeBetaLLMService outputs a TTSTextFrame --- src/pipecat/services/openai_realtime_beta/openai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 00f8cd840..321c66826 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -43,6 +43,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -471,6 +472,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def _handle_evt_audio_transcript_delta(self, evt): if evt.delta: await self.push_frame(LLMTextFrame(evt.delta)) + await self.push_frame(TTSTextFrame(evt.delta)) async def _handle_evt_speech_started(self, evt): await self._truncate_current_audio_response() From 571c10403f9684bdcac175272c51d7e692f98378 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:23:00 -0400 Subject: [PATCH 307/427] tests: Add additional coverage to test_transcript_processor --- tests/test_transcript_processor.py | 298 ++++++++++++++++++++++++++++- 1 file changed, 297 insertions(+), 1 deletion(-) diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index 1c6db277f..5f80b3ca6 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -275,7 +275,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): # First update should be interrupted message first_message = received_updates[0].messages[0] self.assertEqual(first_message.role, "assistant") - self.assertEqual(first_message.content, "Hello world !") + self.assertEqual(first_message.content, "Hello world!") self.assertIsNotNone(first_message.timestamp) # Second update should be new response @@ -426,3 +426,299 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): self.assertEqual(received_updates[0].content, "User message") self.assertEqual(received_updates[1].role, "assistant") self.assertEqual(received_updates[1].content, "Assistant message") + + async def test_text_fragments_with_spaces(self): + """Test aggregating text fragments with various spacing patterns""" + processor = AssistantTranscriptProcessor() + + # Track received updates + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Test the specific pattern shared + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Hello"), + TTSTextFrame(text=" there"), + TTSTextFrame(text="!"), + TTSTextFrame(text=" How"), + TTSTextFrame(text="'s"), + TTSTextFrame(text=" it"), + TTSTextFrame(text=" going"), + TTSTextFrame(text="?"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + # Run test + received_frames, _ = await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + # Verify result + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.role, "assistant") + # Should be properly joined without extra spaces + self.assertEqual(message.content, "Hello there! How's it going?") + + async def test_mixed_spacing_styles(self): + """Test handling mixed word-by-word and pre-spaced fragments""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Mix of spacing styles within the same utterance + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + # Word-by-word style + TTSTextFrame(text="First"), + TTSTextFrame(text="style."), + # Pre-spaced style + TTSTextFrame(text=" Second"), + TTSTextFrame(text=" style"), + TTSTextFrame(text="!"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.content, "First style. Second style!") + + async def test_punctuation_handling(self): + """Test handling of various punctuation patterns""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Test various punctuation types + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Commas"), + TTSTextFrame(text=","), + TTSTextFrame(text="colons"), + TTSTextFrame(text=":"), + TTSTextFrame(text="semicolons"), + TTSTextFrame(text=";"), + TTSTextFrame(text="quotes"), + TTSTextFrame(text="'"), + TTSTextFrame(text="and"), + TTSTextFrame(text='"'), + TTSTextFrame(text="double quotes"), + TTSTextFrame(text="!"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual( + message.content, "Commas, colons: semicolons; quotes' and\" double quotes!" + ) + + async def test_complex_mixed_case(self): + """Test a complex mix of patterns to ensure robustness""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + # Complex mixed case with various patterns + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + # Pre-spaced fragments + TTSTextFrame(text="Hello"), + TTSTextFrame(text=" there"), + TTSTextFrame(text="!"), + # Sentence boundary + TTSTextFrame(text=" I'm"), + TTSTextFrame(text=" testing"), + TTSTextFrame(text=" spacing"), + TTSTextFrame(text="."), + # Word-by-word fragments + TTSTextFrame(text="Does"), + TTSTextFrame(text="this"), + TTSTextFrame(text="work"), + TTSTextFrame(text="correctly"), + TTSTextFrame(text="?"), + # Mixed punctuation and spacing + TTSTextFrame(text=" Let's"), + TTSTextFrame(text=" see:"), + TTSTextFrame(text="commas"), + TTSTextFrame(text=","), + TTSTextFrame(text=" semicolons"), + TTSTextFrame(text=";"), + TTSTextFrame(text=" and"), + TTSTextFrame(text=" quotes"), + TTSTextFrame(text="'"), + TTSTextFrame(text="!"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + expected = "Hello there! I'm testing spacing. Does this work correctly? Let's see: commas, semicolons; and quotes'!" + self.assertEqual(message.content, expected) + + async def test_multiple_consecutive_punctuation(self): + """Test handling of multiple consecutive punctuation marks""" + processor = AssistantTranscriptProcessor() + + received_updates = [] + + @processor.event_handler("on_transcript_update") + async def handle_update(proc, frame: TranscriptionUpdateFrame): + received_updates.append(frame) + + frames_to_send = [ + BotStartedSpeakingFrame(), + SleepFrame(sleep=0.1), + TTSTextFrame(text="Wow"), + TTSTextFrame(text="!"), + TTSTextFrame(text="!"), + TTSTextFrame(text="!"), + TTSTextFrame(text=" That's"), + TTSTextFrame(text=" amazing"), + TTSTextFrame(text="..."), + TTSTextFrame(text=" Don't"), + TTSTextFrame(text=" you"), + TTSTextFrame(text=" think"), + TTSTextFrame(text="?"), + TTSTextFrame(text="?"), + BotStoppedSpeakingFrame(), + ] + + expected_down_frames = [ + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TTSTextFrame, + TranscriptionUpdateFrame, + ] + + await run_test( + processor, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + + self.assertEqual(len(received_updates), 1) + message = received_updates[0].messages[0] + self.assertEqual(message.content, "Wow!!! That's amazing... Don't you think??") From 6e6905405b48f1548370650a0109419e96cbaebd Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:25:02 -0400 Subject: [PATCH 308/427] Update CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47f1010a..b24b74bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `TranscriptProcessor` to support text output from + `OpenAIRealtimeBetaLLMService`. + +- `OpenAIRealtimeBetaLLMService` now pushes a `TTSTextFrame`. + - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. From d5776c27f4cbeb8a576c00e6c32545f994f29a55 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:31:57 -0400 Subject: [PATCH 309/427] Update 19-openai-realtime-beta --- examples/foundational/19-openai-realtime-beta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 504f8fbf1..8796e0141 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -147,8 +147,8 @@ Remember, your responses should be short. Just one or two sentences, usually.""" transport.input(), # Transport user input context_aggregator.user(), llm, # LLM - context_aggregator.assistant(), transport.output(), # Transport bot output + context_aggregator.assistant(), ] ) From 3f002f8ffb837f55d0f08a52cf2a28644e75a766 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 13 Mar 2025 14:34:05 -0400 Subject: [PATCH 310/427] Remove unnecessary TranscriptProcessor examples --- CHANGELOG.md | 4 + ...penai.py => 28-transcription-processor.py} | 0 .../28b-transcript-processor-anthropic.py | 177 --------------- .../28c-transcription-processor-gemini.py | 210 ------------------ 4 files changed, 4 insertions(+), 387 deletions(-) rename examples/foundational/{28a-transcription-processor-openai.py => 28-transcription-processor.py} (100%) delete mode 100644 examples/foundational/28b-transcript-processor-anthropic.py delete mode 100644 examples/foundational/28c-transcription-processor-gemini.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b24b74bc5..596a3e1e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added a Pipecat Cloud deployment example to the `examples` directory. +- Removed foundational examples 28b and 28c as the TranscriptProcessor no + longer has an LLM depedency. Renamed foundational example 28a to + `28-transcript-processor.py`. + ## [0.0.58] - 2025-02-26 ### Added diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28-transcription-processor.py similarity index 100% rename from examples/foundational/28a-transcription-processor-openai.py rename to examples/foundational/28-transcription-processor.py diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py deleted file mode 100644 index c9f2672e0..000000000 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ /dev/null @@ -1,177 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sys -from typing import List, Optional - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.transcript_processor import TranscriptProcessor -from pipecat.services.anthropic import AnthropicLLMService -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -class TranscriptHandler: - """Handles real-time transcript processing and output. - - Maintains a list of conversation messages and outputs them either to a log - or to a file as they are received. Each message includes its timestamp and role. - - Attributes: - messages: List of all processed transcript messages - output_file: Optional path to file where transcript is saved. If None, outputs to log only. - """ - - def __init__(self, output_file: Optional[str] = None): - """Initialize handler with optional file output. - - Args: - output_file: Path to output file. If None, outputs to log only. - """ - self.messages: List[TranscriptionMessage] = [] - self.output_file: Optional[str] = output_file - logger.debug( - f"TranscriptHandler initialized {'with output_file=' + output_file if output_file else 'with log output only'}" - ) - - async def save_message(self, message: TranscriptionMessage): - """Save a single transcript message. - - Outputs the message to the log and optionally to a file. - - Args: - message: The message to save - """ - timestamp = f"[{message.timestamp}] " if message.timestamp else "" - line = f"{timestamp}{message.role}: {message.content}" - - # Always log the message - logger.info(f"Transcript: {line}") - - # Optionally write to file - if self.output_file: - try: - with open(self.output_file, "a", encoding="utf-8") as f: - f.write(line + "\n") - except Exception as e: - logger.error(f"Error saving transcript message to file: {e}") - - async def on_transcript_update( - self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame - ): - """Handle new transcript messages. - - Args: - processor: The TranscriptProcessor that emitted the update - frame: TranscriptionUpdateFrame containing new messages - """ - logger.debug(f"Received transcript update with {len(frame.messages)} new messages") - - for msg in frame.messages: - self.messages.append(msg) - await self.save_message(msg) - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20241022" - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", - }, - {"role": "user", "content": "Say hello."}, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - # Create transcript processor and handler - transcript = TranscriptProcessor() - transcript_handler = TranscriptHandler() # Output to log only - # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - transcript.user(), # User transcripts - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask(pipeline, params=PipelineParams(allow_interruptions=True)) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - # Register event handler for transcript updates - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - # Stop the pipeline immediately when the participant leaves - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py deleted file mode 100644 index 558edc76d..000000000 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ /dev/null @@ -1,210 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - -import asyncio -import os -import sqlite3 -import sys -from typing import List, Optional - -import aiohttp -from dotenv import load_dotenv -from loguru import logger -from runner import configure - -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.transcript_processor import TranscriptProcessor -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.services.google import GoogleLLMService -from pipecat.services.openai import OpenAILLMContext -from pipecat.transports.services.daily import DailyParams, DailyTransport - -load_dotenv(override=True) - -logger.remove(0) -logger.add(sys.stderr, level="DEBUG") - - -class TranscriptHandler: - """Handles real-time transcript processing and output. - - Maintains a list of conversation messages and outputs them either to a log - or to a file as they are received. Each message includes its timestamp and role. - - Attributes: - messages: List of all processed transcript messages - output_file: Optional path to file where transcript is saved. If None, outputs to log only. - """ - - def __init__(self, output_file: Optional[str] = None, output_db: Optional[str] = None): - """Initialize handler with optional file or database output. - - Args: - output_file: Path to output file. If None, outputs to log only. - """ - self.messages: List[TranscriptionMessage] = [] - self.output_file: Optional[str] = output_file - self.output_db: Optional[str] = output_db - - if self.output_db: - self.con = sqlite3.connect("example.db") - self.db = self.con.cursor() - - table = self.db.execute("SELECT name FROM sqlite_master WHERE name='messages'") - if not (table.fetchone()): - self.db.execute( - "CREATE TABLE messages(role TEXT, content TEXT, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP )" - ) - logger.debug( - f"TranscriptHandler initialized; output file: {output_file}, output DB: {output_db}" - ) - - async def save_message(self, message: TranscriptionMessage): - """Save a single transcript message. - - Outputs the message to the log and optionally to a SQLite database or file. - - Args: - message: The message to save - """ - timestamp = f"[{message.timestamp}] " if message.timestamp else "" - line = f"{timestamp}{message.role}: {message.content}" - - # Always log the message - logger.info(f"Transcript: {line}") - - # Optionally write to file - if self.output_file: - try: - with open(self.output_file, "a", encoding="utf-8") as f: - f.write(line + "\n") - except Exception as e: - logger.error(f"Error saving transcript message to file: {e}") - - # and/or to a SQLite database - if self.output_db: - self.db.execute( - "INSERT INTO messages VALUES (?, ?, ?)", - (message.role, message.content, message.timestamp), - ) - self.con.commit() - - async def on_transcript_update( - self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame - ): - """Handle new transcript messages. - - Args: - processor: The TranscriptProcessor that emitted the update - frame: TranscriptionUpdateFrame containing new messages - """ - logger.debug(f"Received transcript update with {len(frame.messages)} new messages") - - for msg in frame.messages: - self.messages.append(msg) - await self.save_message(msg) - - -async def main(): - async with aiohttp.ClientSession() as session: - (room_url, token) = await configure(session) - - transport = DailyTransport( - room_url, - None, - "Respond bot", - DailyParams( - audio_out_enabled=True, - vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), - vad_audio_passthrough=True, - ), - ) - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = GoogleLLMService( - model="models/gemini-2.0-flash-exp", - # model="gemini-exp-1114", - api_key=os.getenv("GOOGLE_API_KEY"), - ) - - messages = [ - { - "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", - }, - {"role": "user", "content": "Say hello."}, - ] - - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - # Create transcript processor and handler - transcript = TranscriptProcessor() - # Select a TranscriptHandler output method - # Uncomment out only one of the following lines: - transcript_handler = TranscriptHandler() # Output to log only - # transcript_handler = TranscriptHandler(output_file="transcript.txt") # Output to file and log - # transcript_handler = TranscriptHandler(output_db="example.db") # Output to SQLite DB and log - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, # STT - transcript.user(), # User transcripts - context_aggregator.user(), # User responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - transcript.assistant(), # Assistant transcripts - context_aggregator.assistant(), # Assistant spoken responses - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - allow_interruptions=True, - enable_metrics=True, - enable_usage_metrics=True, - ), - ) - - @transport.event_handler("on_first_participant_joined") - async def on_first_participant_joined(transport, participant): - await transport.capture_participant_transcription(participant["id"]) - # Kick off the conversation. - await task.queue_frames([context_aggregator.user().get_context_frame()]) - - # Register event handler for transcript updates - @transcript.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - - @transport.event_handler("on_participant_left") - async def on_participant_left(transport, participant, reason): - # Stop the pipeline immediately when the participant leaves - await task.cancel() - - runner = PipelineRunner() - - await runner.run(task) - - -if __name__ == "__main__": - asyncio.run(main()) From acd0660f66ccecaabf3089480045faba0c37a586 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 14 Mar 2025 14:00:38 -0400 Subject: [PATCH 311/427] Update GeminiMultimodalLiveLLMService to work with the TranscriptProcessor --- CHANGELOG.md | 3 ++- src/pipecat/services/gemini_multimodal_live/gemini.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 596a3e1e0..7376603c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,7 +93,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `TranscriptProcessor` to support text output from `OpenAIRealtimeBetaLLMService`. -- `OpenAIRealtimeBetaLLMService` now pushes a `TTSTextFrame`. +- `OpenAIRealtimeBetaLLMService` and `GeminiMultimodalLiveLLMService` now push + a `TTSTextFrame`. - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index ef49df329..5fe8a792b 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -38,6 +38,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, + TTSTextFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -312,6 +313,7 @@ class GeminiMultimodalLiveLLMService(LLMService): # context.add_message({"role": "assistant", "content": [{"type": "text", "text": text}]}) await self.push_frame(LLMFullResponseStartFrame()) await self.push_frame(LLMTextFrame(text=text)) + await self.push_frame(TTSTextFrame(text=text)) await self.push_frame(LLMFullResponseEndFrame()) async def _transcribe_audio(self, audio, context): From 6885d07e880341d1a5ae46054ae8d64609c3e9eb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 16:30:46 -0400 Subject: [PATCH 312/427] Simplify the TranscriptProcessor _emit_aggregated_text logic --- .../processors/transcript_processor.py | 86 +++--- tests/test_transcript_processor.py | 248 +----------------- 2 files changed, 50 insertions(+), 284 deletions(-) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 6a7793335..3eaff66ca 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -90,52 +90,62 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): self._aggregation_start_time: Optional[str] = None async def _emit_aggregated_text(self): - """Emit aggregated text as a transcript message. + """Aggregates and emits text fragments as a transcript message. - This method intelligently joins text fragments to create natural spacing, - handling both word-by-word and pre-spaced text fragments appropriately. + This method uses a heuristic to automatically detect whether text fragments + use pre-spacing (spaces at the beginning of fragments) or not, and applies + the appropriate joining strategy. It handles fragments from different TTS + services with different formatting patterns. - The implementation handles two common patterns from TTS services: + Examples: + Pre-spaced fragments (concatenated): + ``` + TTSTextFrame: ["Hello"] + TTSTextFrame: [" there"] + TTSTextFrame: ["!"] + TTSTextFrame: [" How"] + TTSTextFrame: ["'s"] + TTSTextFrame: [" it"] + TTSTextFrame: [" going"] + TTSTextFrame: ["?"] + ``` + Result: "Hello there! How's it going?" - 1. Word-by-word fragments without spacing: - ``` - TTSTextFrame: ['Hello.'] - TTSTextFrame: ['How'] - TTSTextFrame: ['can'] - TTSTextFrame: ['I'] - TTSTextFrame: ['assist'] - TTSTextFrame: ['you'] - TTSTextFrame: ['today?'] - ``` - Result: "Hello. How can I assist you today?" - - 2. Pre-spaced fragments: - ``` - TTSTextFrame: ['Hello'] - TTSTextFrame: [' there'] - TTSTextFrame: ['!'] - TTSTextFrame: [' How'] - TTSTextFrame: ["'s"] - TTSTextFrame: [' it'] - TTSTextFrame: [' going'] - TTSTextFrame: ['?'] - ``` - Result: "Hello there! How's it going?" + Word-by-word fragments (joined with spaces): + ``` + TTSTextFrame: ["Hello"] + TTSTextFrame: ["there!"] + TTSTextFrame: ["How"] + TTSTextFrame: ["is"] + TTSTextFrame: ["it"] + TTSTextFrame: ["going?"] + ``` + Result: "Hello there! How is it going?" """ if self._current_text_parts and self._aggregation_start_time: - # Build content with intelligent spacing - content = "" - for i, part in enumerate(self._current_text_parts): - # Add a space only when the current part doesn't start with - # whitespace or punctuation/special characters - if i > 0 and not part.startswith((" ", ".", ",", "!", "?", ";", ":", "'", '"')): - content += " " - content += part + # Heuristic to detect pre-spaced fragments + uses_prespacing = False + if len(self._current_text_parts) > 1: + # Check if any fragment after the first one starts with whitespace + has_spaced_parts = any( + part and part[0].isspace() for part in self._current_text_parts[1:] + ) + if has_spaced_parts: + uses_prespacing = True + # Apply appropriate joining method + if uses_prespacing: + # Pre-spaced fragments - just concatenate + content = "".join(self._current_text_parts) + else: + # Word-by-word fragments - join with spaces + content = " ".join(self._current_text_parts) + + # Clean up any excessive whitespace content = content.strip() if content: - logger.debug(f"Emitting aggregated assistant message: {content}") + logger.trace(f"Emitting aggregated assistant message: {content}") message = TranscriptionMessage( role="assistant", content=content, @@ -143,7 +153,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): ) await self._emit_update([message]) else: - logger.debug("No content to emit after stripping whitespace") + logger.trace("No content to emit after stripping whitespace") # Reset aggregation state self._current_text_parts = [] diff --git a/tests/test_transcript_processor.py b/tests/test_transcript_processor.py index 5f80b3ca6..d13246b2c 100644 --- a/tests/test_transcript_processor.py +++ b/tests/test_transcript_processor.py @@ -235,8 +235,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): BotStartedSpeakingFrame(), SleepFrame(sleep=0.1), TTSTextFrame(text="Hello"), - TTSTextFrame(text="world"), - TTSTextFrame(text="!"), + TTSTextFrame(text="world!"), SleepFrame(sleep=0.1), StartInterruptionFrame(), # User interrupts here BotStartedSpeakingFrame(), @@ -251,8 +250,7 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames = [ BotStartedSpeakingFrame, TTSTextFrame, # "Hello" - TTSTextFrame, # "world" - TTSTextFrame, # "!" + TTSTextFrame, # "world!" TranscriptionUpdateFrame, # First message (emitted due to interruption) StartInterruptionFrame, # Interruption frame comes after the update BotStartedSpeakingFrame, @@ -480,245 +478,3 @@ class TestUserTranscriptProcessor(unittest.IsolatedAsyncioTestCase): self.assertEqual(message.role, "assistant") # Should be properly joined without extra spaces self.assertEqual(message.content, "Hello there! How's it going?") - - async def test_mixed_spacing_styles(self): - """Test handling mixed word-by-word and pre-spaced fragments""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Mix of spacing styles within the same utterance - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - # Word-by-word style - TTSTextFrame(text="First"), - TTSTextFrame(text="style."), - # Pre-spaced style - TTSTextFrame(text=" Second"), - TTSTextFrame(text=" style"), - TTSTextFrame(text="!"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual(message.content, "First style. Second style!") - - async def test_punctuation_handling(self): - """Test handling of various punctuation patterns""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Test various punctuation types - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - TTSTextFrame(text="Commas"), - TTSTextFrame(text=","), - TTSTextFrame(text="colons"), - TTSTextFrame(text=":"), - TTSTextFrame(text="semicolons"), - TTSTextFrame(text=";"), - TTSTextFrame(text="quotes"), - TTSTextFrame(text="'"), - TTSTextFrame(text="and"), - TTSTextFrame(text='"'), - TTSTextFrame(text="double quotes"), - TTSTextFrame(text="!"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual( - message.content, "Commas, colons: semicolons; quotes' and\" double quotes!" - ) - - async def test_complex_mixed_case(self): - """Test a complex mix of patterns to ensure robustness""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - # Complex mixed case with various patterns - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - # Pre-spaced fragments - TTSTextFrame(text="Hello"), - TTSTextFrame(text=" there"), - TTSTextFrame(text="!"), - # Sentence boundary - TTSTextFrame(text=" I'm"), - TTSTextFrame(text=" testing"), - TTSTextFrame(text=" spacing"), - TTSTextFrame(text="."), - # Word-by-word fragments - TTSTextFrame(text="Does"), - TTSTextFrame(text="this"), - TTSTextFrame(text="work"), - TTSTextFrame(text="correctly"), - TTSTextFrame(text="?"), - # Mixed punctuation and spacing - TTSTextFrame(text=" Let's"), - TTSTextFrame(text=" see:"), - TTSTextFrame(text="commas"), - TTSTextFrame(text=","), - TTSTextFrame(text=" semicolons"), - TTSTextFrame(text=";"), - TTSTextFrame(text=" and"), - TTSTextFrame(text=" quotes"), - TTSTextFrame(text="'"), - TTSTextFrame(text="!"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - expected = "Hello there! I'm testing spacing. Does this work correctly? Let's see: commas, semicolons; and quotes'!" - self.assertEqual(message.content, expected) - - async def test_multiple_consecutive_punctuation(self): - """Test handling of multiple consecutive punctuation marks""" - processor = AssistantTranscriptProcessor() - - received_updates = [] - - @processor.event_handler("on_transcript_update") - async def handle_update(proc, frame: TranscriptionUpdateFrame): - received_updates.append(frame) - - frames_to_send = [ - BotStartedSpeakingFrame(), - SleepFrame(sleep=0.1), - TTSTextFrame(text="Wow"), - TTSTextFrame(text="!"), - TTSTextFrame(text="!"), - TTSTextFrame(text="!"), - TTSTextFrame(text=" That's"), - TTSTextFrame(text=" amazing"), - TTSTextFrame(text="..."), - TTSTextFrame(text=" Don't"), - TTSTextFrame(text=" you"), - TTSTextFrame(text=" think"), - TTSTextFrame(text="?"), - TTSTextFrame(text="?"), - BotStoppedSpeakingFrame(), - ] - - expected_down_frames = [ - BotStartedSpeakingFrame, - BotStoppedSpeakingFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TTSTextFrame, - TranscriptionUpdateFrame, - ] - - await run_test( - processor, - frames_to_send=frames_to_send, - expected_down_frames=expected_down_frames, - ) - - self.assertEqual(len(received_updates), 1) - message = received_updates[0].messages[0] - self.assertEqual(message.content, "Wow!!! That's amazing... Don't you think??") From c57fa93a700206885ea01a42414391708e58792d Mon Sep 17 00:00:00 2001 From: Lucas Rothman Date: Mon, 17 Mar 2025 16:22:36 -0700 Subject: [PATCH 313/427] Renamed to sample_rate --- src/pipecat/services/tavus.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index abb8b921d..9083dd53d 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -37,7 +37,7 @@ class TavusVideoService(AIService): replica_id: str, persona_id: str = "pipecat0", # Use `pipecat0` so that your TTS voice is used in place of the Tavus persona session: aiohttp.ClientSession, - output_sample_rate: int = 16000, + sample_rate: int = 16000, **kwargs, ) -> None: super().__init__(**kwargs) @@ -45,7 +45,7 @@ class TavusVideoService(AIService): self._replica_id = replica_id self._persona_id = persona_id self._session = session - self._output_sample_rate = output_sample_rate + self._sample_rate = sample_rate self._conversation_id: str @@ -96,7 +96,7 @@ class TavusVideoService(AIService): async def _encode_audio_and_send(self, audio: bytes, in_rate: int, done: bool) -> None: """Encodes audio to base64 and sends it to Tavus""" if not done: - audio = await self._resampler.resample(audio, in_rate, self._output_sample_rate) + audio = await self._resampler.resample(audio, in_rate, self._sample_rate) audio_base64 = base64.b64encode(audio).decode("utf-8") logger.trace(f"{self}: sending {len(audio)} bytes") await self._send_audio_message(audio_base64, done=done) @@ -110,7 +110,7 @@ class TavusVideoService(AIService): elif isinstance(frame, TTSAudioRawFrame): await self._encode_audio_and_send(frame.audio, frame.sample_rate, done=False) elif isinstance(frame, TTSStoppedFrame): - await self._encode_audio_and_send(b"\x00", self._output_sample_rate, done=True) + await self._encode_audio_and_send(b"\x00", self._sample_rate, done=True) await self.stop_ttfb_metrics() await self.stop_processing_metrics() elif isinstance(frame, StartInterruptionFrame): @@ -139,7 +139,7 @@ class TavusVideoService(AIService): "inference_id": self._current_idx_str, "audio": audio_base64, "done": done, - "sample_rate": self._output_sample_rate, + "sample_rate": self._sample_rate, }, } ) From 2e1a18503b9da4c5cee86c188483ce39e6701811 Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 10:41:43 +0100 Subject: [PATCH 314/427] Set tool choice from context aggregator --- src/pipecat/frames/frames.py | 6 ++++++ .../processors/aggregators/llm_response.py | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 74dd2accb..aec433033 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -362,6 +362,12 @@ class LLMSetToolsFrame(DataFrame): tools: List[dict] +@dataclass +class LLMSetToolChoiceFrame(DataFrame): + """A frame containing a tool choice for an LLM to use for function calling.""" + + tool_choice: Literal["none", "auto", "required"] + @dataclass class LLMEnablePromptCachingFrame(DataFrame): diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index d8582e32f..620e34b2c 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -7,7 +7,7 @@ import asyncio import time from abc import abstractmethod -from typing import List +from typing import List, Literal from pipecat.frames.frames import ( CancelFrame, @@ -22,6 +22,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, + LLMSetToolChoiceFrame, LLMTextFrame, StartFrame, StartInterruptionFrame, @@ -132,6 +133,11 @@ class BaseLLMResponseAggregator(FrameProcessor): """Set LLM tools to be used in the current conversation.""" pass + @abstractmethod + def set_tool_choice(self, tool_choice): + """Set the tool choice. This should modify the LLM context.""" + pass + @abstractmethod def reset(self): """Reset the internals of this aggregator. This should not modify the @@ -185,6 +191,9 @@ class LLMResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools): pass + def set_tool_choice(self, tool_choice): + pass + def reset(self): self._aggregation = "" @@ -244,6 +253,9 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools: List): self._context.set_tools(tools) + def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"]): + self._context.set_tool_choice(tool_choice) + def reset(self): self._aggregation = "" @@ -328,6 +340,8 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self.set_messages(frame.messages) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + elif isinstance(frame, LLMSetToolChoiceFrame): + self.set_tool_choice(frame.tool_choice) else: await self.push_frame(frame, direction) @@ -448,6 +462,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self.set_messages(frame.messages) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + elif isinstance(frame, LLMSetToolChoiceFrame): + self.set_tool_choice(frame.tool_choice) else: await self.push_frame(frame, direction) From 1c19777d5eda4cf237f83d8dcf3dc82c4f2a4540 Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 11:09:40 +0100 Subject: [PATCH 315/427] Fix format --- src/pipecat/frames/frames.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index aec433033..7b07a331e 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -362,6 +362,7 @@ class LLMSetToolsFrame(DataFrame): tools: List[dict] + @dataclass class LLMSetToolChoiceFrame(DataFrame): """A frame containing a tool choice for an LLM to use for function calling.""" From dc5067407d6b2136e70b411e070cf774bdfea1e0 Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 11:12:51 +0100 Subject: [PATCH 316/427] Fix ruff check --- src/pipecat/processors/aggregators/llm_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 620e34b2c..9c18e67ca 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -21,8 +21,8 @@ from pipecat.frames.frames import ( LLMMessagesAppendFrame, LLMMessagesFrame, LLMMessagesUpdateFrame, - LLMSetToolsFrame, LLMSetToolChoiceFrame, + LLMSetToolsFrame, LLMTextFrame, StartFrame, StartInterruptionFrame, From 188677e601658e3e6b0d4c389e8b30e90837431f Mon Sep 17 00:00:00 2001 From: Adnan Siddiquei Date: Tue, 18 Mar 2025 10:35:22 +0000 Subject: [PATCH 317/427] Added 4 new languages: FR, PT, RU, ZH, HI. --- src/pipecat/services/neuphonic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index a2ea98f42..b935885b6 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -49,6 +49,11 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: Language.ES: "es", Language.NL: "nl", Language.AR: "ar", + Language.FR: "fr", + Language.PT: "pt", + Language.RU: "ru", + Language.HI: "HI", + Language.ZH: "zh", } result = BASE_LANGUAGES.get(language) From e731a0d41f96860aaf053e1a64f172b6e81b31b2 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 22:44:12 -0400 Subject: [PATCH 318/427] Add PairPatternAggregator --- .../utils/text/pattern_pair_aggregator.py | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 src/pipecat/utils/text/pattern_pair_aggregator.py diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py new file mode 100644 index 000000000..734e6a9bd --- /dev/null +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -0,0 +1,262 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import re +from typing import Callable, Dict, Optional, Pattern, Tuple, Union + +from loguru import logger + +from pipecat.utils.string import match_endofsentence +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator + + +class PatternMatch: + """Represents a matched pattern pair with its content. + + A PatternMatch object is created when a complete pattern pair is found + in the text. It contains information about which pattern was matched, + the full matched text (including start and end patterns), and the + content between the patterns. + + Attributes: + pattern_id: The identifier of the matched pattern pair. + full_match: The complete text including start and end patterns. + content: The text content between the start and end patterns. + """ + + def __init__(self, pattern_id: str, full_match: str, content: str): + """Initialize a pattern match. + + Args: + pattern_id: ID of the pattern pair. + full_match: Complete matched text including start and end patterns. + content: Content between the start and end patterns. + """ + self.pattern_id = pattern_id + self.full_match = full_match + self.content = content + + def __str__(self) -> str: + """Return a string representation of the pattern match. + + Returns: + A string describing the pattern match. + """ + return f"PatternMatch(id={self.pattern_id}, content={self.content})" + + +class PatternPairAggregator(BaseTextAggregator): + """Aggregator that identifies and processes content between pattern pairs. + + This aggregator buffers text until it can identify complete pattern pairs + (defined by start and end patterns), processes the content between these + patterns using registered handlers, and returns text at sentence boundaries. + It's particularly useful for processing structured content in streaming text, + such as XML tags, markdown formatting, or custom delimiters. + + The aggregator ensures that patterns spanning multiple text chunks are + correctly identified and handles cases where patterns contain sentence + boundaries. + """ + + def __init__(self): + """Initialize the pattern pair aggregator. + + Creates an empty aggregator with no patterns or handlers registered. + """ + self._text = "" + self._patterns = {} + self._handlers = {} + + @property + def text(self) -> str: + """Get the currently buffered text. + + Returns: + The current text buffer content. + """ + return self._text + + def add_pattern_pair( + self, pattern_id: str, start_pattern: str, end_pattern: str, remove_match: bool = True + ) -> "PatternPairAggregator": + """Add a pattern pair to detect in the text. + + Registers a new pattern pair with a unique identifier. The aggregator + will look for text that starts with the start pattern and ends with + the end pattern, and treat the content between them as a match. + + Args: + pattern_id: Unique identifier for this pattern pair. + start_pattern: Pattern that marks the beginning of content. + end_pattern: Pattern that marks the end of content. + remove_match: Whether to remove the matched content from the text. + + Returns: + Self for method chaining. + """ + self._patterns[pattern_id] = { + "start": start_pattern, + "end": end_pattern, + "remove_match": remove_match, + } + return self + + def on_pattern_match( + self, pattern_id: str, handler: Callable[[PatternMatch], None] + ) -> "PatternPairAggregator": + """Register a handler for when a pattern pair is matched. + + The handler will be called whenever a complete match for the + specified pattern ID is found in the text. + + Args: + pattern_id: ID of the pattern pair to match. + handler: Function to call when pattern is matched. + The function should accept a PatternMatch object. + + Returns: + Self for method chaining. + """ + self._handlers[pattern_id] = handler + return self + + def _process_complete_patterns(self, text: str) -> Tuple[str, bool]: + """Process all complete pattern pairs in the text. + + Searches for all complete pattern pairs in the text, calls the + appropriate handlers, and optionally removes the matches. + + Args: + text: The text to process. + + Returns: + Tuple of (processed_text, was_modified) where: + - processed_text is the text after processing patterns + - was_modified indicates whether any changes were made + """ + processed_text = text + modified = False + + for pattern_id, pattern_info in self._patterns.items(): + # Escape special regex characters in the patterns + start = re.escape(pattern_info["start"]) + end = re.escape(pattern_info["end"]) + remove_match = pattern_info["remove_match"] + + # Create regex to match from start pattern to end pattern + # The .*? is non-greedy to handle nested patterns + regex = f"{start}(.*?){end}" + + # Find all matches + match_iter = re.finditer(regex, processed_text, re.DOTALL) + matches = list(match_iter) # Convert to list for safe iteration + + for match in matches: + content = match.group(1) # Content between patterns + full_match = match.group(0) # Full match including patterns + + # Create pattern match object + pattern_match = PatternMatch( + pattern_id=pattern_id, full_match=full_match, content=content + ) + + # Call the appropriate handler if registered + if pattern_id in self._handlers: + try: + self._handlers[pattern_id](pattern_match) + except Exception as e: + logger.error(f"Error in pattern handler for {pattern_id}: {e}") + + # Remove the pattern from the text if configured + if remove_match: + processed_text = processed_text.replace(full_match, "", 1) + modified = True + + return processed_text, modified + + def _has_incomplete_patterns(self, text: str) -> bool: + """Check if text contains incomplete pattern pairs. + + Determines whether the text contains any start patterns without + matching end patterns, which would indicate incomplete content. + + Args: + text: The text to check. + + Returns: + True if there are incomplete patterns, False otherwise. + """ + for pattern_id, pattern_info in self._patterns.items(): + start = pattern_info["start"] + end = pattern_info["end"] + + # Count occurrences + start_count = text.count(start) + end_count = text.count(end) + + # If there are more starts than ends, we have incomplete patterns + if start_count > end_count: + return True + + return False + + def aggregate(self, text: str) -> Optional[str]: + """Aggregate text and process pattern pairs. + + This method adds the new text to the buffer, processes any complete pattern + pairs, and returns processed text up to sentence boundaries if possible. + If there are incomplete patterns (start without matching end), it will + continue buffering text. + + Args: + text: New text to add to the buffer. + + Returns: + Processed text up to a sentence boundary, or None if more + text is needed to form a complete sentence or pattern. + """ + # Add new text to buffer + self._text += text + + # Process any complete patterns in the buffer + processed_text, modified = self._process_complete_patterns(self._text) + + # Only update the buffer if modifications were made + if modified: + self._text = processed_text + + # Check if we have incomplete patterns + if self._has_incomplete_patterns(self._text): + # Still waiting for complete patterns + return None + + # Find sentence boundary if no incomplete patterns + eos_marker = match_endofsentence(self._text) + if eos_marker: + # Extract text up to the sentence boundary + result = self._text[:eos_marker] + self._text = self._text[eos_marker:] + return result + + # No complete sentence found yet + return None + + def handle_interruption(self): + """Handle interruptions by clearing the buffer. + + Called when an interruption occurs in the processing pipeline, + to reset the state and discard any partially aggregated text. + """ + self._text = "" + + def reset(self): + """Clear the internally aggregated text. + + Resets the aggregator to its initial state, discarding any + buffered text. + """ + self._text = "" From ddcc1fbb2fc28b0547c5025aaecba4d8f04cd89f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 23:05:22 -0400 Subject: [PATCH 319/427] Add foundational example 35 --- examples/foundational/35-voice-switching.py | 192 ++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 examples/foundational/35-voice-switching.py diff --git a/examples/foundational/35-voice-switching.py b/examples/foundational/35-voice-switching.py new file mode 100644 index 000000000..5dd986bc3 --- /dev/null +++ b/examples/foundational/35-voice-switching.py @@ -0,0 +1,192 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +# Define voice IDs +VOICE_IDS = { + "narrator": "c45bc5ec-dc68-4feb-8829-6e6b2748095d", # Narrator voice + "female": "71a7ad14-091c-4e8e-a314-022ece01c121", # Female character voice + "male": "7cf0e2b1-8daf-4fe4-89ad-f6039398f359", # Male character voice +} + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Storytelling Bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # Initialize TTS with narrator voice as default + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id=VOICE_IDS["narrator"], + ) + + # Create pattern pair aggregator for voice switching + pattern_aggregator = PatternPairAggregator() + + # Add pattern for voice switching + pattern_aggregator.add_pattern_pair( + pattern_id="voice_tag", + start_pattern="", + end_pattern="", + remove_match=True, + ) + + # Register handler for voice switching + def on_voice_tag(match: PatternMatch): + voice_name = match.content.strip().lower() + if voice_name in VOICE_IDS: + voice_id = VOICE_IDS[voice_name] + tts.set_voice(voice_id) + logger.info(f"Switched to {voice_name} voice") + else: + logger.warning(f"Unknown voice: {voice_name}") + + pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) + + # Set the pattern aggregator on the TTS service + tts._text_aggregator = pattern_aggregator + + # Initialize LLM + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + # System prompt for storytelling with voice switching + system_prompt = """You are an engaging storyteller that uses different voices to bring stories to life. + +You have three voices to use, but each has a specific purpose: + +narrator +This is the default narrator voice. Use this for all narration, descriptions, and non-dialogue text. + +female +Use this ONLY for direct speech by female characters (just the quoted text). + +male +Use this ONLY for direct speech by male characters (just the quoted text). + +IMPORTANT: Switch back to narrator voice immediately after character dialogue. + +Here's an EXAMPLE of correct voice usage: + +narrator +Sarah spotted her old friend across the café. She couldn't believe her eyes. + +female +"Jacob! It's been so long!" + +narrator +Sarah exclaimed, jumping up from her seat with a radiant smile. + +male +"Sarah, is it really you? I can't believe it!" + +narrator +Jacob replied, grinning widely as he walked over to her. The two friends embraced warmly, as if trying to make up for all the years spent apart. + +female +"What are you doing in town? Last I heard you were in Seattle." + +narrator +She asked, gesturing for him to join her at the table. + +FOLLOW THESE RULES: +1. Always begin with the narrator voice +2. Only use character voices for the EXACT words they speak (in quotes) +3. SWITCH BACK to narrator voice for speech tags and all other text +4. Begin by asking what kind of story the user would like to hear +5. Create engaging dialogue with distinct characters + +Remember: Use narrator voice for EVERYTHING except the actual quoted dialogue.""" + + # Set up LLM context + messages = [ + { + "role": "system", + "content": system_prompt, + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create pipeline + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, # TTS with pattern aggregator + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + logger.info(f"First participant joined: {participant['id']}") + await transport.capture_participant_transcription(participant["id"]) + + # Start conversation - empty prompt to let LLM follow system instructions + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + logger.info(f"Participant left: {participant['id']}") + await task.cancel() + + logger.info(f"Starting storytelling bot at: {room_url}") + logger.info("Join the room to interact with the bot!") + + runner = PipelineRunner() + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 6ec4052f29271d8fcbade96b77401c2cc86f0033 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 23:17:41 -0400 Subject: [PATCH 320/427] Add CHANGELOG entries --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 406e9a304..c43971da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `PatternPairAggregator` that extends `BaseTextAggregator` to + identify content between matching pattern pairs in streamed text. This allows + for detection and processing of structured content like XML-style tags that + may span across multiple text chunks or sentence boundaries. + - Added new `BaseTextAggregator`. Text aggregators are used by the TTS service to aggregate LLM tokens and decide when the aggregated text should be pushed to the TTS service. It also allows for the text to be manipulated while it's @@ -124,6 +129,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update the `34-audio-recording.py` example to include an STT processor. +- Added foundational example `35-voice-switching.py` showing how to use the new + `PatternPairAggregator`. + - Added a Pipecat Cloud deployment example to the `examples` directory. - Removed foundational examples 28b and 28c as the TranscriptProcessor no From 2dee882710a771509afd9b4a6db7c4777a431b33 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 17 Mar 2025 23:29:08 -0400 Subject: [PATCH 321/427] Add unit tests --- tests/test_pattern_pair_aggregator.py | 147 ++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 tests/test_pattern_pair_aggregator.py diff --git a/tests/test_pattern_pair_aggregator.py b/tests/test_pattern_pair_aggregator.py new file mode 100644 index 000000000..e1086e577 --- /dev/null +++ b/tests/test_pattern_pair_aggregator.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest +from unittest.mock import Mock + +from pipecat.utils.text.pattern_pair_aggregator import PatternMatch, PatternPairAggregator + + +class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.aggregator = PatternPairAggregator() + self.test_handler = Mock() + + # Add a test pattern + self.aggregator.add_pattern_pair( + pattern_id="test_pattern", + start_pattern="", + end_pattern="", + remove_match=True, + ) + + # Register the mock handler + self.aggregator.on_pattern_match("test_pattern", self.test_handler) + + async def test_pattern_match_and_removal(self): + # First part doesn't complete the pattern + result = self.aggregator.aggregate("Hello pattern") + self.assertIsNone(result) + self.assertEqual(self.aggregator.text, "Hello pattern") + + # Second part completes the pattern and includes an exclamation point + result = self.aggregator.aggregate(" content!") + + # Verify the handler was called with correct PatternMatch object + self.test_handler.assert_called_once() + call_args = self.test_handler.call_args[0][0] + self.assertIsInstance(call_args, PatternMatch) + self.assertEqual(call_args.pattern_id, "test_pattern") + self.assertEqual(call_args.full_match, "pattern content") + self.assertEqual(call_args.content, "pattern content") + + # The exclamation point should be treated as a sentence boundary, + # so the result should include just text up to and including "!" + self.assertEqual(result, "Hello !") + + # Next sentence should be processed separately + result = self.aggregator.aggregate(" This is another sentence.") + self.assertEqual(result, " This is another sentence.") + + # Buffer should be empty after returning a complete sentence + self.assertEqual(self.aggregator.text, "") + + async def test_incomplete_pattern(self): + # Add text with incomplete pattern + result = self.aggregator.aggregate("Hello pattern content") + + # No complete pattern yet, so nothing should be returned + self.assertIsNone(result) + + # The handler should not be called yet + self.test_handler.assert_not_called() + + # Buffer should contain the incomplete text + self.assertEqual(self.aggregator.text, "Hello pattern content") + + # Reset and confirm buffer is cleared + self.aggregator.reset() + self.assertEqual(self.aggregator.text, "") + + async def test_multiple_patterns(self): + # Set up multiple patterns and handlers + voice_handler = Mock() + emphasis_handler = Mock() + + self.aggregator.add_pattern_pair( + pattern_id="voice", start_pattern="", end_pattern="", remove_match=True + ) + + self.aggregator.add_pattern_pair( + pattern_id="emphasis", + start_pattern="", + end_pattern="", + remove_match=False, # Keep emphasis tags + ) + + self.aggregator.on_pattern_match("voice", voice_handler) + self.aggregator.on_pattern_match("emphasis", emphasis_handler) + + # Test with multiple patterns in one text block + text = "Hello female I am very excited to meet you!" + result = self.aggregator.aggregate(text) + + # Both handlers should be called with correct data + voice_handler.assert_called_once() + voice_match = voice_handler.call_args[0][0] + self.assertEqual(voice_match.pattern_id, "voice") + self.assertEqual(voice_match.content, "female") + + emphasis_handler.assert_called_once() + emphasis_match = emphasis_handler.call_args[0][0] + self.assertEqual(emphasis_match.pattern_id, "emphasis") + self.assertEqual(emphasis_match.content, "very") + + # Voice pattern should be removed, emphasis pattern should remain + self.assertEqual(result, "Hello I am very excited to meet you!") + + # Buffer should be empty + self.assertEqual(self.aggregator.text, "") + + async def test_handle_interruption(self): + # Start with incomplete pattern + result = self.aggregator.aggregate("Hello pattern") + self.assertIsNone(result) + + # Simulate interruption + self.aggregator.handle_interruption() + + # Buffer should be cleared + self.assertEqual(self.aggregator.text, "") + + # Handler should not have been called + self.test_handler.assert_not_called() + + async def test_pattern_across_sentences(self): + # Test pattern that spans multiple sentences + result = self.aggregator.aggregate("Hello This is sentence one.") + + # First sentence contains start of pattern but no end, so no complete pattern yet + self.assertIsNone(result) + + # Add second part with pattern end + result = self.aggregator.aggregate(" This is sentence two. Final sentence.") + + # Handler should be called with entire content + self.test_handler.assert_called_once() + call_args = self.test_handler.call_args[0][0] + self.assertEqual(call_args.content, "This is sentence one. This is sentence two.") + + # Pattern should be removed, resulting in text with sentences merged + self.assertEqual(result, "Hello Final sentence.") + + # Buffer should be empty + self.assertEqual(self.aggregator.text, "") From b28276446d15e565645f0a024a01b2893711b09a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 18 Mar 2025 07:47:18 -0400 Subject: [PATCH 322/427] Code review feedback --- CHANGELOG.md | 5 +- ....py => 35-pattern-pair-voice-switching.py} | 56 ++++++++++++++++--- .../utils/text/pattern_pair_aggregator.py | 2 +- 3 files changed, 52 insertions(+), 11 deletions(-) rename examples/foundational/{35-voice-switching.py => 35-pattern-pair-voice-switching.py} (77%) diff --git a/CHANGELOG.md b/CHANGELOG.md index c43971da8..6d200ab87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update the `34-audio-recording.py` example to include an STT processor. - Added foundational example `35-voice-switching.py` showing how to use the new - `PatternPairAggregator`. + `PatternPairAggregator`. This example shows how to encode information for the + LLM to instruct TTS voice changes, but this can be used to encode any + information into the LLM response, which you want to parse and use in other + parts of your application. - Added a Pipecat Cloud deployment example to the `examples` directory. diff --git a/examples/foundational/35-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py similarity index 77% rename from examples/foundational/35-voice-switching.py rename to examples/foundational/35-pattern-pair-voice-switching.py index 5dd986bc3..bb9587706 100644 --- a/examples/foundational/35-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -4,6 +4,46 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Pattern Pair Voice Switching Example with Pipecat. + +This example demonstrates how to use the PatternPairAggregator to dynamically switch +between different voices in a storytelling application. It showcases how pattern matching +can be used to control TTS behavior in streaming text from an LLM. + +The example: + 1. Sets up a storytelling bot with three distinct voices (narrator, male, female) + 2. Uses pattern pairs (name) to trigger voice switching + 3. Processes the patterns in real-time as text streams from the LLM + 4. Removes the pattern tags before sending text to TTS + +The PatternPairAggregator: + - Buffers text until complete patterns are detected + - Identifies content between start/end pattern pairs + - Triggers callbacks when patterns are matched + - Processes patterns that may span across multiple text chunks + - Returns processed text at sentence boundaries + +Example usage (run from pipecat root directory): + $ pip install "pipecat-ai[daily,openai,cartesia,silero]" + $ pip install -r dev-requirements.txt + $ python examples/foundational/35-pattern-pair-voice-switching.py + +Requirements: + - OpenAI API key (for GPT-4o) + - Cartesia API key (for text-to-speech) + - Daily API key (for video/audio transport) + + Environment variables (.env file): + OPENAI_API_KEY=your_openai_key + CARTESIA_API_KEY=your_cartesia_key + DAILY_API_KEY=your_daily_key + +Note: + This example shows one application of PatternPairAggregator (voice switching), + but the same approach can be used for various pattern-based text processing needs, + such as formatting instructions, command recognition, or structured data extraction. +""" + import asyncio import os import sys @@ -43,7 +83,7 @@ async def main(): transport = DailyTransport( room_url, token, - "Storytelling Bot", + "Multi-voice storyteller", DailyParams( audio_out_enabled=True, transcription_enabled=True, @@ -52,12 +92,6 @@ async def main(): ), ) - # Initialize TTS with narrator voice as default - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id=VOICE_IDS["narrator"], - ) - # Create pattern pair aggregator for voice switching pattern_aggregator = PatternPairAggregator() @@ -81,8 +115,12 @@ async def main(): pattern_aggregator.on_pattern_match("voice_tag", on_voice_tag) - # Set the pattern aggregator on the TTS service - tts._text_aggregator = pattern_aggregator + # Initialize TTS with narrator voice as default + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id=VOICE_IDS["narrator"], + text_aggregator=pattern_aggregator, + ) # Initialize LLM llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") diff --git a/src/pipecat/utils/text/pattern_pair_aggregator.py b/src/pipecat/utils/text/pattern_pair_aggregator.py index 734e6a9bd..86f87103b 100644 --- a/src/pipecat/utils/text/pattern_pair_aggregator.py +++ b/src/pipecat/utils/text/pattern_pair_aggregator.py @@ -5,7 +5,7 @@ # import re -from typing import Callable, Dict, Optional, Pattern, Tuple, Union +from typing import Callable, Optional, Tuple from loguru import logger From 4303ed4991a930bb4c4d3d268ea7991db3f78e56 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 18 Mar 2025 20:58:21 +0530 Subject: [PATCH 323/427] rename service --- CHANGELOG.md | 2 +- .../foundational/14p-function-calling-gemini-vertex-ai.py | 6 +++--- src/pipecat/services/google/google.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ee2646b..74cd0b15f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `AzureRealtimeBetaLLMService` to support Azure's OpeanAI Realtime API. Added foundational example `19a-azure-realtime-beta.py`. -- Introduced `GoogleVertexAIService`, a new class for integrating with Vertex AI +- Introduced `GoogleVertexLLMService`, a new class for integrating with Vertex AI Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index 68608e932..d64bae89d 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -21,7 +21,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.services.elevenlabs import ElevenLabsTTSService -from pipecat.services.google import GoogleVertexAIService +from pipecat.services.google import GoogleVertexLLMService from pipecat.services.openai import OpenAILLMContext from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -62,9 +62,9 @@ async def main(): voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), ) - llm = GoogleVertexAIService( + llm = GoogleVertexLLMService( # credentials="", - params=GoogleVertexAIService.InputParams( + params=GoogleVertexLLMService.InputParams( project_id="", ) ) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index f741154b9..f9a8d4894 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1334,7 +1334,7 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): ) -class GoogleVertexAIService(OpenAILLMService): +class GoogleVertexLLMService(OpenAILLMService): """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library From 32609b1132595340fee838f5762ee32f2f9c0296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 13 Mar 2025 18:04:25 -0700 Subject: [PATCH 324/427] event handlers are now executed in separate tasks --- CHANGELOG.md | 5 +++ src/pipecat/pipeline/runner.py | 6 ++++ src/pipecat/pipeline/task.py | 4 +++ src/pipecat/processors/frame_processor.py | 1 + .../transports/network/fastapi_websocket.py | 18 ++++++++-- .../transports/network/websocket_client.py | 28 +++++++++++++--- .../transports/network/websocket_server.py | 19 +++++++++-- src/pipecat/transports/services/daily.py | 24 +++++++++++--- src/pipecat/transports/services/livekit.py | 33 ++++++++++++++++--- src/pipecat/utils/base_object.py | 29 ++++++++++++++++ 10 files changed, 150 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d200ab87..0a5d29e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- All event handlers are now executed in separate tasks in order to prevent + blocking the pipeline. It is possible that event handlers take some time to + execute in which case the pipeline would be blocked waiting for the event + handler to complete. + - Updated `TranscriptProcessor` to support text output from `OpenAIRealtimeBetaLLMService`. diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 3209fa92a..7ac07064f 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -40,12 +40,18 @@ class PipelineRunner(BaseObject): task.set_event_loop(self._loop) await task.run() del self._tasks[task.name] + + # Cleanup base object. + await self.cleanup() + # If we are cancelling through a signal, make sure we wait for it so # everything gets cleaned up nicely. if self._sig_task: await self._sig_task + if self._force_gc: self._gc_collect() + logger.debug(f"Runner {self} finished running {task}") async def stop_when_done(self): diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index c10b33189..12c56ef3f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -354,6 +354,10 @@ class PipelineTask(BaseTask): self._pipeline_end_event.clear() async def _cleanup(self, cleanup_pipeline: bool): + # Cleanup base object. + await self.cleanup() + + # Cleanup pipeline processors. await self._source.cleanup() if cleanup_pipeline: await self._pipeline.cleanup() diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 6a1669ff1..847cdf175 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -164,6 +164,7 @@ class FrameProcessor(BaseObject): await self._task_manager.wait_for_task(task, timeout) async def cleanup(self): + await super().cleanup() await self.__cancel_input_task() await self.__cancel_push_task() diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index bfb1e8146..937dda80c 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -102,11 +102,13 @@ class FastAPIWebsocketClient: class FastAPIWebsocketInputTransport(BaseInputTransport): def __init__( self, + transport: BaseTransport, client: FastAPIWebsocketClient, params: FastAPIWebsocketParams, **kwargs, ): super().__init__(params, **kwargs) + self._transport = transport self._client = client self._params = params self._receive_task = None @@ -139,6 +141,10 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): await self._stop_tasks() await self._client.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def _receive_messages(self): try: async for message in self._client.receive(): @@ -165,11 +171,14 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): class FastAPIWebsocketOutputTransport(BaseOutputTransport): def __init__( self, + transport: BaseTransport, client: FastAPIWebsocketClient, params: FastAPIWebsocketParams, **kwargs, ): super().__init__(params, **kwargs) + + self._transport = transport self._client = client self._params = params @@ -194,6 +203,10 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport): await super().cancel(frame) await self._client.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -266,6 +279,7 @@ class FastAPIWebsocketTransport(BaseTransport): output_name: Optional[str] = None, ): super().__init__(input_name=input_name, output_name=output_name) + self._params = params self._callbacks = FastAPIWebsocketCallbacks( @@ -278,10 +292,10 @@ class FastAPIWebsocketTransport(BaseTransport): self._client = FastAPIWebsocketClient(websocket, is_binary, self._callbacks) self._input = FastAPIWebsocketInputTransport( - self._client, self._params, name=self._input_name + self, self._client, self._params, name=self._input_name ) self._output = FastAPIWebsocketOutputTransport( - self._client, self._params, name=self._output_name + self, self._client, self._params, name=self._output_name ) # Register supported handlers. The user will only be able to register diff --git a/src/pipecat/transports/network/websocket_client.py b/src/pipecat/transports/network/websocket_client.py index eb2b5cfb8..11a000e69 100644 --- a/src/pipecat/transports/network/websocket_client.py +++ b/src/pipecat/transports/network/websocket_client.py @@ -118,9 +118,15 @@ class WebsocketClientSession: class WebsocketClientInputTransport(BaseInputTransport): - def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams): + def __init__( + self, + transport: BaseTransport, + session: WebsocketClientSession, + params: WebsocketClientParams, + ): super().__init__(params) + self._transport = transport self._session = session self._params = params @@ -138,6 +144,10 @@ class WebsocketClientInputTransport(BaseInputTransport): await super().cancel(frame) await self._session.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def on_message(self, websocket, message): frame = await self._params.serializer.deserialize(message) if not frame: @@ -149,9 +159,15 @@ class WebsocketClientInputTransport(BaseInputTransport): class WebsocketClientOutputTransport(BaseOutputTransport): - def __init__(self, session: WebsocketClientSession, params: WebsocketClientParams): + def __init__( + self, + transport: BaseTransport, + session: WebsocketClientSession, + params: WebsocketClientParams, + ): super().__init__(params) + self._transport = transport self._session = session self._params = params @@ -178,6 +194,10 @@ class WebsocketClientOutputTransport(BaseOutputTransport): await super().cancel(frame) await self._session.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): await self._write_frame(frame) @@ -250,12 +270,12 @@ class WebsocketClientTransport(BaseTransport): def input(self) -> WebsocketClientInputTransport: if not self._input: - self._input = WebsocketClientInputTransport(self._session, self._params) + self._input = WebsocketClientInputTransport(self, self._session, self._params) return self._input def output(self) -> WebsocketClientOutputTransport: if not self._output: - self._output = WebsocketClientOutputTransport(self._session, self._params) + self._output = WebsocketClientOutputTransport(self, self._session, self._params) return self._output async def _on_connected(self, websocket): diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 19ebe4a45..e542342a2 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -55,6 +55,7 @@ class WebsocketServerCallbacks(BaseModel): class WebsocketServerInputTransport(BaseInputTransport): def __init__( self, + transport: BaseTransport, host: str, port: int, params: WebsocketServerParams, @@ -63,6 +64,7 @@ class WebsocketServerInputTransport(BaseInputTransport): ): super().__init__(params, **kwargs) + self._transport = transport self._host = host self._port = port self._params = params @@ -102,6 +104,10 @@ class WebsocketServerInputTransport(BaseInputTransport): await self.cancel_task(self._server_task) self._server_task = None + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def _server_task_handler(self): logger.info(f"Starting websocket server on {self._host}:{self._port}") async with websockets.serve(self._client_handler, self._host, self._port) as server: @@ -163,9 +169,10 @@ class WebsocketServerInputTransport(BaseInputTransport): class WebsocketServerOutputTransport(BaseOutputTransport): - def __init__(self, params: WebsocketServerParams, **kwargs): + def __init__(self, transport: BaseTransport, params: WebsocketServerParams, **kwargs): super().__init__(params, **kwargs) + self._transport = transport self._params = params self._websocket: Optional[websockets.WebSocketServerProtocol] = None @@ -189,6 +196,10 @@ class WebsocketServerOutputTransport(BaseOutputTransport): await self._params.serializer.setup(frame) self._send_interval = (self._audio_chunk_size / self.sample_rate) / 2 + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -283,13 +294,15 @@ class WebsocketServerTransport(BaseTransport): def input(self) -> WebsocketServerInputTransport: if not self._input: self._input = WebsocketServerInputTransport( - self._host, self._port, self._params, self._callbacks, name=self._input_name + self, self._host, self._port, self._params, self._callbacks, name=self._input_name ) return self._input def output(self) -> WebsocketServerOutputTransport: if not self._output: - self._output = WebsocketServerOutputTransport(self._params, name=self._output_name) + self._output = WebsocketServerOutputTransport( + self, self._params, name=self._output_name + ) return self._output async def _on_client_connected(self, websocket): diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 873b13cd1..f4b83dfa7 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -811,9 +811,16 @@ class DailyInputTransport(BaseInputTransport): params: Configuration parameters. """ - def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs): + def __init__( + self, + transport: BaseTransport, + client: DailyTransportClient, + params: DailyParams, + **kwargs, + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client self._params = params @@ -881,6 +888,7 @@ class DailyInputTransport(BaseInputTransport): async def cleanup(self): await super().cleanup() await self._client.cleanup() + await self._transport.cleanup() # # FrameProcessor @@ -971,9 +979,12 @@ class DailyOutputTransport(BaseOutputTransport): params: Configuration parameters. """ - def __init__(self, client: DailyTransportClient, params: DailyParams, **kwargs): + def __init__( + self, transport: BaseTransport, client: DailyTransportClient, params: DailyParams, **kwargs + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client # Whether we have seen a StartFrame already. @@ -1008,6 +1019,7 @@ class DailyOutputTransport(BaseOutputTransport): async def cleanup(self): await super().cleanup() await self._client.cleanup() + await self._transport.cleanup() async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): await self._client.send_message(frame) @@ -1109,12 +1121,16 @@ class DailyTransport(BaseTransport): def input(self) -> DailyInputTransport: if not self._input: - self._input = DailyInputTransport(self._client, self._params, name=self._input_name) + self._input = DailyInputTransport( + self, self._client, self._params, name=self._input_name + ) return self._input def output(self) -> DailyOutputTransport: if not self._output: - self._output = DailyOutputTransport(self._client, self._params, name=self._output_name) + self._output = DailyOutputTransport( + self, self._client, self._params, name=self._output_name + ) return self._output # diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 7018ea520..149ca4b7c 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -345,9 +345,17 @@ class LiveKitTransportClient: class LiveKitInputTransport(BaseInputTransport): - def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs): + def __init__( + self, + transport: BaseTransport, + client: LiveKitTransportClient, + params: LiveKitParams, + **kwargs, + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client + self._audio_in_task = None self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer self._resampler = create_default_resampler() @@ -377,6 +385,10 @@ class LiveKitInputTransport(BaseInputTransport): if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): await self.cancel_task(self._audio_in_task) + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def push_app_message(self, message: Any, sender: str): frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender) await self.push_frame(frame) @@ -414,8 +426,15 @@ class LiveKitInputTransport(BaseInputTransport): class LiveKitOutputTransport(BaseOutputTransport): - def __init__(self, client: LiveKitTransportClient, params: LiveKitParams, **kwargs): + def __init__( + self, + transport: BaseTransport, + client: LiveKitTransportClient, + params: LiveKitParams, + **kwargs, + ): super().__init__(params, **kwargs) + self._transport = transport self._client = client async def start(self, frame: StartFrame): @@ -433,6 +452,10 @@ class LiveKitOutputTransport(BaseOutputTransport): await super().cancel(frame) await self._client.disconnect() + async def cleanup(self): + await super().cleanup() + await self._transport.cleanup() + async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)): await self._client.send_data(frame.message.encode(), frame.participant_id) @@ -499,13 +522,15 @@ class LiveKitTransport(BaseTransport): def input(self) -> LiveKitInputTransport: if not self._input: - self._input = LiveKitInputTransport(self._client, self._params, name=self._input_name) + self._input = LiveKitInputTransport( + self, self._client, self._params, name=self._input_name + ) return self._input def output(self) -> LiveKitOutputTransport: if not self._output: self._output = LiveKitOutputTransport( - self._client, self._params, name=self._output_name + self, self._client, self._params, name=self._output_name ) return self._output diff --git a/src/pipecat/utils/base_object.py b/src/pipecat/utils/base_object.py index e51eac35d..1dee24ce7 100644 --- a/src/pipecat/utils/base_object.py +++ b/src/pipecat/utils/base_object.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import asyncio import inspect from abc import ABC from typing import Optional @@ -17,8 +18,15 @@ class BaseObject(ABC): def __init__(self, *, name: Optional[str] = None): self._id: int = obj_id() self._name = name or f"{self.__class__.__name__}#{obj_count(self)}" + + # Registered event handlers. self._event_handlers: dict = {} + # Set of tasks being executed. When a task finishes running it gets + # automatically removed from the set. When we cleanup we wait for all + # event tasks still being executed. + self._event_tasks = set() + @property def id(self) -> int: return self._id @@ -27,6 +35,12 @@ class BaseObject(ABC): def name(self) -> str: return self._name + async def cleanup(self): + if self._event_tasks: + event_names, tasks = zip(*self._event_tasks) + logger.debug(f"{self} wating on event handlers to finish {list(event_names)}...") + await asyncio.wait(tasks) + def event_handler(self, event_name: str): def decorator(handler): self.add_event_handler(event_name, handler) @@ -45,6 +59,16 @@ class BaseObject(ABC): self._event_handlers[event_name] = [] async def _call_event_handler(self, event_name: str, *args, **kwargs): + # Create the task. + task = asyncio.create_task(self._run_task(event_name, *args, **kwargs)) + + # Add it to our list of event tasks. + self._event_tasks.add((event_name, task)) + + # Remove the task from the event tasks list when the task completes. + task.add_done_callback(self._event_task_finished) + + async def _run_task(self, event_name: str, *args, **kwargs): try: for handler in self._event_handlers[event_name]: if inspect.iscoroutinefunction(handler): @@ -54,5 +78,10 @@ class BaseObject(ABC): except Exception as e: logger.exception(f"Exception in event handler {event_name}: {e}") + def _event_task_finished(self, task: asyncio.Task): + tuple_to_remove = next((t for t in self._event_tasks if t[1] == task), None) + if tuple_to_remove: + self._event_tasks.discard(tuple_to_remove) + def __str__(self): return self.name From 5dc8b48fbe353b97048a018320eaa5f83fe3dda4 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 18 Mar 2025 09:42:03 -0400 Subject: [PATCH 325/427] Fix an issue where GoogleSTTService would timeout due to stream inactivity --- CHANGELOG.md | 6 ++++++ src/pipecat/services/google/google.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d200ab87..7afea60be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -122,6 +122,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue with the `GoogleSTTService` where stream timeouts during + periods of inactivity were causing connection failures. The service now + properly detects timeout errors and handles reconnection gracefully, + ensuring continuous operation even after periods of silence or when using an + `STTMuteFilter`. + - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 9d4a0a8a8..b2af64b57 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1972,7 +1972,8 @@ class GoogleSTTService(STTService): break except Exception as e: - logger.error(f"Stream error, attempting to reconnect: {e}") + logger.warning(f"{self} Reconnecting: {e}") + await asyncio.sleep(1) # Brief delay before reconnecting self._stream_start_time = int(time.time() * 1000) continue @@ -2025,3 +2026,6 @@ class GoogleSTTService(STTService): except Exception as e: logger.error(f"Error processing Google STT responses: {e}") + + # Re-raise the exception to let it propagate (e.g. in the case of a timeout, propagate to _stream_audio to reconnect) + raise From 48b6850df43d8bf6f74f72845f85188a0e2e153a Mon Sep 17 00:00:00 2001 From: balalo Date: Tue, 18 Mar 2025 20:45:31 +0100 Subject: [PATCH 326/427] allow other function names --- src/pipecat/frames/frames.py | 2 +- src/pipecat/processors/aggregators/llm_response.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 7b07a331e..734f0ea6a 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -367,7 +367,7 @@ class LLMSetToolsFrame(DataFrame): class LLMSetToolChoiceFrame(DataFrame): """A frame containing a tool choice for an LLM to use for function calling.""" - tool_choice: Literal["none", "auto", "required"] + tool_choice: Literal["none", "auto", "required"] | str @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 9c18e67ca..44d41f535 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -253,7 +253,7 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools: List): self._context.set_tools(tools) - def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"]): + def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | str): self._context.set_tool_choice(tool_choice) def reset(self): From 514ecda7552c1d26a648477de1d8001e498c8615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 17:31:01 -0700 Subject: [PATCH 327/427] TTSService: allow passing multiple text filters and aggregators --- CHANGELOG.md | 11 +++- .../foundational/14j-function-calling-nim.py | 2 +- .../35-pattern-pair-voice-switching.py | 2 +- examples/news-chatbot/server/news_bot.py | 2 +- src/pipecat/services/ai_services.py | 64 ++++++++++++++----- 5 files changed, 61 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c29b8e98..f90dd80dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new `BaseTextAggregator`. Text aggregators are used by the TTS service to aggregate LLM tokens and decide when the aggregated text should be pushed - to the TTS service. It also allows for the text to be manipulated while it's - being aggregated. + to the TTS service. They also allow for the text to be manipulated while it's + being aggregated. Multiple text aggregators can be passed with + `text_aggregators` to the TTS service. - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) @@ -113,6 +114,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default mode for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. +### Deprecated + +- `TTSService` parameter `text_filter` is now deprecated, use `text_filters` + instead which is now a list. This allows passing multiple filters that will be + executed in order. + ### Removed - Removed deprecated `audio.resample_audio()`, use `create_default_resampler()` diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index d703d637a..ea8e25cf6 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -60,7 +60,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - # text_filter=MarkdownTextFilter(), + # text_filters=[MarkdownTextFilter()], ) llm = NimLLMService( diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index bb9587706..7d0094132 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -119,7 +119,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id=VOICE_IDS["narrator"], - text_aggregator=pattern_aggregator, + text_aggregators=[pattern_aggregator], ) # Initialize LLM diff --git a/examples/news-chatbot/server/news_bot.py b/examples/news-chatbot/server/news_bot.py index 2a389094e..b9f60200f 100644 --- a/examples/news-chatbot/server/news_bot.py +++ b/examples/news-chatbot/server/news_bot.py @@ -97,7 +97,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - text_filter=MarkdownTextFilter(), + text_filters=[MarkdownTextFilter()], ) llm = GoogleLLMService( diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 3fe33d69e..904e5cf90 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -8,7 +8,7 @@ import asyncio import io import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple, Type +from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple, Type from loguru import logger @@ -239,8 +239,9 @@ class TTSService(AIService): # TTS output sample rate sample_rate: Optional[int] = None, # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. - text_aggregator: Optional[BaseTextAggregator] = None, + text_aggregators: Sequence[BaseTextAggregator] = [], # Text filter executed after text has been aggregated. + text_filters: Sequence[BaseTextFilter] = [], text_filter: Optional[BaseTextFilter] = None, **kwargs, ): @@ -256,8 +257,21 @@ class TTSService(AIService): self._sample_rate = 0 self._voice_id: str = "" self._settings: Dict[str, Any] = {} - self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() - self._text_filter: Optional[BaseTextFilter] = text_filter + # Ensure there's at least one text aggregator. + self._text_aggregators: Sequence[BaseTextAggregator] = text_aggregators or [ + SimpleTextAggregator() + ] + self._text_filters: Sequence[BaseTextFilter] = text_filters + if text_filter: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'text_filter' is deprecated, use 'text_filters' instead.", + DeprecationWarning, + ) + self._text_filters = [text_filter] self._stop_frame_task: Optional[asyncio.Task] = None self._stop_frame_queue: asyncio.Queue = asyncio.Queue() @@ -317,8 +331,9 @@ class TTSService(AIService): self.set_model_name(value) elif key == "voice": self.set_voice(value) - elif key == "text_filter" and self._text_filter: - self._text_filter.update_settings(value) + elif key == "text_filter": + for filter in self._text_filters: + filter.update_settings(value) else: logger.warning(f"Unknown setting for TTS service: {key}") @@ -343,8 +358,8 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._text_aggregator.text - self._text_aggregator.reset() + sentence = self._text_aggregators[-1].text + self._reset_aggregators() self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): @@ -390,9 +405,10 @@ class TTSService(AIService): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): self._processing_text = False - self._text_aggregator.handle_interruption() - if self._text_filter: - self._text_filter.handle_interruption() + for aggregator in self._text_aggregators: + aggregator.handle_interruption() + for filter in self._text_filters: + filter.handle_interruption() async def _maybe_pause_frame_processing(self): if self._processing_text and self._pause_frame_processing: @@ -402,12 +418,25 @@ class TTSService(AIService): if self._pause_frame_processing: await self.resume_processing_frames() + def _reset_aggregators(self): + for aggregator in self._text_aggregators: + aggregator.reset() + async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None if not self._aggregate_sentences: text = frame.text else: - text = self._text_aggregator.aggregate(frame.text) + current_text = frame.text + + # Process all aggregators except the last one. + for aggregator in self._text_aggregators[:-1]: + aggregator.aggregate(current_text) + current_text = aggregator.text + + # The last aggregator decides whether we are sending text to the + # TTS or not. + text = self._text_aggregators[-1].aggregate(current_text) if text: await self._push_tts_frames(text) @@ -427,11 +456,16 @@ class TTSService(AIService): self._processing_text = True await self.start_processing_metrics() - if self._text_filter: - self._text_filter.reset_interruption() - text = self._text_filter.filter(text) + + # Process all filter. + for filter in self._text_filters: + filter.reset_interruption() + text = filter.filter(text) + await self.process_generator(self.run_tts(text)) + await self.stop_processing_metrics() + if self._push_text_frames: # We send the original text after the audio. This way, if we are # interrupted, the text is not added to the assistant context. From 7f1ccab445c4246a0e0758dab7da3c2fcd602048 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 19 Mar 2025 07:07:45 -0400 Subject: [PATCH 328/427] Fix: RTVI message disconnect-bot now pushes EndTaskFrame --- CHANGELOG.md | 4 ++++ src/pipecat/processors/frameworks/rtvi.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f90dd80dc..249474080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue where the RTVI message `disconnect-bot` was pushing an + `EndFrame`, resulting in the pipeline not shutting down. It now pushes an + `EndTaskFrame` upstream to shutdown the pipeline. + - Fixed an issue with the `GoogleSTTService` where stream timeouts during periods of inactivity were causing connection failures. The service now properly detects timeout errors and handles reconnection gracefully, diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 29747a582..bb97c2098 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( CancelFrame, DataFrame, EndFrame, + EndTaskFrame, ErrorFrame, Frame, FunctionCallResultFrame, @@ -766,7 +767,7 @@ class RTVIProcessor(FrameProcessor): update_config = RTVIUpdateConfig.model_validate(message.data) await self._handle_update_config(message.id, update_config) case "disconnect-bot": - await self.push_frame(EndFrame()) + await self.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM) case "action": action = RTVIActionRun.model_validate(message.data) action_frame = RTVIActionFrame(message_id=message.id, rtvi_action_run=action) From 5f28834588dba5ddd1f8466bab598886d5b6664f Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:49:51 +0100 Subject: [PATCH 329/427] feature: add custom headers to AsyncOpenAI --- src/pipecat/services/openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 5a3a993aa..feddc8c44 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -116,6 +116,7 @@ class BaseOpenAILLMService(LLMService): base_url=None, organization=None, project=None, + default_headers: Mapping[str, str] | None = None, params: InputParams = InputParams(), **kwargs, ): @@ -132,10 +133,10 @@ class BaseOpenAILLMService(LLMService): } self.set_model_name(model) self._client = self.create_client( - api_key=api_key, base_url=base_url, organization=organization, project=project, **kwargs + api_key=api_key, base_url=base_url, organization=organization, project=project, default_headers=default_headers, **kwargs ) - def create_client(self, api_key=None, base_url=None, organization=None, project=None, **kwargs): + def create_client(self, api_key=None, base_url=None, organization=None, project=None, default_headers=None, **kwargs): return AsyncOpenAI( api_key=api_key, base_url=base_url, @@ -146,6 +147,7 @@ class BaseOpenAILLMService(LLMService): max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None ) ), + default_headers=default_headers ) def can_generate_metrics(self) -> bool: From 1dbad2326aeeb2fccda94e392be6cdd8393dba4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 17:23:06 -0800 Subject: [PATCH 330/427] utils(string): support email addresses in end of sentence matching --- CHANGELOG.md | 3 +++ src/pipecat/utils/string.py | 19 ++++++++++++++++++- tests/test_utils_string.py | 4 ++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 249474080..6e626a7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `match_endofsentence` issue that would result in emails to be + considered an end of sentence. + - Fixed an issue where the RTVI message `disconnect-bot` was pushing an `EndFrame`, resulting in the pipeline not shutting down. It now pushes an `EndTaskFrame` upstream to shutdown the pipeline. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 154d03174..06f2fc175 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -17,9 +17,26 @@ ENDOFSENTENCE_PATTERN_STR = r""" (\。\s*\。\s*\。|[。？！；।]) # the full-width version (mainly used in East Asian languages such as Chinese, Hindi) $ # End of string """ + ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE) +EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") + def match_endofsentence(text: str) -> int: - match = ENDOFSENTENCE_PATTERN.search(text.rstrip()) + text = text.rstrip() + + # Find all emails. + emails = list(EMAIL_PATTERN.finditer(text)) + + # Replace email dots by ampersands so we can find the end of sentence. + for email_match in emails: + start = email_match.start() + end = email_match.end() + new_email = text[start:end].replace(".", "&") + text = text[:start] + new_email + text[end:] + + # Match against the new text. + match = ENDOFSENTENCE_PATTERN.search(text) + return match.end() if match else 0 diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index e4cdb4cb4..24519f724 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -18,6 +18,9 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is a sentence...") == 21 assert match_endofsentence("This is a sentence . . .") == 24 assert match_endofsentence("This is a sentence. ..") == 22 + assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31 + assert match_endofsentence("U.S.A and U.S.A..") == 17 + assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") @@ -28,6 +31,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert not match_endofsentence("Heute ist Dienstag, der 3.") # 3. Juli 2024 assert not match_endofsentence("America, or the U.") # U.S.A. assert not match_endofsentence("It still early, it's 3:00 a.") # 3:00 a.m. + assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai") async def test_endofsentence_zh(self): chinese_sentences = [ From 11984b89b79661d6d7ae2e0987446434dfe8021b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 18:22:37 -0800 Subject: [PATCH 331/427] utils(string): add support for floating point numbers --- CHANGELOG.md | 3 +++ src/pipecat/utils/string.py | 28 ++++++++++++++++++++-------- tests/test_utils_string.py | 6 ++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e626a7b1..c530b0986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `match_endofsentence` issue that would result in floating point + numbers to be considered an end of sentence. + - Fixed a `match_endofsentence` issue that would result in emails to be considered an end of sentence. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 06f2fc175..a89127de9 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -8,7 +8,8 @@ import re ENDOFSENTENCE_PATTERN_STR = r""" (? str: + start = match.start() + end = match.end() + replacement = text[start:end].replace(old, new) + text = text[:start] + replacement + text[end:] + return text + def match_endofsentence(text: str) -> int: text = text.rstrip() - # Find all emails. + # Replace email dots by ampersands so we can find the end of sentence. For + # example, first.last@email.com becomes first&last@email&com. emails = list(EMAIL_PATTERN.finditer(text)) - - # Replace email dots by ampersands so we can find the end of sentence. for email_match in emails: - start = email_match.start() - end = email_match.end() - new_email = text[start:end].replace(".", "&") - text = text[:start] + new_email + text[end:] + text = replace_match(text, email_match, ".", "&") + + # Replace number dots by ampersands so we can find the end of sentence. + numbers = list(NUMBER_PATTERN.finditer(text)) + for number_match in numbers: + text = replace_match(text, number_match, ".", "&") # Match against the new text. match = ENDOFSENTENCE_PATTERN.search(text) diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index 24519f724..ee0946d69 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -21,6 +21,11 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31 assert match_endofsentence("U.S.A and U.S.A..") == 17 assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 + assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 31 + assert match_endofsentence("My email is spell(foo.bar@pipecat.ai).") == 38 + assert match_endofsentence("The number pi is 3.14159.") == 25 + assert match_endofsentence("Valid scientific notation 1.23e4.") == 33 + assert match_endofsentence("Valid scientific notation 0.e4.") == 31 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") @@ -32,6 +37,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert not match_endofsentence("America, or the U.") # U.S.A. assert not match_endofsentence("It still early, it's 3:00 a.") # 3:00 a.m. assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai") + assert not match_endofsentence("The number pi is 3.14159") async def test_endofsentence_zh(self): chinese_sentences = [ From 1a3a268c9de7d3a0d59ea43da1cc7718c9c430bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 28 Feb 2025 17:23:54 -0800 Subject: [PATCH 332/427] utils(string): add new function parse_start_end_tags() --- src/pipecat/utils/string.py | 76 +++++++++++++++++++++++++++++++++++++ tests/test_utils_string.py | 50 +++++++++++++++++++++++- 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index a89127de9..69036a665 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -5,6 +5,7 @@ # import re +from typing import Optional, Sequence, Tuple ENDOFSENTENCE_PATTERN_STR = r""" (? str: + """Replace occurrences of a substring within a matched section of a given + text. + + Args: + text (str): The input text in which replacements will be made. + match (re.Match): A regex match object representing the section of text to modify. + old (str): The substring to be replaced. + new (str): The substring to replace `old` with. + + Returns: + str: The modified text with the specified replacements made within the matched section. + + """ start = match.start() end = match.end() replacement = text[start:end].replace(old, new) @@ -35,6 +51,20 @@ def replace_match(text: str, match: re.Match, old: str, new: str) -> str: def match_endofsentence(text: str) -> int: + """Finds the position of the end of a sentence in the provided text string. + + This function processes the input text by replacing periods in email + addresses and numbers with ampersands to prevent them from being + misidentified as sentence terminals. It then searches for the end of a + sentence using a specified regex pattern. + + Args: + text (str): The input text in which to find the end of the sentence. + + Returns: + int: The position of the end of the sentence if found, otherwise 0. + + """ text = text.rstrip() # Replace email dots by ampersands so we can find the end of sentence. For @@ -52,3 +82,49 @@ def match_endofsentence(text: str) -> int: match = ENDOFSENTENCE_PATTERN.search(text) return match.end() if match else 0 + + +def parse_start_end_tags( + text: str, + tags: Sequence[StartEndTags], + current_tag: Optional[StartEndTags], + current_tag_index: int, +) -> Tuple[Optional[StartEndTags], int]: + """Parses the given text to identify a pair of start/end tags. + + If a start tag was previously found (i.e. current_tags is valid), wait for + the corresponding end tag. Otherwise, wait for a start tag. + + This function will return the index in the text that we should start parsing + in the next call and the current or new tags. + + Parameters: + - text (str): The text to be parsed. + - tags (Sequence[StartEndTags]): List of tuples containing start and end tags. + - current_tags (Optional[StartEndTags]): The currently active tags, if any. + - current_tags_index (int): The current index in the text. + + Returns: + Tuple[Optional[StartEndTags], int]: A tuple containing None or the current + tag and the index of the text. + + """ + # If we are already inside a tag, check if the end tag is in the text. + if current_tag: + _, end_tag = current_tag + if end_tag in text[current_tag_index:]: + return (None, len(text)) + return (current_tag, current_tag_index) + + # Check if any start tag appears in the text + for start_tag, end_tag in tags: + start_tag_count = text[current_tag_index:].count(start_tag) + end_tag_count = text[current_tag_index:].count(end_tag) + if start_tag_count == 0 and end_tag_count == 0: + return (None, current_tag_index) + elif start_tag_count > end_tag_count: + return ((start_tag, end_tag), len(text)) + elif start_tag_count == end_tag_count: + return (None, len(text)) + + return (None, current_tag_index) diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index ee0946d69..cabd88a36 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -6,7 +6,7 @@ import unittest -from pipecat.utils.string import match_endofsentence +from pipecat.utils.string import match_endofsentence, parse_start_end_tags class TestUtilsString(unittest.IsolatedAsyncioTestCase): @@ -23,6 +23,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 31 assert match_endofsentence("My email is spell(foo.bar@pipecat.ai).") == 38 + assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 46 assert match_endofsentence("The number pi is 3.14159.") == 25 assert match_endofsentence("Valid scientific notation 1.23e4.") == 33 assert match_endofsentence("Valid scientific notation 0.e4.") == 31 @@ -60,3 +61,50 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): for i in hindi_sentences: assert match_endofsentence(i) assert not match_endofsentence("हैलो，") + + +class TestStartEndTags(unittest.IsolatedAsyncioTestCase): + async def test_empty(self): + assert parse_start_end_tags("", [], None, 0) == (None, 0) + assert parse_start_end_tags("Hello from Pipecat!", [], None, 0) == (None, 0) + + async def test_simple(self): + # (, ) + assert parse_start_end_tags("Hello from Pipecat!", [("", "")], None, 0) == ( + None, + 26, + ) + assert parse_start_end_tags("Hello from Pipecat", [("", "")], None, 0) == ( + ("", ""), + 21, + ) + assert parse_start_end_tags("Hello from Pipecat", [("", "")], None, 6) == ( + ("", ""), + 21, + ) + + # (spell(, )) + assert parse_start_end_tags("Hello from spell(Pipecat)!", [("spell(", ")")], None, 0) == ( + None, + 26, + ) + assert parse_start_end_tags("Hello from spell(Pipecat", [("spell(", ")")], None, 0) == ( + ("spell(", ")"), + 24, + ) + + async def test_multiple(self): + # (, ) + assert parse_start_end_tags( + "Hello from Pipecat! Hello World!", [("", "")], None, 0 + ) == ( + None, + 46, + ) + + assert parse_start_end_tags( + "Hello from Pipecat! Hello World", [("", "")], None, 0 + ) == ( + ("", ""), + 41, + ) From e7224473f2728e37c95aee52d48b9099a16bbdc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 23:04:14 -0700 Subject: [PATCH 333/427] utils(text): add new SkipTagsAggregator --- CHANGELOG.md | 4 + .../utils/text/skip_tags_aggregator.py | 94 +++++++++++++++++++ tests/test_skip_tags_aggregator.py | 54 +++++++++++ 3 files changed, 152 insertions(+) create mode 100644 src/pipecat/utils/text/skip_tags_aggregator.py create mode 100644 tests/test_skip_tags_aggregator.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c530b0986..5abdf4ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added new `SkipTagsAggregator` that extends `BaseTextAggregator` to aggregate + text and skips end of sentence matching if aggregated text is between + start/end tags. + - Added new `PatternPairAggregator` that extends `BaseTextAggregator` to identify content between matching pattern pairs in streamed text. This allows for detection and processing of structured content like XML-style tags that diff --git a/src/pipecat/utils/text/skip_tags_aggregator.py b/src/pipecat/utils/text/skip_tags_aggregator.py new file mode 100644 index 000000000..00129028e --- /dev/null +++ b/src/pipecat/utils/text/skip_tags_aggregator.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Optional, Sequence + +from pipecat.utils.string import StartEndTags, match_endofsentence, parse_start_end_tags +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator + + +class SkipTagsAggregator(BaseTextAggregator): + """Aggregator that prevents end of sentence matching between start/end tags. + + This aggregator buffers text until it finds an end of sentence or a start + tag. If a start tag is found the aggregator will keep aggregating text + unconditionally until the corresponding end tag is found. It's particularly + useful for processing content with custom delimiters that should prevent + text from being considered for end of sentence matching.. + + The aggregator ensures that tags spanning multiple text chunks are correctly + identified. + + """ + + def __init__(self, tags: Sequence[StartEndTags]): + """Initialize the pattern pair aggregator. + + Creates an empty aggregator with no patterns or handlers registered. + """ + self._text = "" + self._tags = tags + self._current_tag: Optional[StartEndTags] = None + self._current_tag_index: int = 0 + + @property + def text(self) -> str: + """Get the currently buffered text. + + Returns: + The current text buffer content. + """ + return self._text + + def aggregate(self, text: str) -> Optional[str]: + """Aggregate text and process pattern pairs. + + This method adds the new text to the buffer, processes any complete pattern + pairs, and returns processed text up to sentence boundaries if possible. + If there are incomplete patterns (start without matching end), it will + continue buffering text. + + Args: + text: New text to add to the buffer. + + Returns: + Processed text up to a sentence boundary, or None if more + text is needed to form a complete sentence or pattern. + """ + # Add new text to buffer + self._text += text + + (self._current_tag, self._current_tag_index) = parse_start_end_tags( + self._text, self._tags, self._current_tag, self._current_tag_index + ) + + # Find sentence boundary if no incomplete patterns + if not self._current_tag: + eos_marker = match_endofsentence(self._text) + if eos_marker: + # Extract text up to the sentence boundary + result = self._text[:eos_marker] + self._text = self._text[eos_marker:] + return result + + # No complete sentence found yet + return None + + def handle_interruption(self): + """Handle interruptions by clearing the buffer. + + Called when an interruption occurs in the processing pipeline, + to reset the state and discard any partially aggregated text. + """ + self._text = "" + + def reset(self): + """Clear the internally aggregated text. + + Resets the aggregator to its initial state, discarding any + buffered text. + """ + self._text = "" diff --git a/tests/test_skip_tags_aggregator.py b/tests/test_skip_tags_aggregator.py new file mode 100644 index 000000000..8f36c4c05 --- /dev/null +++ b/tests/test_skip_tags_aggregator.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import unittest + +from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator + + +class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.aggregator = SkipTagsAggregator([("", "")]) + + async def test_no_tags(self): + self.aggregator.reset() + + # No tags involved, aggregate at end of sentence. + result = self.aggregator.aggregate("Hello Pipecat!") + self.assertEqual(result, "Hello Pipecat!") + self.assertEqual(self.aggregator.text, "") + + async def test_basic_tags(self): + self.aggregator.reset() + + # Tags involved, avoid aggregation during tags. + result = self.aggregator.aggregate("My email is foo@pipecat.ai.") + self.assertEqual(result, "My email is foo@pipecat.ai.") + self.assertEqual(self.aggregator.text, "") + + async def test_streaming_tags(self): + self.aggregator.reset() + + # Tags involved, stream small chunk of texts. + result = self.aggregator.aggregate("My email is foo.") + self.assertIsNone(result) + self.assertEqual(self.aggregator.text, "My email is foo.") + + result = self.aggregator.aggregate("bar@pipecat.") + self.assertIsNone(result) + self.assertEqual(self.aggregator.text, "My email is foo.bar@pipecat.") + + result = self.aggregator.aggregate("aifoo.bar@pipecat.ai.") + self.assertEqual(result, "My email is foo.bar@pipecat.ai.") + self.assertEqual(self.aggregator.text, "") From 54620133d4473e3f8d10d26112fbf002ece6b8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 23:08:34 -0700 Subject: [PATCH 334/427] services: add spelling out support to CartesiaTTSService and RimeTTSService --- CHANGELOG.md | 3 +++ src/pipecat/services/cartesia.py | 6 +++++- src/pipecat/services/rime.py | 6 +++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5abdf4ef5..3006ed5ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `CartesiaTTSService` and `RimeTTSService` issue that would consider + text between spelling out tags end of sentence. + - Fixed a `match_endofsentence` issue that would result in floating point numbers to be considered an end of sentence. diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 8b7f57c63..5a795d750 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, List, Optional, Union +from typing import AsyncGenerator, List, Optional, Sequence, Union from loguru import logger from pydantic import BaseModel @@ -26,6 +26,8 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator # See .env.example for Cartesia configuration needed try: @@ -89,6 +91,7 @@ class CartesiaTTSService(AudioContextWordTTSService): encoding: str = "pcm_s16le", container: str = "raw", params: InputParams = InputParams(), + text_aggregators: Sequence[BaseTextAggregator] = [], **kwargs, ): # Aggregating sentences still gives cleaner-sounding results and fewer @@ -106,6 +109,7 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, + text_aggregators=text_aggregators or [SkipTagsAggregator([("", "")])], **kwargs, ) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index b2610b06c..471f82d66 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, Optional +from typing import AsyncGenerator, Optional, Sequence import aiohttp from loguru import logger @@ -27,6 +27,8 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language +from pipecat.utils.text.base_text_aggregator import BaseTextAggregator +from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator try: import websockets @@ -78,6 +80,7 @@ class RimeTTSService(AudioContextWordTTSService): model: str = "mistv2", sample_rate: Optional[int] = None, params: InputParams = InputParams(), + text_aggregators: Sequence[BaseTextAggregator] = [], **kwargs, ): """Initialize Rime TTS service. @@ -97,6 +100,7 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, + text_aggregators=text_aggregators or [SkipTagsAggregator([("spell(", ")")])], **kwargs, ) From fc0f404d26ddb8f7cb589c2f84c2c3ef96885f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 26 Feb 2025 17:23:58 -0800 Subject: [PATCH 335/427] examples: add new 36-user-email-gathering.py --- CHANGELOG.md | 4 + .../foundational/36-user-email-gathering.py | 141 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 examples/foundational/36-user-email-gathering.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3006ed5ca..64f342dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Added a new example `examples/foundational/36-user-email-gathering.py` to show + how to gather user emails. The example uses's Cartesia's `` + tags and Rime `spell()` function to spell out the emails for confirmation. + - Update the `34-audio-recording.py` example to include an STT processor. - Added foundational example `35-voice-switching.py` showing how to use the new diff --git a/examples/foundational/36-user-email-gathering.py b/examples/foundational/36-user-email-gathering.py new file mode 100644 index 000000000..0e76826e3 --- /dev/null +++ b/examples/foundational/36-user-email-gathering.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMContext, OpenAILLMService +from pipecat.services.rime import RimeHttpTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def store_user_emails(function_name, tool_call_id, args, llm, context, result_callback): + print(f"User emails: {args}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + # Cartesia offers a `` tags that we can use to ask the user + # to confirm the emails. + # (see https://docs.cartesia.ai/build-with-sonic/formatting-text-for-sonic/spelling-out-input-text) + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + aiohttp_session=session, + ) + + # Rime offers a function `spell()` that we can use to ask the user + # to confirm the emails. + # (see https://docs.rime.ai/api-reference/spell) + # tts = RimeHttpTTSService( + # api_key=os.getenv("RIME_API_KEY", ""), + # voice_id="eva", + # aiohttp_session=session, + # ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + # You can aslo register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function("store_user_emails", store_user_emails) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "store_user_emails", + "description": "Store user emails when confirmed", + "parameters": { + "type": "object", + "properties": { + "emails": { + "type": "array", + "description": "The list of user emails", + "items": {"type": "string"}, + }, + }, + "required": ["emails"], + }, + }, + ) + ] + messages = [ + { + "role": "system", + # Cartesia + "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with tags, for example a@a.com.", + # Rime spell() + # "content": "You need to gather a valid email or emails from the user. Your output will be converted to audio so don't include special characters in your answers. If the user provides one or more email addresses confirm them with the user. Enclose all emails with spell(), for example spell(a@a.com).", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 336e2f1579d53fe8d6c0fa336fc91546cc74618c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 11:02:29 -0700 Subject: [PATCH 336/427] TTSServices: for now just specify a single text aggregator --- CHANGELOG.md | 4 +-- .../35-pattern-pair-voice-switching.py | 2 +- src/pipecat/services/ai_services.py | 29 ++++--------------- src/pipecat/services/cartesia.py | 6 ++-- src/pipecat/services/rime.py | 6 ++-- 5 files changed, 15 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64f342dc5..125ca20d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new `BaseTextAggregator`. Text aggregators are used by the TTS service to aggregate LLM tokens and decide when the aggregated text should be pushed to the TTS service. They also allow for the text to be manipulated while it's - being aggregated. Multiple text aggregators can be passed with - `text_aggregators` to the TTS service. + being aggregated. A text aggregator can be passed via `text_aggregator` to the + TTS service. - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) diff --git a/examples/foundational/35-pattern-pair-voice-switching.py b/examples/foundational/35-pattern-pair-voice-switching.py index 7d0094132..bb9587706 100644 --- a/examples/foundational/35-pattern-pair-voice-switching.py +++ b/examples/foundational/35-pattern-pair-voice-switching.py @@ -119,7 +119,7 @@ async def main(): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id=VOICE_IDS["narrator"], - text_aggregators=[pattern_aggregator], + text_aggregator=pattern_aggregator, ) # Initialize LLM diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 904e5cf90..eae030b27 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -239,7 +239,7 @@ class TTSService(AIService): # TTS output sample rate sample_rate: Optional[int] = None, # Text aggregator to aggregate incoming tokens and decide when to push to the TTS. - text_aggregators: Sequence[BaseTextAggregator] = [], + text_aggregator: Optional[BaseTextAggregator] = None, # Text filter executed after text has been aggregated. text_filters: Sequence[BaseTextFilter] = [], text_filter: Optional[BaseTextFilter] = None, @@ -257,10 +257,7 @@ class TTSService(AIService): self._sample_rate = 0 self._voice_id: str = "" self._settings: Dict[str, Any] = {} - # Ensure there's at least one text aggregator. - self._text_aggregators: Sequence[BaseTextAggregator] = text_aggregators or [ - SimpleTextAggregator() - ] + self._text_aggregator: BaseTextAggregator = text_aggregator or SimpleTextAggregator() self._text_filters: Sequence[BaseTextFilter] = text_filters if text_filter: import warnings @@ -358,8 +355,8 @@ class TTSService(AIService): # pause to avoid audio overlapping. await self._maybe_pause_frame_processing() - sentence = self._text_aggregators[-1].text - self._reset_aggregators() + sentence = self._text_aggregator.text + self._text_aggregator.reset() self._processing_text = False await self._push_tts_frames(sentence) if isinstance(frame, LLMFullResponseEndFrame): @@ -405,8 +402,7 @@ class TTSService(AIService): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): self._processing_text = False - for aggregator in self._text_aggregators: - aggregator.handle_interruption() + self._text_aggregator.handle_interruption() for filter in self._text_filters: filter.handle_interruption() @@ -418,25 +414,12 @@ class TTSService(AIService): if self._pause_frame_processing: await self.resume_processing_frames() - def _reset_aggregators(self): - for aggregator in self._text_aggregators: - aggregator.reset() - async def _process_text_frame(self, frame: TextFrame): text: Optional[str] = None if not self._aggregate_sentences: text = frame.text else: - current_text = frame.text - - # Process all aggregators except the last one. - for aggregator in self._text_aggregators[:-1]: - aggregator.aggregate(current_text) - current_text = aggregator.text - - # The last aggregator decides whether we are sending text to the - # TTS or not. - text = self._text_aggregators[-1].aggregate(current_text) + text = self._text_aggregator.aggregate(frame.text) if text: await self._push_tts_frames(text) diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 5a795d750..3b491d26b 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, List, Optional, Sequence, Union +from typing import AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel @@ -91,7 +91,7 @@ class CartesiaTTSService(AudioContextWordTTSService): encoding: str = "pcm_s16le", container: str = "raw", params: InputParams = InputParams(), - text_aggregators: Sequence[BaseTextAggregator] = [], + text_aggregator: Optional[BaseTextAggregator] = None, **kwargs, ): # Aggregating sentences still gives cleaner-sounding results and fewer @@ -109,7 +109,7 @@ class CartesiaTTSService(AudioContextWordTTSService): push_text_frames=False, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregators=text_aggregators or [SkipTagsAggregator([("", "")])], + text_aggregator=text_aggregator or SkipTagsAggregator([("", "")]), **kwargs, ) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 471f82d66..c6fc50001 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -7,7 +7,7 @@ import base64 import json import uuid -from typing import AsyncGenerator, Optional, Sequence +from typing import AsyncGenerator, Optional import aiohttp from loguru import logger @@ -80,7 +80,7 @@ class RimeTTSService(AudioContextWordTTSService): model: str = "mistv2", sample_rate: Optional[int] = None, params: InputParams = InputParams(), - text_aggregators: Sequence[BaseTextAggregator] = [], + text_aggregator: Optional[BaseTextAggregator] = None, **kwargs, ): """Initialize Rime TTS service. @@ -100,7 +100,7 @@ class RimeTTSService(AudioContextWordTTSService): push_stop_frames=True, pause_frame_processing=True, sample_rate=sample_rate, - text_aggregators=text_aggregators or [SkipTagsAggregator([("spell(", ")")])], + text_aggregator=text_aggregator or SkipTagsAggregator([("spell(", ")")]), **kwargs, ) From 48d73a263619145159bb05c25d25ce799f960bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 23:53:48 -0700 Subject: [PATCH 337/427] SegmentedSTTService: allow audio to pass-through downstream --- CHANGELOG.md | 3 +++ src/pipecat/services/ai_services.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 125ca20d8..5d30759a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue with `SegmentedSTTService` based services + (e.g. `GroqSTTService`) that was not allow audio to pass-through downstream. + - Fixed a `CartesiaTTSService` and `RimeTTSService` issue that would consider text between spelling out tags end of sentence. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index eae030b27..ba895c53d 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -811,8 +811,6 @@ class STTService(AIService): return await self.process_generator(self.run_stt(frame.audio)) - if self._audio_passthrough: - await self.push_frame(frame, direction) async def process_frame(self, frame: Frame, direction: FrameDirection): """Processes a frame of audio data, either buffering or transcribing it.""" @@ -823,6 +821,8 @@ class STTService(AIService): # push a TextFrame. We also push audio downstream in case someone # else needs it. await self.process_audio_frame(frame, direction) + if self._audio_passthrough: + await self.push_frame(frame, direction) elif isinstance(frame, STTUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, STTMuteFrame): From 8942c2e05359606c8083a22172cc3d5938a2c593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 15:33:33 -0700 Subject: [PATCH 338/427] GeminiMultimodalLiveLLMService: fix duplicated messages in context Fixes #1384 --- CHANGELOG.md | 3 +++ .../services/gemini_multimodal_live/gemini.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d30759a6..7350fe6d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `GeminiMultimodalLiveLLMService` issue that was causing messages to be + duplicated in the context when pushing `LLMMessagesAppendFrame` frames. + - Fixed an issue with `SegmentedSTTService` based services (e.g. `GroqSTTService`) that was not allow audio to pass-through downstream. diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 5fe8a792b..9484fc27d 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -341,10 +341,8 @@ class GeminiMultimodalLiveLLMService(LLMService): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - # logger.debug(f"Processing frame: {frame}") - if isinstance(frame, TranscriptionFrame): - pass + await self.push_frame(frame, direction) elif isinstance(frame, OpenAILLMContextFrame): context: GeminiMultimodalLiveContext = GeminiMultimodalLiveContext.upgrade( frame.context @@ -361,31 +359,35 @@ class GeminiMultimodalLiveLLMService(LLMService): # Support just one tool call per context frame for now tool_result_message = context.messages[-1] await self._tool_result(tool_result_message) - elif isinstance(frame, InputAudioRawFrame): await self._send_user_audio(frame) + await self.push_frame(frame, direction) elif isinstance(frame, InputImageRawFrame): await self._send_user_video(frame) + await self.push_frame(frame, direction) elif isinstance(frame, StartInterruptionFrame): await self._handle_interruption() + await self.push_frame(frame, direction) elif isinstance(frame, UserStartedSpeakingFrame): await self._handle_user_started_speaking(frame) + await self.push_frame(frame, direction) elif isinstance(frame, UserStoppedSpeakingFrame): await self._handle_user_stopped_speaking(frame) + await self.push_frame(frame, direction) elif isinstance(frame, BotStartedSpeakingFrame): # Ignore this frame. Use the serverContent API message instead - pass + await self.push_frame(frame, direction) elif isinstance(frame, BotStoppedSpeakingFrame): # ignore this frame. Use the serverContent.turnComplete API message - pass + await self.push_frame(frame, direction) elif isinstance(frame, LLMMessagesAppendFrame): await self._create_single_response(frame.messages) elif isinstance(frame, LLMUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, LLMSetToolsFrame): await self._update_settings() - - await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) # # websocket communication From f31e77c4f644e302b7f65d70ba44bde8e4ef82af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 18:43:07 -0700 Subject: [PATCH 339/427] pyproject: added empty tavus dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 586c5c026..4176ae2a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ sentry = [ "sentry-sdk~=2.20.0" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] +tavus=[] together = [] ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] From a3b5e4413ac7370031b6e3a6ceb17a2ca5c6da06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 16:00:57 -0700 Subject: [PATCH 340/427] WebsocketTTSService: add `on_connection_error` and `reconnect_on_error` --- CHANGELOG.md | 7 ++++ src/pipecat/pipeline/task.py | 4 ++- src/pipecat/services/ai_services.py | 36 +++++++++++++++++--- src/pipecat/services/cartesia.py | 3 +- src/pipecat/services/elevenlabs.py | 3 +- src/pipecat/services/fish.py | 3 +- src/pipecat/services/lmnt.py | 3 +- src/pipecat/services/neuphonic.py | 3 +- src/pipecat/services/playht.py | 8 +++-- src/pipecat/services/rime.py | 3 +- src/pipecat/services/websocket_service.py | 40 +++++++++++++---------- 11 files changed, 81 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d30759a6..35df891ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a `reconnect_on_error` parameter to websocket-based TTS services as well + as a `on_connection_error` event handler. The `reconnect_on_error` indicates + whether the TTS service should reconnect on error. The `on_connection_error` + will always get called if there's any error no matter the value of + `reconnect_on_error`. This allows, for example, to fallback to a different TTS + provider if something goes wrong with the current one. + - Added new `SkipTagsAggregator` that extends `BaseTextAggregator` to aggregate text and skips end of sentence matching if aggregated text is between start/end tags. diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 12c56ef3f..e9744fd04 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -425,12 +425,14 @@ class PipelineTask(BaseTask): # Tell the task we should stop nicely. await self.queue_frame(StopFrame()) elif isinstance(frame, ErrorFrame): - logger.error(f"Error running app: {frame}") if frame.fatal: + logger.error(f"A fatal error occurred: {frame}") # Cancel all tasks downstream. await self.queue_frame(CancelFrame()) # Tell the task we should stop. await self.queue_frame(StopTaskFrame()) + else: + logger.warning(f"Something went wrong: {frame}") self._up_queue.task_done() async def _process_down_queue(self): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index ba895c53d..d29e0e714 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -549,11 +549,25 @@ class WordTTSService(TTSService): class WebsocketTTSService(TTSService, WebsocketService): - """This is a base class for websocket-based TTS services.""" + """This is a base class for websocket-based TTS services. - def __init__(self, **kwargs): + If an error occurs with the websocket, an "on_connection_error" event will + be triggered: + + @tts.event_handler("on_connection_error") + async def on_connection_error(tts: TTSService, error: str): + ... + + """ + + def __init__(self, *, reconnect_on_error: bool = True, **kwargs): TTSService.__init__(self, **kwargs) - WebsocketService.__init__(self) + WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) + self._register_event_handler("on_connection_error") + + async def _report_error(self, error: ErrorFrame): + await self._call_event_handler("on_connection_error", error.error) + await self.push_error(error) class InterruptibleTTSService(WebsocketTTSService): @@ -590,11 +604,23 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """This is a base class for websocket-based TTS services that support word timestamps. + If an error occurs with the websocket a "on_connection_error" event will be + triggered: + + @tts.event_handler("on_connection_error") + async def on_connection_error(tts: TTSService, error: str): + ... + """ - def __init__(self, **kwargs): + def __init__(self, *, reconnect_on_error: bool = True, **kwargs): WordTTSService.__init__(self, **kwargs) - WebsocketService.__init__(self) + WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) + self._register_event_handler("on_connection_error") + + async def _report_error(self, error: ErrorFrame): + await self._call_event_handler("on_connection_error", error.error) + await self.push_error(error) class InterruptibleWordTTSService(WebsocketWordTTSService): diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 3b491d26b..11796464d 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -187,7 +187,7 @@ class CartesiaTTSService(AudioContextWordTTSService): async def _connect(self): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -207,6 +207,7 @@ class CartesiaTTSService(AudioContextWordTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index c41e5de02..568b9eb64 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -309,7 +309,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) if not self._keepalive_task: self._keepalive_task = self.create_task(self._keepalive_task_handler()) @@ -364,6 +364,7 @@ class ElevenLabsTTSService(InterruptibleWordTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 96968d6e9..9e6a8b91e 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -107,7 +107,7 @@ class FishAudioTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -132,6 +132,7 @@ class FishAudioTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"Fish Audio initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index cfdf8e6cd..d3cc92603 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -112,7 +112,7 @@ class LmntTTSService(InterruptibleTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -147,6 +147,7 @@ class LmntTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): """Disconnect from LMNT websocket.""" diff --git a/src/pipecat/services/neuphonic.py b/src/pipecat/services/neuphonic.py index b935885b6..407e54a83 100644 --- a/src/pipecat/services/neuphonic.py +++ b/src/pipecat/services/neuphonic.py @@ -162,7 +162,7 @@ class NeuphonicTTSService(InterruptibleTTSService): async def _connect(self): await self._connect_websocket() - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) self._keepalive_task = self.create_task(self._keepalive_task_handler()) async def _disconnect(self): @@ -197,6 +197,7 @@ class NeuphonicTTSService(InterruptibleTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 80d313765..75677876f 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -160,7 +160,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): if self._receive_task: @@ -183,12 +183,14 @@ class PlayHTTTSService(InterruptibleTTSService): raise ValueError("WebSocket URL is not a string") self._websocket = await websockets.connect(self._websocket_url) - except ValueError as ve: - logger.error(f"{self} initialization error: {ve}") + except ValueError as e: + logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): try: diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index c6fc50001..a1a455eb5 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -171,7 +171,7 @@ class RimeTTSService(AudioContextWordTTSService): await self._connect_websocket() if not self._receive_task: - self._receive_task = self.create_task(self._receive_task_handler(self.push_error)) + self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): """Close websocket connection and clean up tasks.""" @@ -194,6 +194,7 @@ class RimeTTSService(AudioContextWordTTSService): except Exception as e: logger.error(f"{self} initialization error: {e}") self._websocket = None + await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): """Close websocket connection and reset state.""" diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 19fb4ae01..2e82ddaba 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -19,9 +19,10 @@ from pipecat.utils.network import exponential_backoff_time class WebsocketService(ABC): """Base class for websocket-based services with reconnection logic.""" - def __init__(self): + def __init__(self, *, reconnect_on_error: bool = True, **kwargs): """Initialize websocket attributes.""" self._websocket: Optional[websockets.WebSocketClientProtocol] = None + self._reconnect_on_error = reconnect_on_error async def _verify_connection(self) -> bool: """Verify websocket connection is working. @@ -72,24 +73,29 @@ class WebsocketService(ABC): self._websocket.close_rcvd_then_sent, ) except Exception as e: - retry_count += 1 - if retry_count >= MAX_RETRIES: - message = f"{self} error receiving messages: {e}" - logger.error(message) - await report_error(ErrorFrame(message, fatal=True)) + message = f"{self} error receiving messages: {e}" + logger.error(message) + + if self._reconnect_on_error: + retry_count += 1 + if retry_count >= MAX_RETRIES: + await report_error(ErrorFrame(message, fatal=True)) + break + + logger.warning(f"{self} connection error, will retry: {e}") + await report_error(ErrorFrame(message)) + + try: + if await self._reconnect_websocket(retry_count): + retry_count = 0 # Reset counter on successful reconnection + wait_time = exponential_backoff_time(retry_count) + await asyncio.sleep(wait_time) + except Exception as reconnect_error: + logger.error(f"{self} reconnection failed: {reconnect_error}") + else: + await report_error(ErrorFrame(message)) break - logger.warning(f"{self} connection error, will retry: {e}") - - try: - if await self._reconnect_websocket(retry_count): - retry_count = 0 # Reset counter on successful reconnection - wait_time = exponential_backoff_time(retry_count) - await asyncio.sleep(wait_time) - except Exception as reconnect_error: - logger.error(f"{self} reconnection failed: {reconnect_error}") - continue - @abstractmethod async def _connect(self): """Implement service-specific connection logic. This function will From 07a77e066fa515b7fdf526639b3d3c077ca45dd4 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 19 Mar 2025 23:18:30 -0400 Subject: [PATCH 341/427] Update to Claude 3.7 Sonnet latest in examples --- examples/foundational/14a-function-calling-anthropic.py | 2 +- examples/foundational/14b-function-calling-anthropic-video.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 13505550b..923b1488d 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -57,7 +57,7 @@ async def main(): ) llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20240620" + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" ) llm.register_function("get_weather", get_weather) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 8c49900fa..302722be2 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -67,8 +67,7 @@ async def main(): llm = AnthropicLLMService( api_key=os.getenv("ANTHROPIC_API_KEY"), - # model="claude-3-5-sonnet-20240620", - model="claude-3-5-sonnet-latest", + model="claude-3-7-sonnet-latest", enable_prompt_caching_beta=True, ) llm.register_function("get_weather", get_weather) From e18d9f6a111d27ac3009ac9609b3e83d59d9b39a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 17:26:38 -0700 Subject: [PATCH 342/427] PipelineTask: automatically cancel tasks if pipeline is idle --- CHANGELOG.md | 13 +++++ src/pipecat/pipeline/task.py | 94 +++++++++++++++++++++++++++++++++++- src/pipecat/tests/utils.py | 6 ++- tests/test_pipeline.py | 91 ++++++++++++++++++++++++++++++++-- 4 files changed, 198 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35df891ef..2b0da82a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support for detecting idle pipelines. By default, if no activity has + been detected during 5 minutes, the `PipelineTask` will be automatically + cancelled. It is possible to override this behavior by passing + `cancel_on_idle_timeout=False`. It is also possible to change the default + timeout with `idle_timeout_secs` or the frames that prevent the pipeline from + being idle with `idle_timeout_frames`. Finally, an `on_idle_timeout` event + handler will be triggered if the idle timeout is reached (whether the pipeline + task is cancelled or not). + - Added a `reconnect_on_error` parameter to websocket-based TTS services as well as a `on_connection_error` event handler. The `reconnect_on_error` indicates whether the TTS service should reconnect on error. The `on_connection_error` @@ -111,6 +120,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- ⚠️ `PipelineTask` will now be automatically cancelled if no bot activity is + happening in the pipeline. There are a few settings to configure this + behavior, see `PipelineTask` documentation for more details. + - All event handlers are now executed in separate tasks in order to prevent blocking the pipeline. It is possible that event handlers take some time to execute in which case the pipeline would be blocked waiting for the event diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index e9744fd04..cf62be64f 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -5,6 +5,7 @@ # import asyncio +import time from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type from loguru import logger @@ -13,6 +14,7 @@ from pydantic import BaseModel, ConfigDict from pipecat.clocks.base_clock import BaseClock from pipecat.clocks.system_clock import SystemClock from pipecat.frames.frames import ( + BotSpeakingFrame, CancelFrame, CancelTaskFrame, EndFrame, @@ -20,6 +22,7 @@ from pipecat.frames.frames import ( ErrorFrame, Frame, HeartbeatFrame, + LLMFullResponseEndFrame, MetricsFrame, StartFrame, StopFrame, @@ -133,12 +136,27 @@ class PipelineTask(BaseTask): async def on_frame_reached_downstream(task, frame): ... + It also has an event handler that detects when the pipeline is idle. By + default, a pipeline is idle if no `BotSpeakingFrame` or + `LLMFullResponseEndFrame` are received within `idle_timeout_secs`. + + @task.event_handler("on_idle_timeout") + async def on_idle_timeout(task): + ... + Args: pipeline: The pipeline to execute. params: Configuration parameters for the pipeline. observers: List of observers for monitoring pipeline execution. clock: Clock implementation for timing operations. check_dangling_tasks: Whether to check for processors' tasks finishing properly. + idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or + None. If a pipeline is idle the pipeline task will be cancelled + automatically. + idle_timeout_frames: A tuple with the frames that should trigger an idle + timeout if not received withing `idle_timeout_seconds`. + cancel_on_idle_timeout: Whether the pipeline task should be cancelled if + the idle timeout is reached. """ @@ -151,12 +169,21 @@ class PipelineTask(BaseTask): clock: BaseClock = SystemClock(), task_manager: Optional[BaseTaskManager] = None, check_dangling_tasks: bool = True, + idle_timeout_secs: Optional[float] = 300, + idle_timeout_frames: Tuple[Type[Frame], ...] = ( + BotSpeakingFrame, + LLMFullResponseEndFrame, + ), + cancel_on_idle_timeout: bool = True, ): super().__init__() self._pipeline = pipeline self._clock = clock self._params = params self._check_dangling_tasks = check_dangling_tasks + self._idle_timeout_secs = idle_timeout_secs + self._idle_timeout_frames = idle_timeout_frames + self._cancel_on_idle_timeout = cancel_on_idle_timeout if self._params.observers: import warnings @@ -178,6 +205,10 @@ class PipelineTask(BaseTask): # This is the heartbeat queue. When a heartbeat frame is received in the # down queue we add it to the heartbeat queue for processing. self._heartbeat_queue = asyncio.Queue() + # This is the idle queue. When frames are received downstream they are + # put in the queue. If no frame is received the pipeline is considered + # idle. + self._idle_queue = asyncio.Queue() # This event is used to indicate a finalize frame (e.g. EndFrame, # StopFrame) has been received in the down queue. self._pipeline_end_event = asyncio.Event() @@ -213,6 +244,7 @@ class PipelineTask(BaseTask): self._reached_downstream_types: Tuple[Type[Frame], ...] = () self._register_event_handler("on_frame_reached_upstream") self._register_event_handler("on_frame_reached_downstream") + self._register_event_handler("on_idle_timeout") @property def params(self) -> PipelineParams: @@ -328,19 +360,30 @@ class PipelineTask(BaseTask): self._heartbeat_monitor_handler(), f"{self}::_heartbeat_monitor_handler" ) + def _maybe_start_idle_task(self): + if self._idle_timeout_secs: + self._idle_monitor_task = self._task_manager.create_task( + self._idle_monitor_handler(), f"{self}::_idle_monitor_handler" + ) + async def _cancel_tasks(self): - await self._maybe_cancel_heartbeat_tasks() + await self._observer.stop() await self._task_manager.cancel_task(self._process_up_task) await self._task_manager.cancel_task(self._process_down_task) - await self._observer.stop() + await self._maybe_cancel_heartbeat_tasks() + await self._maybe_cancel_idle_task() async def _maybe_cancel_heartbeat_tasks(self): if self._params.enable_heartbeats: await self._task_manager.cancel_task(self._heartbeat_push_task) await self._task_manager.cancel_task(self._heartbeat_monitor_task) + async def _maybe_cancel_idle_task(self): + if self._idle_timeout_secs: + await self._task_manager.cancel_task(self._idle_monitor_task) + def _initial_metrics_frame(self) -> MetricsFrame: processors = self._pipeline.processors_with_metrics() data = [] @@ -372,6 +415,7 @@ class PipelineTask(BaseTask): self._clock.start() self._maybe_start_heartbeat_tasks() + self._maybe_start_idle_task() start_frame = StartFrame( clock=self._clock, @@ -445,6 +489,10 @@ class PipelineTask(BaseTask): while True: frame = await self._down_queue.get() + # Queue received frame to the idle queue so we can monitor idle + # pipelines. + await self._idle_queue.put(frame) + if isinstance(frame, self._reached_downstream_types): await self._call_event_handler("on_frame_reached_downstream", frame) @@ -482,6 +530,48 @@ class PipelineTask(BaseTask): f"{self}: heartbeat frame not received for more than {wait_time} seconds" ) + async def _idle_monitor_handler(self): + """This tasks monitors activity in the pipeline. If no frames are + received (heartbeats don't count) the pipeline is considered idle. + + """ + running = True + last_frame_time = 0 + while running: + try: + frame = await asyncio.wait_for( + self._idle_queue.get(), timeout=self._idle_timeout_secs + ) + + if isinstance(frame, StartFrame) or isinstance(frame, self._idle_timeout_frames): + # If we find a StartFrame or one of the frames that prevents a + # time out we update the time. + last_frame_time = time.time() + else: + # If we find any other frame we check if the pipeline is + # idle by checking the last time we received one of the + # valid frames. + diff_time = time.time() - last_frame_time + if diff_time >= self._idle_timeout_secs: + running = await self._idle_timeout_detected() + + self._idle_queue.task_done() + except asyncio.TimeoutError: + running = await self._idle_timeout_detected() + + async def _idle_timeout_detected(self) -> bool: + """Logic for when the pipeline is idle. + + Returns: + bool: Whther the pipeline task is being cancelled or not. + """ + await self._call_event_handler("on_idle_timeout") + if self._cancel_on_idle_timeout: + logger.warning(f"Idle pipeline detected, cancelling pipeline task...") + await self.cancel() + return False + return True + def _print_dangling_tasks(self): tasks = [t.get_name() for t in self._task_manager.current_tasks()] if tasks: diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index 7fcecc3ff..d68c87d88 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -101,7 +101,11 @@ async def run_test( pipeline = Pipeline([source, processor, sink]) - task = PipelineTask(pipeline, params=PipelineParams(start_metadata=start_metadata)) + task = PipelineTask( + pipeline, + params=PipelineParams(start_metadata=start_metadata), + cancel_on_idle_timeout=False, + ) async def push_frames(): # Just give a little head start to the runner. diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c3811a672..46114b5b2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -5,6 +5,7 @@ # import asyncio +import time import unittest from pipecat.frames.frames import EndFrame, HeartbeatFrame, StartFrame, TextFrame @@ -100,7 +101,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): identity = IdentityFilter() pipeline = Pipeline([identity]) - task = PipelineTask(pipeline) + task = PipelineTask(pipeline, cancel_on_idle_timeout=False) task.set_event_loop(asyncio.get_event_loop()) task.set_reached_upstream_filter((TextFrame,)) task.set_reached_downstream_filter((TextFrame,)) @@ -123,7 +124,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.queue_frame(TextFrame(text="Hello Downstream!")) try: - await asyncio.wait_for(task.run(), timeout=1.0) + await asyncio.wait_for(asyncio.shield(task.run()), timeout=1.0) except asyncio.TimeoutError: pass @@ -149,6 +150,7 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): heartbeats_period_secs=0.2, ), observers=[heartbeats_observer], + cancel_on_idle_timeout=False, ) task.set_event_loop(asyncio.get_event_loop()) @@ -156,7 +158,90 @@ class TestPipelineTask(unittest.IsolatedAsyncioTestCase): await task.queue_frame(TextFrame(text="Hello!")) try: - await asyncio.wait_for(task.run(), timeout=1.0) + await asyncio.wait_for(asyncio.shield(task.run()), timeout=1.0) except asyncio.TimeoutError: pass assert heartbeats_counter == expected_heartbeats + + async def test_idle_task(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline, idle_timeout_secs=0.2) + task.set_event_loop(asyncio.get_event_loop()) + await task.run() + assert True + + async def test_no_idle_task(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False) + task.set_event_loop(asyncio.get_event_loop()) + try: + await asyncio.wait_for(asyncio.shield(task.run()), timeout=0.3) + except asyncio.TimeoutError: + assert True + else: + assert False + + async def test_idle_task_heartbeats(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_heartbeats=True, + heartbeats_period_secs=0.1, + ), + idle_timeout_secs=0.3, + ) + task.set_event_loop(asyncio.get_event_loop()) + await task.run() + assert True + + async def test_idle_task_event_handler(self): + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask(pipeline, idle_timeout_secs=0.2, cancel_on_idle_timeout=False) + task.set_event_loop(asyncio.get_event_loop()) + + idle_timeout = False + + @task.event_handler("on_idle_timeout") + async def on_idle_timeout(task: PipelineTask): + nonlocal idle_timeout + idle_timeout = True + await task.cancel() + + await task.run() + assert True + + async def test_idle_task_frames(self): + idle_timeout_secs = 0.2 + sleep_time_secs = idle_timeout_secs / 2 + + identity = IdentityFilter() + pipeline = Pipeline([identity]) + task = PipelineTask( + pipeline, + idle_timeout_secs=idle_timeout_secs, + idle_timeout_frames=(TextFrame,), + ) + task.set_event_loop(asyncio.get_event_loop()) + + async def delayed_frames(): + await asyncio.sleep(sleep_time_secs) + await task.queue_frame(TextFrame("Hello Pipecat!")) + await asyncio.sleep(sleep_time_secs) + await task.queue_frame(TextFrame("Hello Pipecat!")) + await asyncio.sleep(sleep_time_secs) + await task.queue_frame(TextFrame("Hello Pipecat!")) + + start_time = time.time() + + tasks = {asyncio.create_task(task.run()), asyncio.create_task(delayed_frames())} + + await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + + diff_time = time.time() - start_time + + self.assertGreater(diff_time, sleep_time_secs * 3) From b6be25ab84d2d1bdcf49cb4493efe0b90c4f7e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 23:56:40 -0700 Subject: [PATCH 343/427] SegmentedSTTService: use VAD events to detect valid audio --- CHANGELOG.md | 4 ++ src/pipecat/services/ai_services.py | 106 ++++++++++++---------------- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4ff6185..1cdc10c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -149,6 +149,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `SegmentedSTTService` issue that was causing audio to be sent + prematurely to the STT service. Instead of analyzing the volume in this + service we rely on VAD events which use both VAD and volume. + - Fixed a `GeminiMultimodalLiveLLMService` issue that was causing messages to be duplicated in the context when pushing `LLMMessagesAppendFrame` frames. diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index d29e0e714..e00970433 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -14,7 +14,6 @@ from loguru import logger from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter -from pipecat.audio.utils import calculate_audio_volume, exp_smoothing from pipecat.frames.frames import ( AudioRawFrame, BotStartedSpeakingFrame, @@ -38,6 +37,8 @@ from pipecat.frames.frames import ( TTSTextFrame, TTSUpdateSettingsFrame, UserImageRequestFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import MetricsData @@ -859,79 +860,64 @@ class STTService(AIService): class SegmentedSTTService(STTService): - """SegmentedSTTService is an STTService that will detect speech and will run - speech-to-text on speech segments only, instead of a continous stream. + """SegmentedSTTService is an STTService that uses VAD events to detect + speech and will run speech-to-text on speech segments only, instead of a + continous stream. Since it uses VAD it means that VAD needs to be enabled in + the pipeline. + + This service always keeps a small audio buffer to take into account that VAD + events are delayed from when the user speech really starts. """ - def __init__( - self, - *, - min_volume: float = 0.6, - max_silence_secs: float = 0.3, - max_buffer_secs: float = 1.5, - sample_rate: Optional[int] = None, - **kwargs, - ): + def __init__(self, *, sample_rate: Optional[int] = None, **kwargs): super().__init__(sample_rate=sample_rate, **kwargs) - self._min_volume = min_volume - self._max_silence_secs = max_silence_secs - self._max_buffer_secs = max_buffer_secs self._content = None self._wave = None - self._silence_num_frames = 0 - # Volume exponential smoothing - self._smoothing_factor = 0.2 - self._prev_volume = 0 - - async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): - # Try to filter out empty background noise - volume = self._get_smoothed_volume(frame) - if volume >= self._min_volume: - # If volume is high enough, write new data to wave file - self._wave.writeframes(frame.audio) - self._silence_num_frames = 0 - else: - self._silence_num_frames += frame.num_frames - self._prev_volume = volume - - # If buffer is not empty and we have enough data or there's been a long - # silence, transcribe the audio gathered so far. - silence_secs = self._silence_num_frames / self.sample_rate - buffer_secs = self._wave.getnframes() / self.sample_rate - if self._content.tell() > 0 and ( - buffer_secs > self._max_buffer_secs or silence_secs > self._max_silence_secs - ): - self._silence_num_frames = 0 - self._wave.close() - self._content.seek(0) - await self.process_generator(self.run_stt(self._content.read())) - (self._content, self._wave) = self._new_wave() + self._audio_buffer = bytearray() + self._audio_buffer_size_1s = 0 + self._user_speaking = False async def start(self, frame: StartFrame): await super().start(frame) - if not self._wave: - (self._content, self._wave) = self._new_wave() + self._audio_buffer_size_1s = self.sample_rate * 2 - async def stop(self, frame: EndFrame): - await super().stop(frame) - self._wave.close() + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) - async def cancel(self, frame: CancelFrame): - await super().cancel(frame) - self._wave.close() + if isinstance(frame, UserStartedSpeakingFrame): + await self._handle_user_started_speaking(frame) + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._handle_user_stopped_speaking(frame) + + async def _handle_user_started_speaking(self, frame: UserStartedSpeakingFrame): + self._user_speaking = True + + async def _handle_user_stopped_speaking(self, frame: UserStoppedSpeakingFrame): + self._user_speaking = False - def _new_wave(self): content = io.BytesIO() - ww = wave.open(content, "wb") - ww.setsampwidth(2) - ww.setnchannels(1) - ww.setframerate(self.sample_rate) - return (content, ww) + wav = wave.open(content, "wb") + wav.setsampwidth(2) + wav.setnchannels(1) + wav.setframerate(self.sample_rate) + wav.writeframes(self._audio_buffer) + wav.close() + content.seek(0) - def _get_smoothed_volume(self, frame: AudioRawFrame) -> float: - volume = calculate_audio_volume(frame.audio, frame.sample_rate) - return exp_smoothing(volume, self._prev_volume, self._smoothing_factor) + await self.process_generator(self.run_stt(content.read())) + + # Start clean. + self._audio_buffer.clear() + + async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): + # If the user is speaking the audio buffer will keep growin. + self._audio_buffer += frame.audio + + # If the user is not speaking we keep just a little bit of audio. + if not self._user_speaking and len(self._audio_buffer) > self._audio_buffer_size_1s: + discarded = len(self._audio_buffer) - self._audio_buffer_size_1s + self._audio_buffer = self._audio_buffer[discarded:] class ImageGenService(AIService): From 7cdcd1c3d1ac4fbdfa2b3e2bddaf1d3513b7a512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 00:36:03 -0700 Subject: [PATCH 344/427] OpenAITTSService: allow specifying any model name --- src/pipecat/services/openai.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 5a3a993aa..89bf4a04b 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -472,22 +472,16 @@ class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz. - When using with DailyTransport, configure the sample rate in DailyParams - as shown below: - - DailyParams( - audio_out_enabled=True, - audio_out_sample_rate=24_000, - ) Args: api_key: OpenAI API key. Defaults to None. voice: Voice ID to use. Defaults to "alloy". - model: TTS model to use ("tts-1" or "tts-1-hd"). Defaults to "tts-1". - sample_rate: Output audio sample rate in Hz. Defaults to 24000. + model: TTS model to use. Defaults to "tts-1". + sample_rate: Output audio sample rate in Hz. Defaults to None. **kwargs: Additional keyword arguments passed to TTSService. The service returns PCM-encoded audio at the specified sample rate. + """ OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz @@ -497,7 +491,7 @@ class OpenAITTSService(TTSService): *, api_key: Optional[str] = None, voice: str = "alloy", - model: Literal["tts-1", "tts-1-hd"] = "tts-1", + model: str = "tts-1", sample_rate: Optional[int] = None, **kwargs, ): From 2417ec4f926eb27053dfeded8725706d3e4d3853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 00:31:54 -0700 Subject: [PATCH 345/427] LLMUserContextAggregator: increase bot_interruption_timeout to 5 seconds --- src/pipecat/processors/aggregators/llm_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index d8582e32f..aed12db9e 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -275,7 +275,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self, context: OpenAILLMContext, aggregation_timeout: float = 1.0, - bot_interruption_timeout: float = 2.0, + bot_interruption_timeout: float = 5.0, **kwargs, ): super().__init__(context=context, role="user", **kwargs) From 0e14cec1392fbaec8d1fab4c64e728501bea945a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 01:22:33 -0700 Subject: [PATCH 346/427] pyproject: update multiple libraries --- pyproject.toml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4176ae2a0..87a34f929 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "pyloudnorm~=0.1.1", "resampy~=0.4.3", "soxr~=0.5.0", - "openai~=1.59.6" + "openai~=1.67.0" ] [project.urls] @@ -39,42 +39,43 @@ Source = "https://github.com/pipecat-ai/pipecat" Website = "https://pipecat.ai" [project.optional-dependencies] -anthropic = [ "anthropic~=0.47.2" ] -assemblyai = [ "assemblyai~=0.36.0" ] -aws = [ "boto3~=1.35.99" ] -azure = [ "azure-cognitiveservices-speech~=1.42.0"] +anthropic = [ "anthropic~=0.49.0" ] +assemblyai = [ "assemblyai~=0.37.0" ] +aws = [ "boto3~=1.37.16" ] +azure = [ "azure-cognitiveservices-speech~=1.43.0"] canonical = [ "aiofiles~=24.1.0" ] -cartesia = [ "cartesia~=1.3.1", "websockets~=13.1" ] +cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] -deepgram = [ "deepgram-sdk~=3.8.0" ] +deepgram = [ "deepgram-sdk~=3.10.1" ] elevenlabs = [ "websockets~=13.1" ] -fal = [ "fal-client~=0.5.6" ] +fal = [ "fal-client~=0.5.9" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] -google = [ "google-cloud-speech~=2.31.0", "google-cloud-texttospeech~=2.25.0", "google-genai~=1.3.0", "google-generativeai~=0.8.4" ] +google = [ "google-cloud-speech~=2.31.1", "google-cloud-texttospeech~=2.25.1", "google-genai~=1.7.0", "google-generativeai~=0.8.4" ] grok = [] groq = [] gstreamer = [ "pygobject~=3.50.0" ] fireworks = [] krisp = [ "pipecat-ai-krisp~=0.3.0" ] koala = [ "pvkoala~=2.0.3" ] -langchain = [ "langchain~=0.3.14", "langchain-community~=0.3.14", "langchain-openai~=0.3.0" ] -livekit = [ "livekit~=0.19.1", "livekit-api~=0.8.1", "tenacity~=9.0.0" ] +langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-openai~=0.3.9" ] +livekit = [ "livekit~=0.22.0", "livekit-api~=0.8.2", "tenacity~=9.0.0" ] lmnt = [ "websockets~=13.1" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.13", "transformers~=4.48.0" ] nim = [] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "websockets~=13.1" ] -openpipe = [ "openpipe~=4.45.0" ] +openpipe = [ "openpipe~=4.48.0" ] +openrouter = [] perplexity = [] playht = [ "pyht~=0.1.12", "websockets~=13.1" ] rime = [ "websockets~=13.1" ] -riva = [ "nvidia-riva-client~=2.18.0" ] -sentry = [ "sentry-sdk~=2.20.0" ] +riva = [ "nvidia-riva-client~=2.19.0" ] +sentry = [ "sentry-sdk~=2.23.1" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.10"] soundfile = [ "soundfile~=0.13.0" ] @@ -83,7 +84,6 @@ together = [] ultravox = [ "transformers~=4.48.0", "vllm~=0.7.3" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.6" ] whisper = [ "faster-whisper~=1.1.1" ] -openrouter = [] [tool.setuptools.packages.find] # All the following settings are optional: From a98000fd1dcf3932f4c85a54e747d3e1d1445775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 14:57:27 -0700 Subject: [PATCH 347/427] function calling now run in tasks --- CHANGELOG.md | 19 +- src/pipecat/frames/frames.py | 21 ++ src/pipecat/pipeline/task.py | 2 +- .../processors/aggregators/llm_response.py | 232 +++++++++++------- .../aggregators/openai_llm_context.py | 65 +---- src/pipecat/services/ai_services.py | 200 ++++++++++++--- src/pipecat/services/anthropic.py | 164 ++++--------- .../services/gemini_multimodal_live/gemini.py | 9 +- src/pipecat/services/google/google.py | 143 +++++------ src/pipecat/services/grok.py | 85 +------ src/pipecat/services/openai.py | 161 ++++-------- tests/test_context_aggregators.py | 3 +- 12 files changed, 537 insertions(+), 567 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b931c083f..4762186cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- When registering a function call it is now possible to indicate if you want + the function call to be cancelled if there's a user interruption via + `cancel_on_interruption` (defaults to False). This is now possible because + function calls are executed concurrently. + - Added support for detecting idle pipelines. By default, if no activity has been detected during 5 minutes, the `PipelineTask` will be automatically cancelled. It is possible to override this behavior by passing @@ -120,6 +125,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Function calls are now executed in tasks. This means that the pipeline will + not be blocked while the function call is being executed. + - ⚠️ `PipelineTask` will now be automatically cancelled if no bot activity is happening in the pipeline. There are a few settings to configure this behavior, see `PipelineTask` documentation for more details. @@ -140,6 +148,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- Passing a `start_callback` to `LLMService.register_function()` is now + deprecated, simply move the code from the start callback to the function call. + - `TTSService` parameter `text_filter` is now deprecated, use `text_filters` instead which is now a list. This allows passing multiple filters that will be executed in order. @@ -162,6 +173,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an assistant aggregator issue that could cause assistant text to be + split into multiple chunks during function calls. + +- Fixed an assistant aggregator issue that was causing assistant text to not be + added to the context during function calls. This could lead to duplications. + - Fixed a `SegmentedSTTService` issue that was causing audio to be sent prematurely to the STT service. Instead of analyzing the volume in this service we rely on VAD events which use both VAD and volume. @@ -1978,7 +1995,7 @@ async def on_connected(processor): completed. If a task is never ran `has_finished()` will return False. - `PipelineRunner` now supports SIGTERM. If received, the runner will be - canceled. + cancelled. ### Fixed diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 74dd2accb..e589e9480 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -634,6 +634,15 @@ class FunctionCallInProgressFrame(SystemFrame): function_name: str tool_call_id: str arguments: str + cancel_on_interruption: bool + + +@dataclass +class FunctionCallCancelFrame(SystemFrame): + """A frame to signal a function call has been cancelled.""" + + function_name: str + tool_call_id: str @dataclass @@ -706,6 +715,18 @@ class VisionImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})" +@dataclass +class UserImageMessageFrame(SystemFrame): + """An image associated to a user.""" + + user_image_raw_frame: UserImageRawFrame + text: Optional[str] = None + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, image: {self.user_image_raw_frame}, text: {self.text})" + + # # Control frames # diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index cf62be64f..8279373cb 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -409,7 +409,7 @@ class PipelineTask(BaseTask): async def _process_push_queue(self): """This is the task that runs the pipeline for the first time by sending a StartFrame and by pushing any other frames queued by the user. It runs - until the tasks is canceled or stopped (e.g. with an EndFrame). + until the tasks is cancelled or stopped (e.g. with an EndFrame). """ self._clock.start() diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index aed12db9e..494d1de38 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -7,14 +7,20 @@ import asyncio import time from abc import abstractmethod -from typing import List +from typing import Dict, List + +from loguru import logger from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, CancelFrame, EmulateUserStartedSpeakingFrame, EmulateUserStoppedSpeakingFrame, EndFrame, Frame, + FunctionCallCancelFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -23,10 +29,12 @@ from pipecat.frames.frames import ( LLMMessagesUpdateFrame, LLMSetToolsFrame, LLMTextFrame, + OpenAILLMContextAssistantTimestampFrame, StartFrame, StartInterruptionFrame, TextFrame, TranscriptionFrame, + UserImageMessageFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -35,6 +43,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.time import time_now_iso8601 class LLMFullResponseAggregator(FrameProcessor): @@ -139,68 +148,20 @@ class BaseLLMResponseAggregator(FrameProcessor): pass @abstractmethod - async def push_aggregation(self): + async def handle_aggregation(self, aggregation: str): + """Adds the given aggregation to the aggregator. The aggregator can use + a simple list of message or a context. It doesn't not push any frames. + + """ pass - -class LLMResponseAggregator(BaseLLMResponseAggregator): - """This is a base LLM aggregator that uses a simple list of messages to - store the conversation. It pushes `LLMMessagesFrame` as an aggregation - frame. - - """ - - def __init__( - self, - *, - messages: List[dict], - role: str = "user", - **kwargs, - ): - super().__init__(**kwargs) - - self._messages = messages - self._role = role - - self._aggregation = "" - - self.reset() - - @property - def messages(self) -> List[dict]: - return self._messages - - @property - def role(self) -> str: - return self._role - - def add_messages(self, messages): - self._messages.extend(messages) - - def set_messages(self, messages): - self.reset() - self._messages.clear() - self._messages.extend(messages) - - def set_tools(self, tools): - pass - - def reset(self): - self._aggregation = "" - + @abstractmethod async def push_aggregation(self): - if len(self._aggregation) > 0: - self._messages.append({"role": self._role, "content": self._aggregation}) + """Pushes the current aggregation. For example, iN the case of context + aggregation this might push a new context frame. - # Reset the aggregation. Reset it before pushing it down, otherwise - # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" - - frame = LLMMessagesFrame(self._messages) - await self.push_frame(frame) - - # Reset our accumulator state. - self.reset() + """ + pass class LLMContextResponseAggregator(BaseLLMResponseAggregator): @@ -247,20 +208,6 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def reset(self): self._aggregation = "" - async def push_aggregation(self): - if len(self._aggregation) > 0: - self._context.add_message({"role": self.role, "content": self._aggregation}) - - # Reset the aggregation. Reset it before pushing it down, otherwise - # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" - - frame = OpenAILLMContextFrame(self._context) - await self.push_frame(frame) - - # Reset our accumulator state. - self.reset() - class LLMUserContextAggregator(LLMContextResponseAggregator): """This is a user LLM aggregator that uses an LLM context to store the @@ -290,12 +237,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self._aggregation_event = asyncio.Event() self._aggregation_task = None - self.reset() - def reset(self): super().reset() self._seen_interim_results = False + async def handle_aggregation(self, aggregation: str): + self._context.add_message({"role": self.role, "content": self._aggregation}) + async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -331,6 +279,20 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): else: await self.push_frame(frame, direction) + async def push_aggregation(self): + if len(self._aggregation) > 0: + await self.handle_aggregation(self._aggregation) + + # Reset the aggregation. Reset it before pushing it down, otherwise + # if the tasks gets cancelled we won't be able to clear things up. + self._aggregation = "" + + frame = OpenAILLMContextFrame(self._context) + await self.push_frame(frame) + + # Reset our accumulator state. + self.reset() + async def _start(self, frame: StartFrame): self._create_aggregation_task() @@ -424,17 +386,29 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): super().__init__(context=context, role="assistant", **kwargs) self._expect_stripped_words = expect_stripped_words - self._started = False + self._started = 0 + self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {} - self.reset() + async def handle_aggregation(self, aggregation: str): + self._context.add_message({"role": "assistant", "content": aggregation}) + + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + pass + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + pass + + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + pass + + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + pass async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): - await self.push_aggregation() - # Reset anyways - self.reset() + await self._handle_interruptions(frame) await self.push_frame(frame, direction) elif isinstance(frame, LLMFullResponseStartFrame): await self._handle_llm_start(frame) @@ -448,14 +422,104 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self.set_messages(frame.messages) elif isinstance(frame, LLMSetToolsFrame): self.set_tools(frame.tools) + elif isinstance(frame, FunctionCallInProgressFrame): + await self._handle_function_call_in_progress(frame) + elif isinstance(frame, FunctionCallResultFrame): + await self._handle_function_call_result(frame) + elif isinstance(frame, FunctionCallCancelFrame): + await self._handle_function_call_cancel(frame) + elif isinstance(frame, UserImageMessageFrame): + await self._handle_image_frame_message(frame) + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.push_aggregation() else: await self.push_frame(frame, direction) + async def push_aggregation(self): + if not self._aggregation: + return + + aggregation = self._aggregation.strip() + self.reset() + + if aggregation: + await self.handle_aggregation(aggregation) + + # Push context frame + await self.push_context_frame() + + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + + async def _handle_interruptions(self, frame: StartInterruptionFrame): + await self.push_aggregation() + self._started = 0 + self.reset() + + async def _handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + logger.debug( + f"{self} FunctionCallInProgressFrame: [{frame.function_name}:{frame.tool_call_id}]" + ) + await self.handle_function_call_in_progress(frame) + self._function_calls_in_progress[frame.tool_call_id] = frame + + async def _handle_function_call_result(self, frame: FunctionCallResultFrame): + logger.debug( + f"{self} FunctionCallResultFrame: [{frame.function_name}:{frame.tool_call_id}]" + ) + if frame.tool_call_id not in self._function_calls_in_progress: + logger.warning( + f"FunctionCallResultFrame tool_call_id [{frame.tool_call_id}] is not running" + ) + return + + del self._function_calls_in_progress[frame.tool_call_id] + + properties = frame.properties + + await self.handle_function_call_result(frame) + + # Run inference if the function call result requires it. + if frame.result: + run_llm = False + + if properties and properties.run_llm is not None: + # If the tool call result has a run_llm property, use it + run_llm = properties.run_llm + else: + # Default behavior is to run the LLM if there are no function calls in progress + run_llm = not bool(self._function_calls_in_progress) + + if run_llm: + await self.push_context_frame(FrameDirection.UPSTREAM) + + # Emit the on_context_updated callback once the function call + # result is added to the context + if properties and properties.on_context_updated: + await properties.on_context_updated() + + async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + logger.debug( + f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]" + ) + if frame.tool_call_id not in self._function_calls_in_progress: + return + + if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption: + await self.handle_function_call_cancel(frame) + del self._function_calls_in_progress[frame.tool_call_id] + + async def _handle_image_frame_message(self, frame: UserImageMessageFrame): + await self.handle_image_frame_message(frame) + await self.push_aggregation() + await self.push_context_frame(FrameDirection.UPSTREAM) + async def _handle_llm_start(self, _: LLMFullResponseStartFrame): - self._started = True + self._started += 1 async def _handle_llm_end(self, _: LLMFullResponseEndFrame): - self._started = False + self._started -= 1 await self.push_aggregation() async def _handle_text(self, frame: TextFrame): @@ -474,7 +538,7 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): async def push_aggregation(self): if len(self._aggregation) > 0: - self._context.add_message({"role": self.role, "content": self._aggregation}) + await self.handle_aggregation(self._aggregation) # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. @@ -493,7 +557,7 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): async def push_aggregation(self): if len(self._aggregation) > 0: - self._context.add_message({"role": self.role, "content": self._aggregation}) + await self.handle_aggregation(self._aggregation) # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index e8391d62b..93c2875be 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -9,9 +9,8 @@ import copy import io import json from dataclasses import dataclass -from typing import Any, Awaitable, Callable, List, Optional +from typing import Any, List, Optional -from loguru import logger from openai._types import NOT_GIVEN, NotGiven from openai.types.chat import ( ChatCompletionMessageParam, @@ -22,12 +21,7 @@ from PIL import Image from pipecat.adapters.base_llm_adapter import BaseLLMAdapter from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.frames.frames import ( - AudioRawFrame, - Frame, - FunctionCallInProgressFrame, - FunctionCallResultFrame, -) +from pipecat.frames.frames import AudioRawFrame, Frame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor # JSON custom encoder to handle bytes arrays so that we can log contexts @@ -187,61 +181,6 @@ class OpenAILLMContext: # todo: implement for OpenAI models and others pass - async def call_function( - self, - f: Callable[ - [str, str, Any, FrameProcessor, "OpenAILLMContext", Callable[[Any], Awaitable[None]]], - Awaitable[None], - ], - *, - function_name: str, - tool_call_id: str, - arguments: str, - llm: FrameProcessor, - run_llm: bool = True, - ) -> None: - logger.info(f"Calling function {function_name} with arguments {arguments}") - # Push a SystemFrame downstream. This frame will let our assistant context aggregator - # know that we are in the middle of a function call. Some contexts/aggregators may - # not need this. But some definitely do (Anthropic, for example). - # Also push a SystemFrame upstream for use by other processors, like STTMuteFilter. - progress_frame_downstream = FunctionCallInProgressFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - ) - progress_frame_upstream = FunctionCallInProgressFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - ) - - # Push frame both downstream and upstream - await llm.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM) - await llm.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM) - - # Define a callback function that pushes a FunctionCallResultFrame upstream & downstream. - async def function_call_result_callback(result, *, properties=None): - result_frame_downstream = FunctionCallResultFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - result=result, - properties=properties, - ) - result_frame_upstream = FunctionCallResultFrame( - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - result=result, - properties=properties, - ) - - await llm.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM) - await llm.push_frame(result_frame_upstream, FrameDirection.UPSTREAM) - - await f(function_name, tool_call_id, arguments, llm, self, function_call_result_callback) - def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size): # RIFF chunk descriptor header = bytearray() diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e00970433..2f30f505f 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -8,7 +8,8 @@ import asyncio import io import wave from abc import abstractmethod -from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple, Type +from dataclasses import dataclass +from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Set, Tuple, Type from loguru import logger @@ -22,6 +23,9 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, + FunctionCallCancelFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InterimTranscriptionFrame, LLMFullResponseEndFrame, StartFrame, @@ -138,6 +142,13 @@ class AIService(FrameProcessor): await self.push_frame(f) +@dataclass +class FunctionEntry: + function_name: Optional[str] + callback: Any # TODO(aleix): add proper typing. + cancel_on_interruption: bool + + class LLMService(AIService): """This class is a no-op but serves as a base class for LLM services.""" @@ -147,38 +158,74 @@ class LLMService(AIService): def __init__(self, **kwargs): super().__init__(**kwargs) - self._callbacks = {} + self._functions = {} self._start_callbacks = {} self._adapter = self.adapter_class() + self._function_call_tasks: Set[Tuple[asyncio.Task, str, str]] = set() + + self._register_event_handler("on_completion_timeout") def get_llm_adapter(self) -> BaseLLMAdapter: return self._adapter def create_context_aggregator( - self, context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + self, + context: OpenAILLMContext, + *, + user_kwargs: Mapping[str, Any] = {}, + assistant_kwargs: Mapping[str, Any] = {}, ) -> Any: pass - self._register_event_handler("on_completion_timeout") + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) - # TODO-CB: callback function type - def register_function(self, function_name: Optional[str], callback, start_callback=None): + if isinstance(frame, StartInterruptionFrame): + await self._handle_interruptions(frame) + + async def _handle_interruptions(self, frame: StartInterruptionFrame): + for function_name, entry in self._functions.items(): + if entry.cancel_on_interruption: + await self._cancel_function_call(function_name) + + def register_function( + self, + function_name: Optional[str], + callback: Any, + start_callback=None, + *, + cancel_on_interruption: bool = False, + ): # Registering a function with the function_name set to None will run that callback # for all functions - self._callbacks[function_name] = callback - # QUESTION FOR CB: maybe this isn't needed anymore? + self._functions[function_name] = FunctionEntry( + function_name=function_name, + callback=callback, + cancel_on_interruption=cancel_on_interruption, + ) + + # Start callbacks are now deprecated. if start_callback: + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Parameter 'start_callback' is deprecated, just put your code on top of the actual function call instead.", + DeprecationWarning, + ) + self._start_callbacks[function_name] = start_callback def unregister_function(self, function_name: Optional[str]): - del self._callbacks[function_name] + del self._functions[function_name] if self._start_callbacks[function_name]: del self._start_callbacks[function_name] def has_function(self, function_name: str): - if None in self._callbacks.keys(): + if None in self._functions.keys(): return True - return function_name in self._callbacks.keys() + return function_name in self._functions.keys() async def call_function( self, @@ -188,25 +235,18 @@ class LLMService(AIService): function_name: str, arguments: str, run_llm: bool = True, - ) -> None: - f = None - if function_name in self._callbacks.keys(): - f = self._callbacks[function_name] - elif None in self._callbacks.keys(): - f = self._callbacks[None] - else: - return None - await self.call_start_function(context, function_name) - await context.call_function( - f, - function_name=function_name, - tool_call_id=tool_call_id, - arguments=arguments, - llm=self, - run_llm=run_llm, + ): + if not function_name in self._functions.keys() and not None in self._functions.keys(): + return + + task = self.create_task( + self._run_function_call(context, tool_call_id, function_name, arguments, run_llm) ) - # QUESTION FOR CB: maybe this isn't needed anymore? + self._function_call_tasks.add((task, tool_call_id, function_name)) + + task.add_done_callback(self._function_call_task_finished) + async def call_start_function(self, context: OpenAILLMContext, function_name: str): if function_name in self._start_callbacks.keys(): await self._start_callbacks[function_name](function_name, self, context) @@ -218,6 +258,106 @@ class LLMService(AIService): UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM ) + async def _run_function_call( + self, + context: OpenAILLMContext, + tool_call_id: str, + function_name: str, + arguments: str, + run_llm: bool = True, + ): + if function_name in self._functions.keys(): + entry = self._functions[function_name] + elif None in self._functions.keys(): + entry = self._functions[None] + else: + return + + logger.info(f"Calling function {function_name} with arguments {arguments}") + + # NOTE(aleix): This needs to be removed after we remove the deprecation. + await self.call_start_function(context, function_name) + + # Push a SystemFrame downstream. This frame will let our assistant context aggregator + # know that we are in the middle of a function call. Some contexts/aggregators may + # not need this. But some definitely do (Anthropic, for example). + # Also push a SystemFrame upstream for use by other processors, like STTMuteFilter. + progress_frame_downstream = FunctionCallInProgressFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + cancel_on_interruption=entry.cancel_on_interruption, + ) + progress_frame_upstream = FunctionCallInProgressFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + cancel_on_interruption=entry.cancel_on_interruption, + ) + + # Push frame both downstream and upstream + await self.push_frame(progress_frame_downstream, FrameDirection.DOWNSTREAM) + await self.push_frame(progress_frame_upstream, FrameDirection.UPSTREAM) + + # Define a callback function that pushes a FunctionCallResultFrame upstream & downstream. + async def function_call_result_callback(result, *, properties=None): + result_frame_downstream = FunctionCallResultFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + result=result, + properties=properties, + ) + result_frame_upstream = FunctionCallResultFrame( + function_name=function_name, + tool_call_id=tool_call_id, + arguments=arguments, + result=result, + properties=properties, + ) + + await self.push_frame(result_frame_downstream, FrameDirection.DOWNSTREAM) + await self.push_frame(result_frame_upstream, FrameDirection.UPSTREAM) + + await entry.callback( + function_name, tool_call_id, arguments, self, context, function_call_result_callback + ) + + async def _cancel_function_call(self, function_name: str): + cancelled_tasks = set() + for task, tool_call_id, name in self._function_call_tasks: + if name == function_name: + # We remove the callback because we are going to cancel the task + # now, otherwise we will be removing it from the set while we + # are iterating. + task.remove_done_callback(self._function_call_task_finished) + + logger.debug(f"{self} Cancelling function call [{name}:{tool_call_id}]...") + + await self.cancel_task(task) + + frame = FunctionCallCancelFrame( + function_name=function_name, tool_call_id=tool_call_id + ) + await self.push_frame(frame) + + logger.debug(f"{self} Function call [{name}:{tool_call_id}] has been cancelled") + + cancelled_tasks.add(task) + + # Remove all cancelled tasks from our set. + for task in cancelled_tasks: + self._function_call_task_finished(task) + + def _function_call_task_finished(self, task: asyncio.Task): + tuple_to_remove = next((t for t in self._function_call_tasks if t[0] == task), None) + if tuple_to_remove: + self._function_call_tasks.discard(tuple_to_remove) + # The task is finished so this should exit immediately. We need to + # do this because otherwise the task manager would have a dangling + # task if we don't remove it. + asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) + class TTSService(AIService): def __init__( @@ -366,12 +506,14 @@ class TTSService(AIService): else: await self.push_frame(frame, direction) elif isinstance(frame, TTSSpeakFrame): + # Store if we were processing text or not so we can set it back. + processing_text = self._processing_text await self._push_tts_frames(frame.text) # We pause processing incoming frames because we are sending data to # the TTS. We pause to avoid audio overlapping. await self._maybe_pause_frame_processing() await self.flush_audio() - self._processing_text = False + self._processing_text = processing_text elif isinstance(frame, TTSUpdateSettingsFrame): await self._update_settings(frame.settings) elif isinstance(frame, BotStoppedSpeakingFrame): diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 10a2ab7b7..8e06f4558 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -21,17 +21,16 @@ from pydantic import BaseModel, Field from pipecat.adapters.services.anthropic_adapter import AnthropicLLMAdapter from pipecat.frames.frames import ( Frame, + FunctionCallCancelFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, - FunctionCallResultProperties, LLMEnablePromptCachingFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - OpenAILLMContextAssistantTimestampFrame, - StartInterruptionFrame, + UserImageMessageFrame, UserImageRawFrame, UserImageRequestFrame, VisionImageRawFrame, @@ -47,7 +46,6 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import LLMService -from pipecat.utils.time import time_now_iso8601 try: from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven @@ -60,13 +58,6 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") -# internal use only -- todo: refactor -@dataclass -class AnthropicImageMessageFrame(Frame): - user_image_raw_frame: UserImageRawFrame - text: Optional[str] = None - - @dataclass class AnthropicContextAggregatorPair: _user: "AnthropicUserContextAggregator" @@ -715,7 +706,7 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): text = self._context._user_image_request_context.get(frame.user_id) or "" if text: del self._context._user_image_request_context[frame.user_id] - frame = AnthropicImageMessageFrame(user_image_raw_frame=frame, text=text) + frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) await self.push_frame(frame) except Exception as e: logger.error(f"Error processing frame: {e}") @@ -734,110 +725,61 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): def __init__(self, context: OpenAILLMContext | AnthropicLLMContext, **kwargs): super().__init__(context=context, **kwargs) - self._function_call_in_progress = None - self._function_call_result = None - self._pending_image_frame_message = None - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # See note above about not calling push_frame() here. - if isinstance(frame, StartInterruptionFrame): - self._function_call_in_progress = None - self._function_call_finished = None - elif isinstance(frame, FunctionCallInProgressFrame): - self._function_call_in_progress = frame - elif isinstance(frame, FunctionCallResultFrame): - if ( - self._function_call_in_progress - and self._function_call_in_progress.tool_call_id == frame.tool_call_id - ): - self._function_call_in_progress = None - self._function_call_result = frame - await self.push_aggregation() - else: - logger.warning( - "FunctionCallResultFrame tool_call_id != InProgressFrame tool_call_id" - ) - self._function_call_in_progress = None - self._function_call_result = None - elif isinstance(frame, AnthropicImageMessageFrame): - self._pending_image_frame_message = frame - await self.push_aggregation() + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + assistant_message = {"role": "assistant", "content": []} + assistant_message["content"].append( + { + "type": "tool_use", + "id": frame.tool_call_id, + "name": frame.function_name, + "input": frame.arguments, + } + ) + self._context.add_message(assistant_message) + self._context.add_message( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": frame.tool_call_id, + "content": "IN_PROGRESS", + } + ], + } + ) - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + if not frame.result: return - run_llm = False - properties: Optional[FunctionCallResultProperties] = None + result = json.dumps(frame.result) - aggregation = self._aggregation.strip() - self.reset() + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) - try: - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - assistant_message = {"role": "assistant", "content": []} - assistant_message["content"].append( - { - "type": "tool_use", - "id": frame.tool_call_id, - "name": frame.function_name, - "input": frame.arguments, - } - ) - self._context.add_message(assistant_message) - self._context.add_message( - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": frame.tool_call_id, - "content": json.dumps(frame.result), - } - ], - } - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior - run_llm = True + async def _update_function_call_result( + self, function_name: str, tool_call_id: str, result: str + ): + for message in self._context.messages: + if message["role"] == "user": + for content in message["content"]: + if ( + isinstance(content, dict) + and content["type"] == "tool_result" + and content["tool_use_id"] == tool_call_id + ): + content["content"] = result - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - # Push context frame - await self.push_context_frame() - - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - - except Exception as e: - logger.error(f"Error processing frame: {e}") + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 9484fc27d..da1697da6 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -39,6 +39,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, + UserImageMessageFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -118,10 +119,10 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def push_aggregation(self): - # We don't want to store any images in the context. Revisit this later when the API evolves. - self._pending_image_frame_message = None - await super().push_aggregation() + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + # We don't want to store any images in the context. Revisit this later + # when the API evolves. + pass @dataclass diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index c10e67531..c07707899 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -10,6 +10,7 @@ import io import json import os import time +import uuid from google.api_core.exceptions import DeadlineExceeded from openai import AsyncStream @@ -33,20 +34,22 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, - FunctionCallResultProperties, + FunctionCallCancelFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - OpenAILLMContextAssistantTimestampFrame, StartFrame, TranscriptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, + UserImageMessageFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -565,91 +568,69 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator): class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): + async def handle_aggregation(self, aggregation: str): + self._context.add_message(glm.Content(role="model", parts=[glm.Part(text=aggregation)])) + + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + self._context.add_message( + glm.Content( + role="model", + parts=[ + glm.Part( + function_call=glm.FunctionCall( + id=frame.tool_call_id, name=frame.function_name, args=frame.arguments + ) + ) + ], + ) + ) + self._context.add_message( + glm.Content( + role="user", + parts=[ + glm.Part( + function_response=glm.FunctionResponse( + id=frame.tool_call_id, + name=frame.function_name, + response={"response": "IN_PROGRESS"}, + ) + ) + ], + ) + ) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + if not frame.result: return - run_llm = False - properties: Optional[FunctionCallResultProperties] = None + if not isinstance(frame.result, str): + return - aggregation = self._aggregation.strip() - self.reset() + response = {"response": frame.result} - try: - if aggregation: - self._context.add_message( - glm.Content(role="model", parts=[glm.Part(text=aggregation)]) - ) + await self._update_function_call_result(frame.function_name, frame.tool_call_id, response) - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - logger.debug(f"FunctionCallResultFrame result: {frame.arguments}") - self._context.add_message( - glm.Content( - role="model", - parts=[ - glm.Part( - function_call=glm.FunctionCall( - name=frame.function_name, args=frame.arguments - ) - ) - ], - ) - ) - response = frame.result - if isinstance(response, str): - response = {"response": response} - self._context.add_message( - glm.Content( - role="user", - parts=[ - glm.Part( - function_response=glm.FunctionResponse( - name=frame.function_name, response=response - ) - ) - ], - ) - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True + async def _update_function_call_result( + self, function_name: str, tool_call_id: str, result: Any + ): + for message in self._context.messages: + if message.role == "user": + for part in message.parts: + if part.function_response and part.function_response.id == tool_call_id: + part.function_response.response = result - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - # Push context frame - await self.push_context_frame() - - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - - except Exception as e: - logger.exception(f"Error processing frame: {e}") + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) @dataclass @@ -1071,7 +1052,7 @@ class GoogleLLMService(LLMService): args = type(c.function_call).to_dict(c.function_call).get("args", {}) await self.call_function( context=context, - tool_call_id="what_should_this_be", + tool_call_id=str(uuid.uuid4()), function_name=c.function_call.name, arguments=args, ) diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index cf7d74f59..faed13050 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -25,94 +25,15 @@ from pipecat.services.openai import ( ) -class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Custom assistant context aggregator for Grok that handles empty content requirement.""" - - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): - return - - run_llm = False - properties: Optional[FunctionCallResultProperties] = None - - aggregation = self._aggregation.strip() - self.reset() - - try: - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) - - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - # Grok requires an empty content field for function calls - self._context.add_message( - { - "role": "assistant", - "content": "", # Required by Grok - "tool_calls": [ - { - "id": frame.tool_call_id, - "function": { - "name": frame.function_name, - "arguments": json.dumps(frame.arguments), - }, - "type": "function", - } - ], - } - ) - self._context.add_message( - { - "role": "tool", - "content": json.dumps(frame.result), - "tool_call_id": frame.tool_call_id, - } - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) - - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - await self.push_context_frame() - - except Exception as e: - logger.error(f"Error processing frame: {e}") - - @dataclass class GrokContextAggregatorPair: _user: "OpenAIUserContextAggregator" - _assistant: "GrokAssistantContextAggregator" + _assistant: "OpenAIAssistantContextAggregator" def user(self) -> "OpenAIUserContextAggregator": return self._user - def assistant(self) -> "GrokAssistantContextAggregator": + def assistant(self) -> "OpenAIAssistantContextAggregator": return self._assistant @@ -235,5 +156,5 @@ class GrokLLMService(OpenAILLMService): context.set_llm_adapter(self.get_llm_adapter()) user = OpenAIUserContextAggregator(context, **user_kwargs) - assistant = GrokAssistantContextAggregator(context, **assistant_kwargs) + assistant = OpenAIAssistantContextAggregator(context, **assistant_kwargs) return GrokContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 89bf4a04b..700f1c0b5 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -27,21 +27,20 @@ from pydantic import BaseModel, Field from pipecat.frames.frames import ( ErrorFrame, Frame, + FunctionCallCancelFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, - FunctionCallResultProperties, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - OpenAILLMContextAssistantTimestampFrame, StartFrame, - StartInterruptionFrame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, + UserImageMessageFrame, UserImageRawFrame, UserImageRequestFrame, VisionImageRawFrame, @@ -63,7 +62,6 @@ from pipecat.services.ai_services import ( ) from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription from pipecat.transcriptions.language import Language -from pipecat.utils.time import time_now_iso8601 ValidVoice = Literal["alloy", "echo", "fable", "onyx", "nova", "shimmer"] @@ -558,13 +556,6 @@ class OpenAITTSService(TTSService): logger.exception(f"{self} error generating TTS: {e}") -# internal use only -- todo: refactor -@dataclass -class OpenAIImageMessageFrame(Frame): - user_image_raw_frame: UserImageRawFrame - text: Optional[str] = None - - class OpenAIUserContextAggregator(LLMUserContextAggregator): def __init__(self, context: OpenAILLMContext, **kwargs): super().__init__(context=context, **kwargs) @@ -596,7 +587,7 @@ class OpenAIUserContextAggregator(LLMUserContextAggregator): text = self._context._user_image_request_context.get(frame.user_id) or "" if text: del self._context._user_image_request_context[frame.user_id] - frame = OpenAIImageMessageFrame(user_image_raw_frame=frame, text=text) + frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) await self.push_frame(frame) except Exception as e: logger.error(f"Error processing frame: {e}") @@ -605,109 +596,59 @@ class OpenAIUserContextAggregator(LLMUserContextAggregator): class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): def __init__(self, context: OpenAILLMContext, **kwargs): super().__init__(context=context, **kwargs) - self._function_calls_in_progress = {} - self._function_call_result = None - self._pending_image_frame_message = None - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # See note above about not calling push_frame() here. - if isinstance(frame, StartInterruptionFrame): - self._function_calls_in_progress.clear() - self._function_call_finished = None - elif isinstance(frame, FunctionCallInProgressFrame): - logger.debug(f"FunctionCallInProgressFrame: {frame}") - self._function_calls_in_progress[frame.tool_call_id] = frame - elif isinstance(frame, FunctionCallResultFrame): - logger.debug(f"FunctionCallResultFrame: {frame}") - if frame.tool_call_id in self._function_calls_in_progress: - del self._function_calls_in_progress[frame.tool_call_id] - self._function_call_result = frame - # TODO-CB: Kwin wants us to refactor this out of here but I REFUSE - await self.push_aggregation() - else: - logger.warning( - "FunctionCallResultFrame tool_call_id does not match any function call in progress" - ) - self._function_call_result = None - elif isinstance(frame, OpenAIImageMessageFrame): - self._pending_image_frame_message = frame - await self.push_aggregation() + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + self._context.add_message( + { + "role": "assistant", + "tool_calls": [ + { + "id": frame.tool_call_id, + "function": { + "name": frame.function_name, + "arguments": json.dumps(frame.arguments), + }, + "type": "function", + } + ], + } + ) + self._context.add_message( + { + "role": "tool", + "content": "IN_PROGRESS", + "tool_call_id": frame.tool_call_id, + } + ) - async def push_aggregation(self): - if not ( - self._aggregation or self._function_call_result or self._pending_image_frame_message - ): + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + if not frame.result: return - run_llm = False - properties: Optional[FunctionCallResultProperties] = None + result = json.dumps(frame.result) - aggregation = self._aggregation.strip() - self.reset() + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) - try: - if aggregation: - self._context.add_message({"role": "assistant", "content": aggregation}) + async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) - if self._function_call_result: - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - self._context.add_message( - { - "role": "assistant", - "tool_calls": [ - { - "id": frame.tool_call_id, - "function": { - "name": frame.function_name, - "arguments": json.dumps(frame.arguments), - }, - "type": "function", - } - ], - } - ) - self._context.add_message( - { - "role": "tool", - "content": json.dumps(frame.result), - "tool_call_id": frame.tool_call_id, - } - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) + async def _update_function_call_result( + self, function_name: str, tool_call_id: str, result: str + ): + for message in self._context.messages: + if ( + message["role"] == "tool" + and message["tool_call_id"] + and message["tool_call_id"] == tool_call_id + ): + message["content"] = result - if self._pending_image_frame_message: - frame = self._pending_image_frame_message - self._pending_image_frame_message = None - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, - ) - run_llm = True - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - # Push context frame - await self.push_context_frame() - - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - - except Exception as e: - logger.error(f"Error processing frame: {e}") + async def handle_image_frame_message(self, frame: UserImageMessageFrame): + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index d4b8c35ce..0c9b6d5e4 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -418,7 +418,7 @@ class BaseTestUserContextAggregator: class BaseTestAssistantContextAggreagator: CONTEXT_CLASS = None # To be set in subclasses AGGREGATOR_CLASS = None # To be set in subclasses - EXPECTED_CONTEXT_FRAMES = [OpenAILLMContextFrame] + EXPECTED_CONTEXT_FRAMES = None # To be set in subclasses def check_message_content(self, context: OpenAILLMContext, index: int, content: str): assert context.messages[index]["content"] == content @@ -577,6 +577,7 @@ class TestLLMAssistantContextAggregator( ): CONTEXT_CLASS = OpenAILLMContext AGGREGATOR_CLASS = LLMAssistantContextAggregator + EXPECTED_CONTEXT_FRAMES = [OpenAILLMContextFrame, OpenAILLMContextAssistantTimestampFrame] # From c15286b14836e35fbbf94eb5e2f39ce38761f79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 14:57:49 -0700 Subject: [PATCH 348/427] examples: deprecate start_callback from LLMService.register_function() --- examples/foundational/14-function-calling.py | 12 ++---- .../14c-function-calling-together.py | 11 ++--- .../14e-function-calling-gemini.py | 9 +---- .../foundational/14f-function-calling-groq.py | 11 ++--- .../foundational/14g-function-calling-grok.py | 11 +---- .../14h-function-calling-azure.py | 11 ++--- .../14i-function-calling-fireworks.py | 13 ++---- .../foundational/14j-function-calling-nim.py | 11 ++--- .../14k-function-calling-cerebras.py | 11 ++--- .../14l-function-calling-deepseek.py | 11 ++--- .../14m-function-calling-openrouter.py | 11 ++--- ...o-function-calling-gemini-openai-format.py | 13 ++---- .../14p-function-calling-gemini-vertex-ai.py | 13 ++---- .../22b-natural-conversation-proposal.py | 11 ++--- .../22c-natural-conversation-mixed-llms.py | 9 +---- examples/foundational/24-stt-mute-filter.py | 6 +-- examples/patient-intake/bot.py | 40 ++++++++++--------- 17 files changed, 67 insertions(+), 147 deletions(-) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 1d4e37344..dddfed1d2 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -30,13 +30,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -62,9 +57,10 @@ async def main(): ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # Register a function_name of None to get all functions + + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index dd3eb714f..94d587799 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -66,9 +61,9 @@ async def main(): api_key=os.getenv("TOGETHER_API_KEY"), model="meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index bf022a65a..f0003fbab 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -33,13 +33,8 @@ logger.add(sys.stderr, level="DEBUG") video_participant_id = None -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) location = arguments["location"] await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") @@ -72,7 +67,7 @@ async def main(): ) llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") - llm.register_function("get_weather", get_weather, start_fetch_weather) + llm.register_function("get_weather", get_weather) llm.register_function("get_image", get_image) weather_function = FunctionSchema( diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index c402ac91a..9818d185f 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -65,9 +60,9 @@ async def main(): ) llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index e6b822e39..e5bb2f9b3 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -16,7 +16,6 @@ from runner import configure from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -31,12 +30,6 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,9 +56,9 @@ async def main(): ) llm = GrokLLMService(api_key=os.getenv("GROK_API_KEY")) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index aefa1a474..6c8465832 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -67,9 +62,9 @@ async def main(): endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"), ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index 50291615b..887e734dd 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -64,11 +59,11 @@ async def main(): llm = FireworksLLMService( api_key=os.getenv("FIREWORKS_API_KEY"), - model="accounts/fireworks/models/firefunction-v2", + model="accounts/fireworks/models/llama-v3p1-405b-instruct", ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index ea8e25cf6..c628bedf0 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -66,9 +61,9 @@ async def main(): llm = NimLLMService( api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 9b3641307..541e6f250 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,9 +58,9 @@ async def main(): ) llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14l-function-calling-deepseek.py b/examples/foundational/14l-function-calling-deepseek.py index 0360f0b84..812a6716e 100644 --- a/examples/foundational/14l-function-calling-deepseek.py +++ b/examples/foundational/14l-function-calling-deepseek.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,9 +58,9 @@ async def main(): ) llm = DeepSeekLLMService(api_key=os.getenv("DEEPSEEK_API_KEY"), model="deepseek-chat") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14m-function-calling-openrouter.py b/examples/foundational/14m-function-calling-openrouter.py index e816f6322..a675c8ac4 100644 --- a/examples/foundational/14m-function-calling-openrouter.py +++ b/examples/foundational/14m-function-calling-openrouter.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -67,9 +62,9 @@ async def main(): llm = OpenRouterLLMService( api_key=os.getenv("OPENROUTER_API_KEY"), model="openai/gpt-4o-2024-11-20" ) - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14o-function-calling-gemini-openai-format.py b/examples/foundational/14o-function-calling-gemini-openai-format.py index a1bb53b6b..78100f0d3 100644 --- a/examples/foundational/14o-function-calling-gemini-openai-format.py +++ b/examples/foundational/14o-function-calling-gemini-openai-format.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -63,11 +58,9 @@ async def main(): ) llm = GoogleLLMOpenAIBetaService(api_key=os.getenv("GEMINI_API_KEY")) - # Register a function_name of None to get all functions + # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function( - "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather - ) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/14p-function-calling-gemini-vertex-ai.py b/examples/foundational/14p-function-calling-gemini-vertex-ai.py index d64bae89d..8888a2922 100644 --- a/examples/foundational/14p-function-calling-gemini-vertex-ai.py +++ b/examples/foundational/14p-function-calling-gemini-vertex-ai.py @@ -31,13 +31,8 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -68,11 +63,9 @@ async def main(): project_id="", ) ) - # Register a function_name of None to get all functions + # You can aslo register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function( - "get_current_weather", fetch_weather_from_api, start_callback=start_fetch_weather - ) + llm.register_function("get_current_weather", fetch_weather_from_api) weather_function = FunctionSchema( name="get_current_weather", diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index c97289217..d7f7eb597 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -199,13 +199,8 @@ class OutputGate(FrameProcessor): break -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -239,9 +234,9 @@ async def main(): # This is the regular LLM. llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - # Register a function_name of None to get all functions + # You can also register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) tools = [ ChatCompletionToolParam( diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index d342a2765..7fa1bb5fa 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -403,13 +403,8 @@ class OutputGate(FrameProcessor): break -async def start_fetch_weather(function_name, llm, context): - """Push a frame to the LLM; this is handy when the LLM response might take a while.""" - await llm.push_frame(TTSSpeakFrame("Let me check on that.")) - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await llm.push_frame(TTSSpeakFrame("Let me check on that.")) await result_callback({"conditions": "nice", "temperature": "75"}) @@ -451,7 +446,7 @@ async def main(): ) # Register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) tools = [ ChatCompletionToolParam( diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index 7ae6c73fa..d2b7174d1 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -30,10 +30,6 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -async def start_fetch_weather(function_name, llm, context): - logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") - - async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): # Add a delay to test interruption during function calls logger.info("Weather API call starting...") @@ -72,7 +68,7 @@ async def main(): tts = DeepgramTTSService(api_key=os.getenv("DEEPGRAM_API_KEY"), voice="aura-helios-en") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") - llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + llm.register_function("get_current_weather", fetch_weather_from_api) tools = [ ChatCompletionToolParam( diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index b6f7fb941..de6ad2051 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -142,7 +142,9 @@ class IntakeProcessor: ] ) - async def start_prescriptions(self, function_name, llm, context): + async def list_prescriptions( + self, function_name, tool_call_id, args, llm, context, result_callback + ): print(f"!!! doing start prescriptions") # Move on to allergies context.set_tools( @@ -182,9 +184,12 @@ class IntakeProcessor: print(f"!!! about to await llm process frame in start prescrpitions") await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) print(f"!!! past await process frame in start prescriptions") + await self.save_data(args, result_callback) - async def start_allergies(self, function_name, llm, context): - print("!!! doing start allergies") + async def list_allergies( + self, function_name, tool_call_id, args, llm, context, result_callback + ): + print("!!! doing list allergies") # Move on to conditions context.set_tools( [ @@ -221,8 +226,11 @@ class IntakeProcessor: } ) await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) + await self.save_data(args, result_callback) - async def start_conditions(self, function_name, llm, context): + async def list_conditions( + self, function_name, tool_call_id, args, llm, context, result_callback + ): print("!!! doing start conditions") # Move on to visit reasons context.set_tools( @@ -260,8 +268,11 @@ class IntakeProcessor: } ) await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) + await self.save_data(args, result_callback) - async def start_visit_reasons(self, function_name, llm, context): + async def list_visit_reasons( + self, function_name, tool_call_id, args, llm, context, result_callback + ): print("!!! doing start visit reasons") # move to finish call context.set_tools([]) @@ -269,8 +280,9 @@ class IntakeProcessor: {"role": "system", "content": "Now, thank the user and end the conversation."} ) await llm.queue_frame(OpenAILLMContextFrame(context), FrameDirection.DOWNSTREAM) + await self.save_data(args, result_callback) - async def save_data(self, function_name, tool_call_id, args, llm, context, result_callback): + async def save_data(self, args, result_callback): logger.info(f"!!! Saving data: {args}") # Since this is supposed to be "async", returning None from the callback # will prevent adding anything to context or re-prompting @@ -319,18 +331,10 @@ async def main(): intake = IntakeProcessor(context) llm.register_function("verify_birthday", intake.verify_birthday) - llm.register_function( - "list_prescriptions", intake.save_data, start_callback=intake.start_prescriptions - ) - llm.register_function( - "list_allergies", intake.save_data, start_callback=intake.start_allergies - ) - llm.register_function( - "list_conditions", intake.save_data, start_callback=intake.start_conditions - ) - llm.register_function( - "list_visit_reasons", intake.save_data, start_callback=intake.start_visit_reasons - ) + llm.register_function("list_prescriptions", intake.list_prescriptions) + llm.register_function("list_allergies", intake.list_allergies) + llm.register_function("list_conditions", intake.list_conditions) + llm.register_function("list_visit_reasons", intake.list_visit_reasons) fl = FrameLogger("LLM Output") From d1550d5a854ca1617068a67d368e28bfedb00b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 15:52:37 -0700 Subject: [PATCH 349/427] tests: remove TestFrameProcessor, reimplement with run_test() --- src/pipecat/tests/utils.py | 42 +++--- src/pipecat/utils/test_frame_processor.py | 48 ------- tests/integration/integration_azure_llm.py | 33 ----- tests/integration/integration_ollama_llm.py | 28 ---- tests/integration/integration_openai_llm.py | 128 ------------------ ...st_integration_unified_function_calling.py | 27 ++-- 6 files changed, 37 insertions(+), 269 deletions(-) delete mode 100644 src/pipecat/utils/test_frame_processor.py delete mode 100644 tests/integration/integration_azure_llm.py delete mode 100644 tests/integration/integration_ollama_llm.py delete mode 100644 tests/integration/integration_openai_llm.py diff --git a/src/pipecat/tests/utils.py b/src/pipecat/tests/utils.py index d68c87d88..e2368ba09 100644 --- a/src/pipecat/tests/utils.py +++ b/src/pipecat/tests/utils.py @@ -6,7 +6,7 @@ import asyncio from dataclasses import dataclass -from typing import Any, Awaitable, Callable, Dict, Sequence, Tuple +from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple from pipecat.frames.frames import ( EndFrame, @@ -80,8 +80,8 @@ async def run_test( processor: FrameProcessor, *, frames_to_send: Sequence[Frame], - expected_down_frames: Sequence[type], - expected_up_frames: Sequence[type] = [], + expected_down_frames: Optional[Sequence[type]] = None, + expected_up_frames: Optional[Sequence[type]] = None, ignore_start: bool = True, start_metadata: Dict[str, Any] = {}, send_end_frame: bool = True, @@ -126,33 +126,35 @@ async def run_test( # Down frames # received_down_frames: Sequence[Frame] = [] - while not received_down.empty(): - frame = await received_down.get() - if not isinstance(frame, EndFrame) or not send_end_frame: - received_down_frames.append(frame) + if expected_down_frames is not None: + while not received_down.empty(): + frame = await received_down.get() + if not isinstance(frame, EndFrame) or not send_end_frame: + received_down_frames.append(frame) - print("received DOWN frames =", received_down_frames) - print("expected DOWN frames =", expected_down_frames) + print("received DOWN frames =", received_down_frames) + print("expected DOWN frames =", expected_down_frames) - assert len(received_down_frames) == len(expected_down_frames) + assert len(received_down_frames) == len(expected_down_frames) - for real, expected in zip(received_down_frames, expected_down_frames): - assert isinstance(real, expected) + for real, expected in zip(received_down_frames, expected_down_frames): + assert isinstance(real, expected) # # Up frames # received_up_frames: Sequence[Frame] = [] - while not received_up.empty(): - frame = await received_up.get() - received_up_frames.append(frame) + if expected_up_frames is not None: + while not received_up.empty(): + frame = await received_up.get() + received_up_frames.append(frame) - print("received UP frames =", received_up_frames) - print("expected UP frames =", expected_up_frames) + print("received UP frames =", received_up_frames) + print("expected UP frames =", expected_up_frames) - assert len(received_up_frames) == len(expected_up_frames) + assert len(received_up_frames) == len(expected_up_frames) - for real, expected in zip(received_up_frames, expected_up_frames): - assert isinstance(real, expected) + for real, expected in zip(received_up_frames, expected_up_frames): + assert isinstance(real, expected) return (received_down_frames, received_up_frames) diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py deleted file mode 100644 index b35864497..000000000 --- a/src/pipecat/utils/test_frame_processor.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import List - -from pipecat.processors.frame_processor import FrameProcessor - - -class TestException(Exception): - pass - - -class TestFrameProcessor(FrameProcessor): - __test__ = False # Prevents pytest from collecting this class as a test - - def __init__(self, test_frames): - self.test_frames = test_frames - self._list_counter = 0 - super().__init__() - - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - - if not self.test_frames[ - 0 - ]: # then we've run out of required frames but the generator is still going? - raise TestException(f"Oops, got an extra frame, {frame}") - if isinstance(self.test_frames[0], List): - # We need to consume frames until we see the next frame type after this - next_frame = self.test_frames[1] - if isinstance(frame, next_frame): - # we're done iterating the list I guess - print(f"TestFrameProcessor got expected list exit frame: {frame}") - # pop twice to get rid of the list, as well as the next frame - self.test_frames.pop(0) - self.test_frames.pop(0) - self.list_counter = 0 - else: - fl = self.test_frames[0] - fl_el = fl[self._list_counter % len(fl)] - if isinstance(frame, fl_el): - print(f"TestFrameProcessor got expected list frame: {frame}") - self._list_counter += 1 - else: - raise TestException(f"Inside a list, expected {fl_el} but got {frame}") - - else: - if not isinstance(frame, self.test_frames[0]): - raise TestException(f"Expected {self.test_frames[0]}, but got {frame}") - print(f"TestFrameProcessor got expected frame: {frame}") - self.test_frames.pop(0) diff --git a/tests/integration/integration_azure_llm.py b/tests/integration/integration_azure_llm.py deleted file mode 100644 index 8e49e9d04..000000000 --- a/tests/integration/integration_azure_llm.py +++ /dev/null @@ -1,33 +0,0 @@ -import asyncio -import os -import unittest - -from openai.types.chat import ( - ChatCompletionSystemMessageParam, -) - -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.services.azure import AzureLLMService - -if __name__ == "__main__": - - @unittest.skip("Skip azure integration test") - async def test_chat(): - llm = AzureLLMService( - api_key=os.getenv("AZURE_CHATGPT_API_KEY"), - endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), - model=os.getenv("AZURE_CHATGPT_MODEL"), - ) - context = OpenAILLMContext() - message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Please tell the world hello.", name="system", role="system" - ) - context.add_message(message) - frame = OpenAILLMContextFrame(context) - async for s in llm.process_frame(frame): - print(s) - - asyncio.run(test_chat()) diff --git a/tests/integration/integration_ollama_llm.py b/tests/integration/integration_ollama_llm.py deleted file mode 100644 index 085500cb8..000000000 --- a/tests/integration/integration_ollama_llm.py +++ /dev/null @@ -1,28 +0,0 @@ -import asyncio -import unittest - -from openai.types.chat import ( - ChatCompletionSystemMessageParam, -) - -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.services.ollama import OLLamaLLMService - -if __name__ == "__main__": - - @unittest.skip("Skip azure integration test") - async def test_chat(): - llm = OLLamaLLMService() - context = OpenAILLMContext() - message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Please tell the world hello.", name="system", role="system" - ) - context.add_message(message) - frame = OpenAILLMContextFrame(context) - async for s in llm.process_frame(frame): - print(s) - - asyncio.run(test_chat()) diff --git a/tests/integration/integration_openai_llm.py b/tests/integration/integration_openai_llm.py deleted file mode 100644 index 2c84c8882..000000000 --- a/tests/integration/integration_openai_llm.py +++ /dev/null @@ -1,128 +0,0 @@ -import asyncio -import json -import os -from typing import List - -from openai.types.chat import ( - ChatCompletionSystemMessageParam, - ChatCompletionToolParam, - ChatCompletionUserMessageParam, -) - -from pipecat.frames.frames import LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService -from pipecat.utils.test_frame_processor import TestFrameProcessor - -tools = [ - ChatCompletionToolParam( - type="function", - function={ - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], - }, - }, - ) -] - -if __name__ == "__main__": - - async def test_simple_functions(): - async def get_weather_from_api(llm, args): - return json.dumps({"conditions": "nice", "temperature": "75"}) - - api_key = os.getenv("OPENAI_API_KEY") - - llm = OpenAILLMService( - api_key=api_key or "", - model="gpt-4-1106-preview", - ) - - llm.register_function("get_current_weather", get_weather_from_api) - t = TestFrameProcessor([LLMFullResponseStartFrame, TextFrame, LLMFullResponseEndFrame]) - llm.link(t) - - context = OpenAILLMContext(tools=tools) - system_message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Ask the user to ask for a weather report", name="system", role="system" - ) - user_message: ChatCompletionUserMessageParam = ChatCompletionUserMessageParam( - content="Could you tell me the weather for Boulder, Colorado", - name="user", - role="user", - ) - context.add_message(system_message) - context.add_message(user_message) - frame = OpenAILLMContextFrame(context) - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) - - async def test_advanced_functions(): - async def get_weather_from_api(llm, args): - return [ - { - "role": "system", - "content": "The user has asked for live weather. Respond by telling them we don't currently support live weather for that area, but it's coming soon.", - } - ] - - api_key = os.getenv("OPENAI_API_KEY") - - llm = OpenAILLMService( - api_key=api_key or "", - model="gpt-4-1106-preview", - ) - - llm.register_function("get_current_weather", get_weather_from_api) - t = TestFrameProcessor([LLMFullResponseStartFrame, TextFrame, LLMFullResponseEndFrame]) - llm.link(t) - - context = OpenAILLMContext(tools=tools) - system_message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Ask the user to ask for a weather report", name="system", role="system" - ) - user_message: ChatCompletionUserMessageParam = ChatCompletionUserMessageParam( - content="Could you tell me the weather for Boulder, Colorado", - name="user", - role="user", - ) - context.add_message(system_message) - context.add_message(user_message) - frame = OpenAILLMContextFrame(context) - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) - - async def test_chat(): - api_key = os.getenv("OPENAI_API_KEY") - t = TestFrameProcessor([LLMFullResponseStartFrame, TextFrame, LLMFullResponseEndFrame]) - llm = OpenAILLMService( - api_key=api_key or "", - model="gpt-4o", - ) - llm.link(t) - context = OpenAILLMContext() - message: ChatCompletionSystemMessageParam = ChatCompletionSystemMessageParam( - content="Please tell the world hello.", name="system", role="system" - ) - context.add_message(message) - frame = OpenAILLMContextFrame(context) - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) - - async def run_tests(): - await test_simple_functions() - await test_advanced_functions() - await test_chat() - - asyncio.run(run_tests()) diff --git a/tests/integration/test_integration_unified_function_calling.py b/tests/integration/test_integration_unified_function_calling.py index 88407d703..0696c531a 100644 --- a/tests/integration/test_integration_unified_function_calling.py +++ b/tests/integration/test_integration_unified_function_calling.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024-2025 Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import os from unittest.mock import AsyncMock @@ -6,17 +12,12 @@ from dotenv import load_dotenv from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema -from pipecat.frames.frames import ( - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, -) -from pipecat.processors.frame_processor import FrameDirection +from pipecat.pipeline.pipeline import Pipeline from pipecat.services.ai_services import LLMService from pipecat.services.anthropic import AnthropicLLMService from pipecat.services.google import GoogleLLMService from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService -from pipecat.utils.test_frame_processor import TestFrameProcessor +from pipecat.tests.utils import run_test load_dotenv(override=True) @@ -47,8 +48,6 @@ async def _test_llm_function_calling(llm: LLMService): mock_fetch_weather = AsyncMock() llm.register_function(None, mock_fetch_weather) - t = TestFrameProcessor([LLMFullResponseStartFrame, LLMTextFrame, LLMFullResponseEndFrame]) - llm.link(t) messages = [ { @@ -61,10 +60,14 @@ async def _test_llm_function_calling(llm: LLMService): # This is done by default inside the create_context_aggregator context.set_llm_adapter(llm.get_llm_adapter()) - frame = OpenAILLMContextFrame(context) + pipeline = Pipeline([llm]) - # This will fail if an exception is raised - await llm.process_frame(frame, FrameDirection.DOWNSTREAM) + frames_to_send = [OpenAILLMContextFrame(context)] + await run_test( + pipeline, + frames_to_send=frames_to_send, + expected_down_frames=None, + ) # Assert that the mock function was called mock_fetch_weather.assert_called_once() From d455fd070eebedcbab3fbf932b09d724dd39df55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 18 Mar 2025 16:43:43 -0700 Subject: [PATCH 350/427] update CHANGELOG --- CHANGELOG.md | 3 +++ src/pipecat/services/google/google.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4762186cd..247c3280d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 being aggregated. A text aggregator can be passed via `text_aggregator` to the TTS service. +- Added new `sample_rate` constructor parameter to `TavusVideoService` to allow + changing the output sample rate. + - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index c07707899..3db2044db 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -1316,9 +1316,12 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): class GoogleVertexLLMService(OpenAILLMService): - """Implements inference with Google's AI models via Vertex AI while maintaining OpenAI API compatibility. + """Implements inference with Google's AI models via Vertex AI while + maintaining OpenAI API compatibility. + Reference: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library + """ class InputParams(OpenAILLMService.InputParams): From 3e9678db8430101a7964d9f3e246a6db48355c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 13:12:42 -0700 Subject: [PATCH 351/427] user image requests can now be related to function calls --- .../14b-function-calling-anthropic-video.py | 7 ++- .../14d-function-calling-video.py | 9 ++- .../14e-function-calling-gemini.py | 7 ++- .../20d-persistent-context-gemini.py | 7 ++- examples/moondream-chatbot/bot.py | 1 - src/pipecat/frames/frames.py | 25 ++++---- .../processors/aggregators/llm_response.py | 12 ++-- .../aggregators/openai_llm_context.py | 1 - src/pipecat/services/ai_services.py | 17 +++++- src/pipecat/services/anthropic.py | 57 ++++--------------- .../services/gemini_multimodal_live/gemini.py | 4 +- src/pipecat/services/google/google.py | 15 +++-- src/pipecat/services/openai.py | 55 ++++-------------- src/pipecat/transports/services/daily.py | 21 ++++--- 14 files changed, 101 insertions(+), 137 deletions(-) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 302722be2..cd9777b72 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -39,7 +39,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def main(): diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 6e290d55f..a7e997f9b 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -39,7 +39,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}") question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def main(): @@ -141,7 +146,7 @@ indicate you should use the get_image tool are: await transport.capture_participant_transcription(participant["id"]) await transport.capture_participant_video(video_participant_id, framerate=0) # Kick off the conversation. - await tts.say("Hi! Ask me about the weather in San Francisco.") + await task.queue_frames([context_aggregator.user().get_context_frame()]) runner = PipelineRunner() diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index f0003fbab..8448643b4 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -42,7 +42,12 @@ async def get_weather(function_name, tool_call_id, arguments, llm, context, resu async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): logger.debug(f"!!! IN get_image {video_participant_id}, {arguments}") question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def main(): diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 740bdc68f..2381ea131 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -54,7 +54,12 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context async def get_image(function_name, tool_call_id, arguments, llm, context, result_callback): question = arguments["question"] - await llm.request_image_frame(user_id=video_participant_id, text_content=question) + await llm.request_image_frame( + user_id=video_participant_id, + function_name=function_name, + tool_call_id=tool_call_id, + text_content=question, + ) async def get_saved_conversation_filenames( diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 5dd88f4d1..6639c08c1 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -23,7 +23,6 @@ from pipecat.frames.frames import ( OutputImageRawFrame, SpriteFrame, TextFrame, - UserImageRawFrame, UserImageRequestFrame, ) from pipecat.pipeline.parallel_pipeline import ParallelPipeline diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index e589e9480..c2a79461f 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -662,13 +662,19 @@ class TransportMessageUrgentFrame(SystemFrame): @dataclass class UserImageRequestFrame(SystemFrame): - """A frame user to request an image from the given user.""" + """A frame to request an image from the given user. The frame might be + generated by a function call in which case the corresponding fields will be + properly set. + + """ user_id: str context: Optional[Any] = None + function_name: Optional[str] = None + tool_call_id: Optional[str] = None def __str__(self): - return f"{self.name}, user: {self.user_id}" + return f"{self.name}(user: {self.user_id}, function: {self.function_name}, request: {self.tool_call_id})" @dataclass @@ -698,10 +704,11 @@ class UserImageRawFrame(InputImageRawFrame): """An image associated to a user.""" user_id: str + request: Optional[UserImageRequestFrame] = None def __str__(self): pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format})" + return f"{self.name}(pts: {pts}, user: {self.user_id}, size: {self.size}, format: {self.format}, request: {self.request})" @dataclass @@ -715,18 +722,6 @@ class VisionImageRawFrame(InputImageRawFrame): return f"{self.name}(pts: {pts}, text: [{self.text}], size: {self.size}, format: {self.format})" -@dataclass -class UserImageMessageFrame(SystemFrame): - """An image associated to a user.""" - - user_image_raw_frame: UserImageRawFrame - text: Optional[str] = None - - def __str__(self): - pts = format_pts(self.pts) - return f"{self.name}(pts: {pts}, image: {self.user_image_raw_frame}, text: {self.text})" - - # # Control frames # diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 494d1de38..ff1b6e1be 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -34,7 +34,7 @@ from pipecat.frames.frames import ( StartInterruptionFrame, TextFrame, TranscriptionFrame, - UserImageMessageFrame, + UserImageRawFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -401,7 +401,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): pass - async def handle_image_frame_message(self, frame: UserImageMessageFrame): + async def handle_user_image_frame(self, frame: UserImageRawFrame): pass async def process_frame(self, frame: Frame, direction: FrameDirection): @@ -428,8 +428,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self._handle_function_call_result(frame) elif isinstance(frame, FunctionCallCancelFrame): await self._handle_function_call_cancel(frame) - elif isinstance(frame, UserImageMessageFrame): - await self._handle_image_frame_message(frame) + elif isinstance(frame, UserImageRawFrame) and frame.request and frame.request.tool_call_id: + await self._handle_user_image_frame(frame) elif isinstance(frame, BotStoppedSpeakingFrame): await self.push_aggregation() else: @@ -510,8 +510,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self.handle_function_call_cancel(frame) del self._function_calls_in_progress[frame.tool_call_id] - async def _handle_image_frame_message(self, frame: UserImageMessageFrame): - await self.handle_image_frame_message(frame) + async def _handle_user_image_frame(self, frame: UserImageRawFrame): + await self.handle_user_image_frame(frame) await self.push_aggregation() await self.push_context_frame(FrameDirection.UPSTREAM) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 93c2875be..2e5ade0a0 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -46,7 +46,6 @@ class OpenAILLMContext: self._messages: List[ChatCompletionMessageParam] = messages if messages else [] self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools - self._user_image_request_context = {} self._llm_adapter: Optional[BaseLLMAdapter] = None def get_llm_adapter(self) -> Optional[BaseLLMAdapter]: diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 2f30f505f..678132cb6 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -253,9 +253,22 @@ class LLMService(AIService): elif None in self._start_callbacks.keys(): return await self._start_callbacks[None](function_name, self, context) - async def request_image_frame(self, user_id: str, *, text_content: Optional[str] = None): + async def request_image_frame( + self, + user_id: str, + *, + function_name: Optional[str] = None, + tool_call_id: Optional[str] = None, + text_content: Optional[str] = None, + ): await self.push_frame( - UserImageRequestFrame(user_id=user_id, context=text_content), FrameDirection.UPSTREAM + UserImageRequestFrame( + user_id=user_id, + function_name=function_name, + tool_call_id=tool_call_id, + context=text_content, + ), + FrameDirection.UPSTREAM, ) async def _run_function_call( diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 8e06f4558..a80e2154c 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -30,9 +30,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMTextFrame, LLMUpdateSettingsFrame, - UserImageMessageFrame, UserImageRawFrame, - UserImageRequestFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -674,42 +672,7 @@ class AnthropicLLMContext(OpenAILLMContext): class AnthropicUserContextAggregator(LLMUserContextAggregator): - def __init__(self, context: OpenAILLMContext | AnthropicLLMContext, **kwargs): - super().__init__(context=context, **kwargs) - - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # Our parent method has already called push_frame(). So we can't interrupt the - # flow here and we don't need to call push_frame() ourselves. Possibly something - # to talk through (tagging @aleix). At some point we might need to refactor these - # context aggregators. - try: - if isinstance(frame, UserImageRequestFrame): - # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with - # that frame so we can use it when we assemble the image message in the assistant - # context aggregator. - if frame.context: - if isinstance(frame.context, str): - self._context._user_image_request_context[frame.user_id] = frame.context - else: - logger.error( - f"Unexpected UserImageRequestFrame context type: {type(frame.context)}" - ) - del self._context._user_image_request_context[frame.user_id] - else: - if frame.user_id in self._context._user_image_request_context: - del self._context._user_image_request_context[frame.user_id] - elif isinstance(frame, UserImageRawFrame): - # Push a new AnthropicImageMessageFrame with the text context we cached - # downstream to be handled by our assistant context aggregator. This is - # necessary so that we add the message to the context in the right order. - text = self._context._user_image_request_context.get(frame.user_id) or "" - if text: - del self._context._user_image_request_context[frame.user_id] - frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) - await self.push_frame(frame) - except Exception as e: - logger.error(f"Error processing frame: {e}") + pass # @@ -723,9 +686,6 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): - def __init__(self, context: OpenAILLMContext | AnthropicLLMContext, **kwargs): - super().__init__(context=context, **kwargs) - async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): assistant_message = {"role": "assistant", "content": []} assistant_message["content"].append( @@ -776,10 +736,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ): content["content"] = result - async def handle_image_frame_message(self, frame: UserImageMessageFrame): - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, + async def handle_user_image_frame(self, frame: UserImageRawFrame): + await self._update_function_call_result( + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + ) + self._context.add_image_frame_message( + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.request.context, ) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index da1697da6..801af46b4 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -39,7 +39,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, TTSTextFrame, - UserImageMessageFrame, + UserImageRawFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) @@ -119,7 +119,7 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def handle_image_frame_message(self, frame: UserImageMessageFrame): + async def handle_user_image_frame(self, frame: UserImageRawFrame): # We don't want to store any images in the context. Revisit this later # when the API evolves. pass diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 3db2044db..128eb43b6 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -49,7 +49,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, - UserImageMessageFrame, + UserImageRawFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -624,12 +624,15 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if part.function_response and part.function_response.id == tool_call_id: part.function_response.response = result - async def handle_image_frame_message(self, frame: UserImageMessageFrame): + async def handle_user_image_frame(self, frame: UserImageRawFrame): + await self._update_function_call_result( + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + ) self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.request.context, ) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 700f1c0b5..8318739de 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -40,9 +40,7 @@ from pipecat.frames.frames import ( TTSStartedFrame, TTSStoppedFrame, URLImageRawFrame, - UserImageMessageFrame, UserImageRawFrame, - UserImageRequestFrame, VisionImageRawFrame, ) from pipecat.metrics.metrics import LLMTokenUsage @@ -557,46 +555,10 @@ class OpenAITTSService(TTSService): class OpenAIUserContextAggregator(LLMUserContextAggregator): - def __init__(self, context: OpenAILLMContext, **kwargs): - super().__init__(context=context, **kwargs) - - async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - # Our parent method has already called push_frame(). So we can't interrupt the - # flow here and we don't need to call push_frame() ourselves. - try: - if isinstance(frame, UserImageRequestFrame): - # The LLM sends a UserImageRequestFrame upstream. Cache any context provided with - # that frame so we can use it when we assemble the image message in the assistant - # context aggregator. - if frame.context: - if isinstance(frame.context, str): - self._context._user_image_request_context[frame.user_id] = frame.context - else: - logger.error( - f"Unexpected UserImageRequestFrame context type: {type(frame.context)}" - ) - del self._context._user_image_request_context[frame.user_id] - else: - if frame.user_id in self._context._user_image_request_context: - del self._context._user_image_request_context[frame.user_id] - elif isinstance(frame, UserImageRawFrame): - # Push a new OpenAIImageMessageFrame with the text context we cached - # downstream to be handled by our assistant context aggregator. This is - # necessary so that we add the message to the context in the right order. - text = self._context._user_image_request_context.get(frame.user_id) or "" - if text: - del self._context._user_image_request_context[frame.user_id] - frame = UserImageMessageFrame(user_image_raw_frame=frame, text=text) - await self.push_frame(frame) - except Exception as e: - logger.error(f"Error processing frame: {e}") + pass class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): - def __init__(self, context: OpenAILLMContext, **kwargs): - super().__init__(context=context, **kwargs) - async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): self._context.add_message( { @@ -645,10 +607,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ): message["content"] = result - async def handle_image_frame_message(self, frame: UserImageMessageFrame): - self._context.add_image_frame_message( - format=frame.user_image_raw_frame.format, - size=frame.user_image_raw_frame.size, - image=frame.user_image_raw_frame.image, - text=frame.text, + async def handle_user_image_frame(self, frame: UserImageRawFrame): + await self._update_function_call_result( + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + ) + self._context.add_image_frame_message( + format=frame.format, + size=frame.size, + image=frame.image, + text=frame.request.context, ) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index f4b83dfa7..af7d2308c 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -898,7 +898,7 @@ class DailyInputTransport(BaseInputTransport): await super().process_frame(frame, direction) if isinstance(frame, UserImageRequestFrame): - await self.request_participant_image(frame.user_id) + await self.request_participant_image(frame) # # Frames @@ -935,16 +935,16 @@ class DailyInputTransport(BaseInputTransport): self._video_renderers[participant_id] = { "framerate": framerate, "timestamp": 0, - "render_next_frame": False, + "render_next_frame": [], } await self._client.capture_participant_video( participant_id, self._on_participant_video_frame, framerate, video_source, color_format ) - async def request_participant_image(self, participant_id: str): - if participant_id in self._video_renderers: - self._video_renderers[participant_id]["render_next_frame"] = True + async def request_participant_image(self, frame: UserImageRequestFrame): + if frame.user_id in self._video_renderers: + self._video_renderers[frame.user_id]["render_next_frame"].append(frame) async def _on_participant_video_frame(self, participant_id: str, buffer, size, format): render_frame = False @@ -953,17 +953,24 @@ class DailyInputTransport(BaseInputTransport): prev_time = self._video_renderers[participant_id]["timestamp"] framerate = self._video_renderers[participant_id]["framerate"] + # Some times we render frames because of a request. + request_frame = None + if framerate > 0: next_time = prev_time + 1 / framerate render_frame = (next_time - curr_time) < 0.1 elif self._video_renderers[participant_id]["render_next_frame"]: - self._video_renderers[participant_id]["render_next_frame"] = False + request_frame = self._video_renderers[participant_id]["render_next_frame"].pop(0) render_frame = True if render_frame: frame = UserImageRawFrame( - user_id=participant_id, image=buffer, size=size, format=format + user_id=participant_id, + request=request_frame, + image=buffer, + size=size, + format=format, ) await self.push_frame(frame) self._video_renderers[participant_id]["timestamp"] = curr_time From 1f6ed01ba6b910a65cf8237bd69aff18800a43d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 14:06:22 -0700 Subject: [PATCH 352/427] LLMAssistantContextAggregator: remove tool call id with image requests --- src/pipecat/processors/aggregators/llm_response.py | 12 ++++++++++++ src/pipecat/services/ai_services.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index ff1b6e1be..9267af364 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -511,6 +511,18 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): del self._function_calls_in_progress[frame.tool_call_id] async def _handle_user_image_frame(self, frame: UserImageRawFrame): + logger.debug( + f"{self} UserImageRawFrame: [{frame.request.function_name}:{frame.request.tool_call_id}]" + ) + + if frame.request.tool_call_id not in self._function_calls_in_progress: + logger.warning( + f"UserImageRawFrame tool_call_id [{frame.request.tool_call_id}] is not running" + ) + return + + del self._function_calls_in_progress[frame.request.tool_call_id] + await self.handle_user_image_frame(frame) await self.push_aggregation() await self.push_context_frame(FrameDirection.UPSTREAM) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 678132cb6..9f9804e65 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -286,7 +286,9 @@ class LLMService(AIService): else: return - logger.info(f"Calling function {function_name} with arguments {arguments}") + logger.debug( + f"{self} Calling function [{function_name}:{tool_call_id}] with arguments {arguments}" + ) # NOTE(aleix): This needs to be removed after we remove the deprecation. await self.call_start_function(context, function_name) From b1d506c137cec966f9aca0fbb835e20ce5822183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 14:09:13 -0700 Subject: [PATCH 353/427] GoogleAssistantContextAggregator: properly update function response --- src/pipecat/services/google/google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 128eb43b6..7aa9b7b7b 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -622,7 +622,7 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if message.role == "user": for part in message.parts: if part.function_response and part.function_response.id == tool_call_id: - part.function_response.response = result + part.function_response.response = {"response": result} async def handle_user_image_frame(self, frame: UserImageRawFrame): await self._update_function_call_result( From e0c3f6ad832b8e583b188c34546dbec0edc6a021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 19 Mar 2025 14:34:31 -0700 Subject: [PATCH 354/427] services: mark function calls as completed even the result is None --- src/pipecat/services/anthropic.py | 13 +++++++------ src/pipecat/services/google/google.py | 18 +++++++++++------- src/pipecat/services/openai.py | 13 +++++++------ 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index a80e2154c..6a95d04e2 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -711,12 +711,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): - if not frame.result: - return - - result = json.dumps(frame.result) - - await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + if frame.result: + result = json.dumps(frame.result) + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + else: + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "COMPLETED" + ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): await self._update_function_call_result( diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 7aa9b7b7b..bfddce46d 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -600,15 +600,19 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): - if not frame.result: - return + if frame.result: + if not isinstance(frame.result, str): + return - if not isinstance(frame.result, str): - return + response = {"response": frame.result} - response = {"response": frame.result} - - await self._update_function_call_result(frame.function_name, frame.tool_call_id, response) + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, response + ) + else: + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "COMPLETED" + ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): await self._update_function_call_result( diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 8318739de..b5f5d9d54 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -584,12 +584,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): - if not frame.result: - return - - result = json.dumps(frame.result) - - await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + if frame.result: + result = json.dumps(frame.result) + await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) + else: + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "COMPLETED" + ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): await self._update_function_call_result( From f298febacf1e0639c4716710b0e404e4e85e1af3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 08:17:55 -0400 Subject: [PATCH 355/427] Add FalSTTService --- CHANGELOG.md | 4 + README.md | 2 +- .../foundational/07w-interruptible-fal.py | 110 +++++++++ src/pipecat/services/fal.py | 227 +++++++++++++++++- src/pipecat/transcriptions/language.py | 61 ++++- 5 files changed, 400 insertions(+), 4 deletions(-) create mode 100644 examples/foundational/07w-interruptible-fal.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 247c3280d..6d55b97c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 handler will be triggered if the idle timeout is reached (whether the pipeline task is cancelled or not). +- Added `FalSTTService`, which provides STT for Fal's Wizper API. + - Added a `reconnect_on_error` parameter to websocket-based TTS services as well as a `on_connection_error` event handler. The `reconnect_on_error` indicates whether the TTS service should reconnect on error. The `on_connection_error` @@ -216,6 +218,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other +- Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`. + - Added a new example `examples/foundational/36-user-email-gathering.py` to show how to gather user emails. The example uses's Cartesia's `` tags and Rime `spell()` function to spell out the emails for confirmation. diff --git a/README.md b/README.md index 9522ce175..f28005618 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ pip install "pipecat-ai[option,...]" | Category | Services | Install Command Example | | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Fal Wizper](https://docs.pipecat.ai/server/services/stt/fal), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Google](https://docs.pipecat.ai/server/services/stt/google), [Groq (Whisper)](https://docs.pipecat.ai/server/services/stt/groq), [OpenAI (Whisper)](https://docs.pipecat.ai/server/services/stt/openai), [Parakeet (NVIDIA)](https://docs.pipecat.ai/server/services/stt/parakeet), [Ultravox](https://docs.pipecat.ai/server/services/stt/ultravox), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | | LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [DeepSeek](https://docs.pipecat.ai/server/services/llm/deepseek), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [OpenRouter](https://docs.pipecat.ai/server/services/llm/openrouter), [Perplexity](https://docs.pipecat.ai/server/services/llm/perplexity), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | | Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [FastPitch (NVIDIA)](https://docs.pipecat.ai/server/services/tts/fastpitch), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [Neuphonic](https://docs.pipecat.ai/server/services/tts/neuphonic), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | | Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[google]"` | diff --git a/examples/foundational/07w-interruptible-fal.py b/examples/foundational/07w-interruptible-fal.py new file mode 100644 index 000000000..526602166 --- /dev/null +++ b/examples/foundational/07w-interruptible-fal.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.fal import FalSTTService +from pipecat.services.gladia import GladiaSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = FalSTTService( + api_key=os.getenv("FAL_KEY"), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + # Register an event handler to exit the application when the user leaves. + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/services/fal.py b/src/pipecat/services/fal.py index 7173861ab..cb39da75f 100644 --- a/src/pipecat/services/fal.py +++ b/src/pipecat/services/fal.py @@ -7,6 +7,7 @@ import asyncio import io import os +import wave from typing import AsyncGenerator, Dict, Optional, Union import aiohttp @@ -14,8 +15,10 @@ from loguru import logger from PIL import Image from pydantic import BaseModel -from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame -from pipecat.services.ai_services import ImageGenService +from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame, URLImageRawFrame +from pipecat.services.ai_services import ImageGenService, SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 try: import fal_client @@ -27,6 +30,120 @@ except ModuleNotFoundError as e: raise Exception(f"Missing module: {e}") +def language_to_fal_language(language: Language) -> Optional[str]: + """Language support for Fal's Wizper API.""" + BASE_LANGUAGES = { + Language.AF: "af", + Language.AM: "am", + Language.AR: "ar", + Language.AS: "as", + Language.AZ: "az", + Language.BA: "ba", + Language.BE: "be", + Language.BG: "bg", + Language.BN: "bn", + Language.BO: "bo", + Language.BR: "br", + Language.BS: "bs", + Language.CA: "ca", + Language.CS: "cs", + Language.CY: "cy", + Language.DA: "da", + Language.DE: "de", + Language.EL: "el", + Language.EN: "en", + Language.ES: "es", + Language.ET: "et", + Language.EU: "eu", + Language.FA: "fa", + Language.FI: "fi", + Language.FO: "fo", + Language.FR: "fr", + Language.GL: "gl", + Language.GU: "gu", + Language.HA: "ha", + Language.HE: "he", + Language.HI: "hi", + Language.HR: "hr", + Language.HT: "ht", + Language.HU: "hu", + Language.HY: "hy", + Language.ID: "id", + Language.IS: "is", + Language.IT: "it", + Language.JA: "ja", + Language.JW: "jw", + Language.KA: "ka", + Language.KK: "kk", + Language.KM: "km", + Language.KN: "kn", + Language.KO: "ko", + Language.LA: "la", + Language.LB: "lb", + Language.LN: "ln", + Language.LO: "lo", + Language.LT: "lt", + Language.LV: "lv", + Language.MG: "mg", + Language.MI: "mi", + Language.MK: "mk", + Language.ML: "ml", + Language.MN: "mn", + Language.MR: "mr", + Language.MS: "ms", + Language.MT: "mt", + Language.MY: "my", + Language.NE: "ne", + Language.NL: "nl", + Language.NN: "nn", + Language.NO: "no", + Language.OC: "oc", + Language.PA: "pa", + Language.PL: "pl", + Language.PS: "ps", + Language.PT: "pt", + Language.RO: "ro", + Language.RU: "ru", + Language.SA: "sa", + Language.SD: "sd", + Language.SI: "si", + Language.SK: "sk", + Language.SL: "sl", + Language.SN: "sn", + Language.SO: "so", + Language.SQ: "sq", + Language.SR: "sr", + Language.SU: "su", + Language.SV: "sv", + Language.SW: "sw", + Language.TA: "ta", + Language.TE: "te", + Language.TG: "tg", + Language.TH: "th", + Language.TK: "tk", + Language.TL: "tl", + Language.TR: "tr", + Language.TT: "tt", + Language.UK: "uk", + Language.UR: "ur", + Language.UZ: "uz", + Language.VI: "vi", + Language.YI: "yi", + Language.YO: "yo", + Language.ZH: "zh", + } + + result = BASE_LANGUAGES.get(language) + + # If not found in base languages, try to find the base language from a variant + if not result: + lang_str = str(language.value) + base_code = lang_str.split("-")[0].lower() + result = base_code if base_code in BASE_LANGUAGES.values() else None + + return result + + class FalImageGenService(ImageGenService): class InputParams(BaseModel): seed: Optional[int] = None @@ -84,3 +201,109 @@ class FalImageGenService(ImageGenService): frame = URLImageRawFrame(url=image_url, image=image_bytes, size=size, format=format) yield frame + + +class FalSTTService(SegmentedSTTService): + """Speech-to-text service using Fal's Wizper API. + + This service uses Fal's Wizper API to perform speech-to-text transcription on audio + segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. + + Args: + api_key: Fal API key. If not provided, will check FAL_KEY environment variable. + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the Wizper API. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ + + class InputParams(BaseModel): + """Configuration parameters for Fal's Wizper API. + + Attributes: + language: Language of the audio input. Defaults to English. + task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'. + chunk_level: Level of chunking ('segment'). Defaults to 'segment'. + version: Version of Wizper model to use. Defaults to '3'. + """ + + language: Optional[Language] = Language.EN + task: str = "transcribe" + chunk_level: str = "segment" + version: str = "3" + + def __init__( + self, + *, + api_key: Optional[str] = None, + sample_rate: Optional[int] = None, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__( + sample_rate=sample_rate, + **kwargs, + ) + + if api_key: + os.environ["FAL_KEY"] = api_key + elif "FAL_KEY" not in os.environ: + raise ValueError( + "FAL_KEY must be provided either through api_key parameter or environment variable" + ) + + self._fal_client = fal_client.AsyncClient(key=api_key or os.getenv("FAL_KEY")) + self._settings = { + "task": params.task, + "language": self.language_to_service_language(params.language) + if params.language + else "en", + "chunk_level": params.chunk_level, + "version": params.version, + } + + def can_generate_metrics(self) -> bool: + return True + + def language_to_service_language(self, language: Language) -> Optional[str]: + return language_to_fal_language(language) + + async def set_language(self, language: Language): + logger.info(f"Switching STT language to: [{language}]") + self._settings["language"] = self.language_to_service_language(language) + + async def set_model(self, model: str): + await super().set_model(model) + logger.info(f"Switching STT model to: [{model}]") + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribes an audio segment using Fal's Wizper API. + + Args: + audio: Raw audio bytes in WAV format (already converted by base class). + + Yields: + Frame: TranscriptionFrame containing the transcribed text. + + Note: + The audio is already in WAV format from the SegmentedSTTService. + Only non-empty transcriptions are yielded. + """ + try: + # Send to Fal directly (audio is already in WAV format from base class) + data_uri = fal_client.encode(audio, "audio/x-wav") + response = await self._fal_client.run( + "fal-ai/wizper", + arguments={"audio_url": data_uri, **self._settings}, + ) + + if response and "text" in response: + text = response["text"].strip() + if text: # Only yield non-empty text + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame( + text, "", time_now_iso8601(), Language(self._settings["language"]) + ) + + except Exception as e: + logger.error(f"Fal Wizper error: {e}") + yield ErrorFrame(f"Fal Wizper error: {str(e)}") diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index b8b9fafe9..75f714a72 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -54,6 +54,9 @@ class Language(StrEnum): AZ = "az" AZ_AZ = "az-AZ" + # Bashkir + BA = "ba" + # Belarusian BE = "be" @@ -66,6 +69,12 @@ class Language(StrEnum): BN_BD = "bn-BD" BN_IN = "bn-IN" + # Tibetan + BO = "bo" + + # Breton + BR = "br" + # Bosnian BS = "bs" BS_BA = "bs-BA" @@ -159,6 +168,9 @@ class Language(StrEnum): FIL = "fil" FIL_PH = "fil-PH" + # Faroese + FO = "fo" + # French FR = "fr" FR_BE = "fr-BE" @@ -178,6 +190,9 @@ class Language(StrEnum): GU = "gu" GU_IN = "gu-IN" + # Hausa + HA = "ha" + # Hebrew HE = "he" HE_IL = "he-IL" @@ -190,6 +205,9 @@ class Language(StrEnum): HR = "hr" HR_HR = "hr-HR" + # Haitian Creole + HT = "ht" + # Hungarian HU = "hu" HU_HU = "hu-HU" @@ -224,6 +242,7 @@ class Language(StrEnum): # Javanese JV = "jv" JV_ID = "jv-ID" + JW = "jw" # Fal requires for Javanese # Georgian KA = "ka" @@ -245,6 +264,15 @@ class Language(StrEnum): KO = "ko" KO_KR = "ko-KR" + # Latin + LA = "la" + + # Luxembourgish + LB = "lb" + + # Lingala + LN = "ln" + # Lao LO = "lo" LO_LA = "lo-LA" @@ -257,6 +285,9 @@ class Language(StrEnum): LV = "lv" LV_LV = "lv-LV" + # Malagasy + MG = "mg" + # Macedonian MK = "mk" MK_MK = "mk-MK" @@ -289,9 +320,10 @@ class Language(StrEnum): MY_MM = "my-MM" # Norwegian - NB = "nb" + NB = "nb" # Norwegian Bokmål NB_NO = "nb-NO" NO = "no" + NN = "nn" # Norwegian Nynorsk # Nepali NE = "ne" @@ -302,6 +334,9 @@ class Language(StrEnum): NL_BE = "nl-BE" NL_NL = "nl-NL" + # Occitan + OC = "oc" + # Odia OR = "or" OR_IN = "or-IN" @@ -331,6 +366,12 @@ class Language(StrEnum): RU = "ru" RU_RU = "ru-RU" + # Sanskrit + SA = "sa" + + # Sindhi + SD = "sd" + # Sinhala SI = "si" SI_LK = "si-LK" @@ -343,6 +384,9 @@ class Language(StrEnum): SL = "sl" SL_SI = "sl-SI" + # Shona + SN = "sn" + # Somali SO = "so" SO_SO = "so-SO" @@ -384,14 +428,23 @@ class Language(StrEnum): TE = "te" TE_IN = "te-IN" + # Tajik + TG = "tg" + # Thai TH = "th" TH_TH = "th-TH" + # Turkmen + TK = "tk" + # Turkish TR = "tr" TR_TR = "tr-TR" + # Tatar + TT = "tt" + # Ukrainian UK = "uk" UK_UA = "uk-UA" @@ -413,6 +466,12 @@ class Language(StrEnum): WUU = "wuu" WUU_CN = "wuu-CN" + # Yiddish + YI = "yi" + + # Yoruba + YO = "yo" + # Yue Chinese YUE = "yue" YUE_CN = "yue-CN" From e4bb4aacb4c2438d13f5422cf54860fa264ca6cb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 12:46:00 -0400 Subject: [PATCH 356/427] Example: Rename 07 ultravox example --- ...7u-interruptible-ultravox.py => 07v-interruptible-ultravox.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/foundational/{07u-interruptible-ultravox.py => 07v-interruptible-ultravox.py} (100%) diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07v-interruptible-ultravox.py similarity index 100% rename from examples/foundational/07u-interruptible-ultravox.py rename to examples/foundational/07v-interruptible-ultravox.py From 2b4debec1116883d520f038b611329b3fb80b9c7 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:23:30 -0700 Subject: [PATCH 357/427] add support for conversation.item.input_audio_transcription.delta --- src/pipecat/services/openai_realtime_beta/events.py | 8 ++++++++ src/pipecat/services/openai_realtime_beta/openai.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 17ce0a6d4..caa7e964c 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -193,6 +193,13 @@ class ConversationItemCreated(ServerEvent): item: ConversationItem +class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: str + content_index: int + delta: str + + class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): type: Literal["conversation.item.input_audio_transcription.completed"] item_id: str @@ -400,6 +407,7 @@ _server_event_types = { "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, "conversation.item.created": ConversationItemCreated, + "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, "conversation.item.truncated": ConversationItemTruncated, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 321c66826..6f0edd67c 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -29,6 +29,7 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, + InterimTranscriptionFrame, InputAudioRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -354,6 +355,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_evt_audio_done(evt) elif evt.type == "conversation.item.created": await self._handle_evt_conversation_item_created(evt) + elif evt.type == "conversation.item.input_audio_transcription.delta": + await self._handle_evt_input_audio_transcription_delta(evt) elif evt.type == "conversation.item.input_audio_transcription.completed": await self.handle_evt_input_audio_transcription_completed(evt) elif evt.type == "response.done": @@ -425,6 +428,13 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._current_assistant_response = evt.item await self.push_frame(LLMFullResponseStartFrame()) + async def _handle_evt_input_audio_transcription_delta(self, evt): + if self._send_transcription_frames: + await self.push_frame( + # no way to get a language code? + InterimTranscriptionFrame(evt.delta, "", time_now_iso8601()) + ) + async def handle_evt_input_audio_transcription_completed(self, evt): if self._send_transcription_frames: await self.push_frame( From bfdf52bd69428faea7cb8eb6077b69dd5b8b7be0 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:27:17 -0700 Subject: [PATCH 358/427] change examples/foundational/19-openai-realtime-beta.py to use the new preview model --- examples/foundational/19-openai-realtime-beta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..8027b0669 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -114,6 +114,7 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o-realtime-preview-latest", session_properties=session_properties, start_audio_paused=False, ) From 4449e9a25bcf79b7659afec00937aebfcd5a06dd Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 16 Mar 2025 18:40:19 -0700 Subject: [PATCH 359/427] add response.done status=failed error --- examples/foundational/19-openai-realtime-beta.py | 1 - src/pipecat/services/openai_realtime_beta/openai.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8027b0669..8796e0141 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -114,7 +114,6 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), - model="gpt-4o-realtime-preview-latest", session_properties=session_properties, start_audio_paused=False, ) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 6f0edd67c..f4c99a73f 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -87,7 +87,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): self, *, api_key: str, - model: str = "gpt-4o-realtime-preview-2024-12-17", + # model: str = "gpt-4o-realtime-preview-2024-12-17", + model: str = "gpt-4o-realtime-preview-latest", base_url: str = "wss://api.openai.com/v1/realtime", session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, @@ -465,6 +466,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self.stop_processing_metrics() await self.push_frame(LLMFullResponseEndFrame()) self._current_assistant_response = None + # error handling + if evt.response.status == "failed": + await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)) + return # response content pair = self._user_and_response_message_tuple if pair: From 16accafa6debc58db7cbd7c5fff40360eda2ca10 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 10:47:00 -0400 Subject: [PATCH 360/427] formatting fix --- src/pipecat/services/openai_realtime_beta/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index f4c99a73f..879d96e6f 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -468,7 +468,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._current_assistant_response = None # error handling if evt.response.status == "failed": - await self.push_error(ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True)) + await self.push_error( + ErrorFrame(error=evt.response.status_details["error"]["message"], fatal=True) + ) return # response content pair = self._user_and_response_message_tuple From 214c8f79eb2b5f372df0b118c1f389bf5d7f433a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 10:50:04 -0400 Subject: [PATCH 361/427] linter fix --- src/pipecat/services/openai_realtime_beta/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 879d96e6f..68dde4d32 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -29,8 +29,8 @@ from pipecat.frames.frames import ( EndFrame, ErrorFrame, Frame, - InterimTranscriptionFrame, InputAudioRawFrame, + InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesAppendFrame, From e80bfe22de734d9449b53fd6d0224159fd8cc6ab Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 12:15:14 -0400 Subject: [PATCH 362/427] Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService --- examples/foundational/19-openai-realtime-beta.py | 4 ++-- examples/foundational/19a-azure-realtime-beta.py | 4 ++-- .../20b-persistent-context-openai-realtime.py | 4 ++-- .../services/openai_realtime_beta/__init__.py | 2 +- src/pipecat/services/openai_realtime_beta/events.py | 13 +++++++++++-- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..c39c0f570 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( - InputAudioTranscription, + InputAudioTranscriptionModels, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -89,7 +89,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), + input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 14d034836..12847ee33 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, - InputAudioTranscription, + InputAudioTranscriptionModels, SessionProperties, TurnDetection, ) @@ -90,7 +90,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), + input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default # turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index ef82bd567..26a6448e9 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.openai_realtime_beta import ( - InputAudioTranscription, + InputAudioTranscriptionModels, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -186,7 +186,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), + input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index 52b00f6c8..fdda648c4 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,3 +1,3 @@ from .azure import AzureRealtimeBetaLLMService -from .events import InputAudioTranscription, SessionProperties, TurnDetection +from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection from .openai import OpenAIRealtimeBetaLLMService diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index caa7e964c..9cc946807 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -16,8 +16,17 @@ from pydantic import BaseModel, Field # -class InputAudioTranscription(BaseModel): - model: Optional[str] = "whisper-1" +class InputAudioTranscriptionModels: + class Whisper1(BaseModel): + model: Optional[Literal["whisper-1"]] = "whisper-1" + + class GPT4o(BaseModel): + model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest" + language: Optional[str] = None + prompt: Optional[str] = None + + +InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o] class TurnDetection(BaseModel): From be2cf6d556aacf8b2233f7e83907657b9414d1ae Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 12:28:11 -0400 Subject: [PATCH 363/427] formatting fix --- src/pipecat/services/openai_realtime_beta/events.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 9cc946807..adfc34133 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -26,7 +26,9 @@ class InputAudioTranscriptionModels: prompt: Optional[str] = None -InputAudioTranscription = Union[InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o] +InputAudioTranscription = Union[ + InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o +] class TurnDetection(BaseModel): From fe5fc302114586b6eeeaed55fd59c2711319345a Mon Sep 17 00:00:00 2001 From: kompfner Date: Mon, 17 Mar 2025 12:36:31 -0400 Subject: [PATCH 364/427] Revert "Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService" --- examples/foundational/19-openai-realtime-beta.py | 4 ++-- examples/foundational/19a-azure-realtime-beta.py | 4 ++-- .../20b-persistent-context-openai-realtime.py | 4 ++-- .../services/openai_realtime_beta/__init__.py | 2 +- .../services/openai_realtime_beta/events.py | 15 ++------------- 5 files changed, 9 insertions(+), 20 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index c39c0f570..8796e0141 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -21,7 +21,7 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( - InputAudioTranscriptionModels, + InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -89,7 +89,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), + input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 12847ee33..14d034836 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -23,7 +23,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, - InputAudioTranscriptionModels, + InputAudioTranscription, SessionProperties, TurnDetection, ) @@ -90,7 +90,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), + input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default # turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 26a6448e9..ef82bd567 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,7 +25,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.openai_realtime_beta import ( - InputAudioTranscriptionModels, + InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, TurnDetection, @@ -186,7 +186,7 @@ async def main(): ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscriptionModels.Whisper1(), + input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default turn_detection=TurnDetection(silence_duration_ms=1000), diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index fdda648c4..52b00f6c8 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,3 +1,3 @@ from .azure import AzureRealtimeBetaLLMService -from .events import InputAudioTranscriptionModels, SessionProperties, TurnDetection +from .events import InputAudioTranscription, SessionProperties, TurnDetection from .openai import OpenAIRealtimeBetaLLMService diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index adfc34133..caa7e964c 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -16,19 +16,8 @@ from pydantic import BaseModel, Field # -class InputAudioTranscriptionModels: - class Whisper1(BaseModel): - model: Optional[Literal["whisper-1"]] = "whisper-1" - - class GPT4o(BaseModel): - model: Optional[Literal["gpt-4o-transcribe-latest"]] = "gpt-4o-transcribe-latest" - language: Optional[str] = None - prompt: Optional[str] = None - - -InputAudioTranscription = Union[ - InputAudioTranscriptionModels.Whisper1, InputAudioTranscriptionModels.GPT4o -] +class InputAudioTranscription(BaseModel): + model: Optional[str] = "whisper-1" class TurnDetection(BaseModel): From d009b804383a5dfc70bc3c6fcd655c6d5cde42cd Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 12:54:32 -0400 Subject: [PATCH 365/427] Add new GPT-4o transcription option to OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/events.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index caa7e964c..fcc330a6e 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,10 +14,25 @@ from pydantic import BaseModel, Field # # session properties # +InputAudioTranscriptionModelArg = Optional[Literal["whisper-1", "gpt-4o-transcribe-latest"]] class InputAudioTranscription(BaseModel): - model: Optional[str] = "whisper-1" + model: InputAudioTranscriptionModelArg + language: Optional[str] + prompt: Optional[str] + + def __init__( + self, + model: InputAudioTranscriptionModelArg = "whisper-1", + language: Optional[str] = None, + prompt: Optional[str] = None, + ): + super().__init__(model=model, language=language, prompt=prompt) + if self.model != "gpt-4o-transcribe-latest" and (self.language or self.prompt): + raise ValueError( + "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe-latest'" + ) class TurnDetection(BaseModel): From 1a20d9bed7c57d993c6a08d0b3ec747839190584 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 13:02:55 -0400 Subject: [PATCH 366/427] Add new input_audio_noise_reduction option to OpenAIRealtimeBetaLLMService --- src/pipecat/services/openai_realtime_beta/__init__.py | 7 ++++++- src/pipecat/services/openai_realtime_beta/events.py | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index 52b00f6c8..a4c102609 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,3 +1,8 @@ from .azure import AzureRealtimeBetaLLMService -from .events import InputAudioTranscription, SessionProperties, TurnDetection +from .events import ( + InputAudioTranscription, + InputAudioNoiseReduction, + SessionProperties, + TurnDetection, +) from .openai import OpenAIRealtimeBetaLLMService diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index fcc330a6e..26fbc0578 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -42,6 +42,10 @@ class TurnDetection(BaseModel): silence_duration_ms: Optional[int] = 800 +class InputAudioNoiseReduction(BaseModel): + type: Optional[Literal["near_field", "far_field"]] + + class SessionProperties(BaseModel): modalities: Optional[List[Literal["text", "audio"]]] = None instructions: Optional[str] = None @@ -49,6 +53,7 @@ class SessionProperties(BaseModel): input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None input_audio_transcription: Optional[InputAudioTranscription] = None + input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None # set turn_detection to False to disable turn detection turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None) tools: Optional[List[Dict]] = None From e91610c69eb1439679cdeb369b565f01276633c5 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 13:18:52 -0400 Subject: [PATCH 367/427] linter fix --- src/pipecat/services/openai_realtime_beta/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index a4c102609..d54640919 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -1,7 +1,7 @@ from .azure import AzureRealtimeBetaLLMService from .events import ( - InputAudioTranscription, InputAudioNoiseReduction, + InputAudioTranscription, SessionProperties, TurnDetection, ) From 1075c25055873344f63d7b9f89669aa3c069b1ab Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 15:28:31 -0400 Subject: [PATCH 368/427] Add new semantic turn detection option to OpenAIRealtimeBetaLLMService --- src/pipecat/services/openai_realtime_beta/__init__.py | 1 + src/pipecat/services/openai_realtime_beta/events.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py index d54640919..595105d7f 100644 --- a/src/pipecat/services/openai_realtime_beta/__init__.py +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -2,6 +2,7 @@ from .azure import AzureRealtimeBetaLLMService from .events import ( InputAudioNoiseReduction, InputAudioTranscription, + SemanticTurnDetection, SessionProperties, TurnDetection, ) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 26fbc0578..c8a7c383f 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -42,6 +42,13 @@ class TurnDetection(BaseModel): silence_duration_ms: Optional[int] = 800 +class SemanticTurnDetection(BaseModel): + type: Optional[Literal["semantic_vad"]] = "semantic_vad" + eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None + create_response: Optional[bool] = None + interrupt_response: Optional[bool] = None + + class InputAudioNoiseReduction(BaseModel): type: Optional[Literal["near_field", "far_field"]] @@ -55,7 +62,9 @@ class SessionProperties(BaseModel): input_audio_transcription: Optional[InputAudioTranscription] = None input_audio_noise_reduction: Optional[InputAudioNoiseReduction] = None # set turn_detection to False to disable turn detection - turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None) + turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = Field( + default=None + ) tools: Optional[List[Dict]] = None tool_choice: Optional[Literal["auto", "none", "required"]] = None temperature: Optional[float] = None From 9840abd85b828043bed7dfb4dab2c23dee19b546 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 14:51:19 -0400 Subject: [PATCH 369/427] Make it so you specifying `model=None` when creating a `InputAudioTranscription` results in a validation error --- src/pipecat/services/openai_realtime_beta/events.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index c8a7c383f..0f42a30b0 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,17 +14,17 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModelArg = Optional[Literal["whisper-1", "gpt-4o-transcribe-latest"]] +InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe-latest"] class InputAudioTranscription(BaseModel): - model: InputAudioTranscriptionModelArg + model: InputAudioTranscriptionModel language: Optional[str] prompt: Optional[str] def __init__( self, - model: InputAudioTranscriptionModelArg = "whisper-1", + model: Optional[InputAudioTranscriptionModel] = "whisper-1", language: Optional[str] = None, prompt: Optional[str] = None, ): From 39ca607bbbe72a481c664c6d90055c23454ec9fc Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 16:52:53 -0400 Subject: [PATCH 370/427] Add `on_conversation_item_created` and `on_conversation_item_updated` events to OpenAIRealtimeBetaLLMService. The hope is that this will expose to the user conversation item ids at relevant times for them to use with the new `conversation.item.retrieve` introspection message. --- src/pipecat/services/openai_realtime_beta/openai.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 68dde4d32..4a3a14cf8 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -117,6 +117,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._messages_added_manually = {} self._user_and_response_message_tuple = None + self._register_event_handler("on_conversation_item_created") + self._register_event_handler("on_conversation_item_updated") + def can_generate_metrics(self) -> bool: return True @@ -413,6 +416,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): # receive a BotStoppedSpeakingFrame from the output transport. async def _handle_evt_conversation_item_created(self, evt): + await self._call_event_handler("on_conversation_item_created", evt.item.id, evt.item) + # This will get sent from the server every time a new "message" is added # to the server's conversation state, whether we create it via the API # or the server creates it from LLM output. @@ -437,6 +442,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) async def handle_evt_input_audio_transcription_completed(self, evt): + await self._call_event_handler("on_conversation_item_updated", evt.item_id, None) + if self._send_transcription_frames: await self.push_frame( # no way to get a language code? @@ -473,6 +480,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) return # response content + for item in evt.response.output: + await self._call_event_handler("on_conversation_item_updated", item.id, item) pair = self._user_and_response_message_tuple if pair: user, assistant = pair From f693a3c70f442ac410d15bed39a0fa9be0f3578b Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Mon, 17 Mar 2025 21:54:56 -0400 Subject: [PATCH 371/427] Add `retrieve_conversation_item()` method to OpenAIRealtimeBetaLLMService, using the new `conversation.item.retrieve` introspection message. --- .../services/openai_realtime_beta/events.py | 11 +++++++++++ .../services/openai_realtime_beta/openai.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 0f42a30b0..88638b5fd 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -179,6 +179,11 @@ class ConversationItemDeleteEvent(ClientEvent): item_id: str +class ConversationItemRetrieveEvent(ClientEvent): + type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" + item_id: str + + class ResponseCreateEvent(ClientEvent): type: Literal["response.create"] = "response.create" response: Optional[ResponseProperties] = None @@ -255,6 +260,11 @@ class ConversationItemDeleted(ServerEvent): item_id: str +class ConversationItemRetrieved(ServerEvent): + type: Literal["conversation.item.retrieved"] + item: ConversationItem + + class ResponseCreated(ServerEvent): type: Literal["response.created"] response: "Response" @@ -441,6 +451,7 @@ _server_event_types = { "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, "conversation.item.truncated": ConversationItemTruncated, "conversation.item.deleted": ConversationItemDeleted, + "conversation.item.retrieved": ConversationItemRetrieved, "response.created": ResponseCreated, "response.done": ResponseDone, "response.output_item.added": ResponseOutputItemAdded, diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 4a3a14cf8..922f6f2ff 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -119,6 +119,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._register_event_handler("on_conversation_item_created") self._register_event_handler("on_conversation_item_updated") + self._retrieve_conversation_item_futures = {} def can_generate_metrics(self) -> bool: return True @@ -126,6 +127,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): def set_audio_input_paused(self, paused: bool): self._audio_input_paused = paused + async def retrieve_conversation_item(self, item_id: str): + future = self.get_event_loop().create_future() + self._retrieve_conversation_item_futures[item_id] = future + await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) + return await future + # # standard AIService frame handling # @@ -363,6 +370,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_evt_input_audio_transcription_delta(evt) elif evt.type == "conversation.item.input_audio_transcription.completed": await self.handle_evt_input_audio_transcription_completed(evt) + elif evt.type == "conversation.item.retrieved": + await self._handle_conversation_item_retrieved(evt) elif evt.type == "response.done": await self._handle_evt_response_done(evt) elif evt.type == "input_audio_buffer.speech_started": @@ -461,6 +470,14 @@ class OpenAIRealtimeBetaLLMService(LLMService): # User message without preceding conversation.item.created. Bug? logger.warning(f"Transcript for unknown user message: {evt}") + async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): + future = self._retrieve_conversation_item_futures.get(evt.item.id) + if future: + # print(f"[pk] setting result: {evt.item}") + future.set_result(evt.item) + # TODO: handle error + # TODO: what happens if we try to receive bogus item id? + async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events # usage metrics From 31317ce77d51abd060556b67a02a44c1fa5870ae Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 09:42:20 -0400 Subject: [PATCH 372/427] Add error handling to the `retrieve_conversation_item()` method of the OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/openai.py | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 922f6f2ff..52a03ba94 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -23,6 +23,8 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") +import re + from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -381,9 +383,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): elif evt.type == "response.audio_transcript.delta": await self._handle_evt_audio_transcript_delta(evt) elif evt.type == "error": - await self._handle_evt_error(evt) - # errors are fatal, so exit the receive loop - return + if not await self._maybe_handle_evt_retrieve_conversation_item_error(evt): + await self._handle_evt_error(evt) + # errors are fatal, so exit the receive loop + return async def _handle_evt_session_created(self, evt): # session.created is received right after connecting. Send a message @@ -471,12 +474,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): logger.warning(f"Transcript for unknown user message: {evt}") async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): - future = self._retrieve_conversation_item_futures.get(evt.item.id) + future = self._retrieve_conversation_item_futures.pop(evt.item.id, None) if future: # print(f"[pk] setting result: {evt.item}") future.set_result(evt.item) - # TODO: handle error - # TODO: what happens if we try to receive bogus item id? async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events @@ -530,6 +531,24 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self.push_frame(StopInterruptionFrame()) await self.push_frame(UserStoppedSpeakingFrame()) + async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent): + """If the given error event is an error retrieving a conversation item: + - set an exception on the future that retrieve_conversation_item() is waiting on + - return true + Otherwise: + - return false + """ + match = re.match( + r"^Error retrieving item: the item with id '(.*)' does not exist\.$", evt.error.message + ) + if match: + item_id = match.group(1) + future = self._retrieve_conversation_item_futures.pop(item_id, None) + if future: + future.set_exception(Exception(evt.error.message)) + return True + return False + async def _handle_evt_error(self, evt): # Errors are fatal to this connection. Send an ErrorFrame. await self.push_error(ErrorFrame(error=f"Error: {evt}", fatal=True)) From 7b594093ddb7b586c0c3c521f082bfee2a362a64 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 10:05:41 -0400 Subject: [PATCH 373/427] Handle the possibility of multiple concurrent calls to `retrieve_conversation_item()` in the OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/openai.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 52a03ba94..4b993c28a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -131,8 +131,14 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def retrieve_conversation_item(self, item_id: str): future = self.get_event_loop().create_future() - self._retrieve_conversation_item_futures[item_id] = future - await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) + retrieval_in_progress = False + if not self._retrieve_conversation_item_futures.get(item_id): + self._retrieve_conversation_item_futures[item_id] = [] + else: + retrieval_in_progress = True + self._retrieve_conversation_item_futures[item_id].append(future) + if not retrieval_in_progress: + await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) return await future # @@ -474,10 +480,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): logger.warning(f"Transcript for unknown user message: {evt}") async def _handle_conversation_item_retrieved(self, evt: events.ConversationItemRetrieved): - future = self._retrieve_conversation_item_futures.pop(evt.item.id, None) - if future: - # print(f"[pk] setting result: {evt.item}") - future.set_result(evt.item) + futures = self._retrieve_conversation_item_futures.pop(evt.item.id, None) + if futures: + for future in futures: + future.set_result(evt.item) async def _handle_evt_response_done(self, evt): # todo: figure out whether there's anything we need to do for "cancelled" events @@ -543,9 +549,10 @@ class OpenAIRealtimeBetaLLMService(LLMService): ) if match: item_id = match.group(1) - future = self._retrieve_conversation_item_futures.pop(item_id, None) - if future: - future.set_exception(Exception(evt.error.message)) + futures = self._retrieve_conversation_item_futures.pop(item_id, None) + if futures: + for future in futures: + future.set_exception(Exception(evt.error.message)) return True return False From e707efbffa84cf77dab967fa3c58d5ac2fa68a6f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 12:30:19 -0400 Subject: [PATCH 374/427] Update changelog with slate of recent updates to OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d55b97c3..42fad1c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,8 +128,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. +- Added support in `OpenAIRealtimeBetaLLMService` for the + `conversation.item.input_audio_transcription.delta` server message. + +- Added error handling in `OpenAIRealtimeBetaLLMService` for the + `response.done` server message reporting a failure. + +- Added support in `OpenAIRealtimeBetaLLMService` for the new + `gpt-4o-transcribe-latest` input audio transcription model. + +- Added support in `OpenAIRealtimeBetaLLMService` for the new + `input_audio_noise_reduction` session property. + + ```python + session_properties = SessionProperties( + # ... + input_audio_noise_reduction=InputAudioNoiseReduction( + type="near_field" # also supported: "far_field" + ) + # ... + ) + ``` + +- Added support in `OpenAIRealtimeBetaLLMService` for the new + `semantic_vad` `turn_detection` session property, which is a more + sophisticated model for detecting when the user has stopped speaking. + +- Added `on_conversation_item_created` and `on_conversation_item_updated` + events to `OpenAIRealtimeBetaLLMService`. + + ```python + @llm.event_handler("on_conversation_item_created") + async def on_conversation_item_created(llm, item_id, item): + # ... + + @llm.event_handler("on_conversation_item_updated") + async def on_conversation_item_updated(llm, item_id, item): + # `item` may not always be available here + # ... + ``` + +- Added `retrieve_conversation_item(item_id)` to `OpenAIRealtimeBetaLLMService`. + + ```python + item = await llm.retrieve_conversation_item(item_id) + ``` + ### Changed +- Updated the default model for `CartesiaTTSService` and + `CartesiaHttpTTSService` to `sonic-2`. + +- Updated the default model for `OpenAIRealtimeBetaLLMService` to + `gpt-4o-realtime-preview-latest`. + - Function calls are now executed in tasks. This means that the pipeline will not be blocked while the function call is being executed. @@ -216,6 +268,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. +- Fixed `OpenAIRealtimeBetaLLMService` by adding support for the + `conversation.item.input_audio_transcription.delta` server message, which was + added server-side at some point and not handled client-side. + ### Other - Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`. From 3dd4ef72303c45c439eebcfd9a4e7c1d1a955047 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 18 Mar 2025 15:52:11 -0400 Subject: [PATCH 375/427] Tweak changelog entries describing slate of recent updates to OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 73 ++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42fad1c44..fe531bf7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,51 +128,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Gemini models. Added foundational example `14p-function-calling-gemini-vertex-ai.py`. -- Added support in `OpenAIRealtimeBetaLLMService` for the - `conversation.item.input_audio_transcription.delta` server message. +- Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features: -- Added error handling in `OpenAIRealtimeBetaLLMService` for the - `response.done` server message reporting a failure. + - The `'gpt-4o-transcribe-latest'` input audio transcription model. + - The `input_audio_noise_reduction` session property. -- Added support in `OpenAIRealtimeBetaLLMService` for the new - `gpt-4o-transcribe-latest` input audio transcription model. - -- Added support in `OpenAIRealtimeBetaLLMService` for the new - `input_audio_noise_reduction` session property. - - ```python - session_properties = SessionProperties( - # ... - input_audio_noise_reduction=InputAudioNoiseReduction( - type="near_field" # also supported: "far_field" + ```python + session_properties = SessionProperties( + # ... + input_audio_noise_reduction=InputAudioNoiseReduction( + type="near_field" # also supported: "far_field" + ) + # ... ) - # ... - ) - ``` + ``` -- Added support in `OpenAIRealtimeBetaLLMService` for the new - `semantic_vad` `turn_detection` session property, which is a more - sophisticated model for detecting when the user has stopped speaking. + - The `'semantic_vad'` `turn_detection` session property value, a more + sophisticated model for detecting when the user has stopped speaking. + - `on_conversation_item_created` and `on_conversation_item_updated` + events to `OpenAIRealtimeBetaLLMService`. -- Added `on_conversation_item_created` and `on_conversation_item_updated` - events to `OpenAIRealtimeBetaLLMService`. + ```python + @llm.event_handler("on_conversation_item_created") + async def on_conversation_item_created(llm, item_id, item): + # ... - ```python - @llm.event_handler("on_conversation_item_created") - async def on_conversation_item_created(llm, item_id, item): - # ... + @llm.event_handler("on_conversation_item_updated") + async def on_conversation_item_updated(llm, item_id, item): + # `item` may not always be available here + # ... + ``` - @llm.event_handler("on_conversation_item_updated") - async def on_conversation_item_updated(llm, item_id, item): - # `item` may not always be available here - # ... - ``` + - The `retrieve_conversation_item(item_id)` method for introspecting a + conversation item on the server. -- Added `retrieve_conversation_item(item_id)` to `OpenAIRealtimeBetaLLMService`. - - ```python - item = await llm.retrieve_conversation_item(item_id) - ``` + ```python + item = await llm.retrieve_conversation_item(item_id) + ``` ### Changed @@ -268,9 +260,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `RimeTTSService` where the last line of text sent didn't result in an audio output being generated. -- Fixed `OpenAIRealtimeBetaLLMService` by adding support for the - `conversation.item.input_audio_transcription.delta` server message, which was - added server-side at some point and not handled client-side. +- Fixed `OpenAIRealtimeBetaLLMService` by adding proper handling for: + - The `conversation.item.input_audio_transcription.delta` server message, + which was added server-side at some point and not handled client-side. + - Errors reported by the `response.done` server message. ### Other From f94a099111a7b23952be85e7b0d13e9a09158117 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 19 Mar 2025 12:00:19 -0400 Subject: [PATCH 376/427] Revert the default model to be "gpt-4o-realtime-preview-2024-12-17" In OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 6 ++---- src/pipecat/services/openai_realtime_beta/openai.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe531bf7f..660cf5c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features: - - The `'gpt-4o-transcribe-latest'` input audio transcription model. + - The `'gpt-4o-transcribe-latest'` input audio transcription model, along + with new `language` and `prompt` options specific to that model. - The `input_audio_noise_reduction` session property. ```python @@ -171,9 +172,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated the default model for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. -- Updated the default model for `OpenAIRealtimeBetaLLMService` to - `gpt-4o-realtime-preview-latest`. - - Function calls are now executed in tasks. This means that the pipeline will not be blocked while the function call is being executed. diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 4b993c28a..ab78a4451 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -89,8 +89,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): self, *, api_key: str, - # model: str = "gpt-4o-realtime-preview-2024-12-17", - model: str = "gpt-4o-realtime-preview-latest", + model: str = "gpt-4o-realtime-preview-2024-12-17", base_url: str = "wss://api.openai.com/v1/realtime", session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, From 0d74bcacb74388724e4b36bcf764869813d3698d Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Thu, 20 Mar 2025 01:12:13 +0000 Subject: [PATCH 377/427] updated models in the 07g example --- examples/foundational/07g-interruptible-openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index f9cac5910..fe4b2a53c 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -51,9 +51,11 @@ async def main(): # api_key="gsk_***", # model="whisper-large-v3", # ) - stt = OpenAISTTService(api_key=os.getenv("OPENAI_API_KEY"), model="whisper-1") + stt = OpenAISTTService( + api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe-latest" + ) - tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), voice="alloy") + tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini-tts-latest") llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") From 2ae5bdd8a96fe3f3333814846e8863758b4ed9f0 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Thu, 20 Mar 2025 01:20:07 +0000 Subject: [PATCH 378/427] lets talk about dogs --- examples/foundational/07g-interruptible-openai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07g-interruptible-openai.py b/examples/foundational/07g-interruptible-openai.py index fe4b2a53c..b66d0346a 100644 --- a/examples/foundational/07g-interruptible-openai.py +++ b/examples/foundational/07g-interruptible-openai.py @@ -52,7 +52,9 @@ async def main(): # model="whisper-large-v3", # ) stt = OpenAISTTService( - api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-transcribe-latest" + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o-transcribe-latest", + prompt="Expect words related to dogs, such as breed names.", ) tts = OpenAITTSService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini-tts-latest") @@ -62,7 +64,7 @@ async def main(): messages = [ { "role": "system", - "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + "content": "You are very knowledgable about dogs. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", }, ] From f0774268cc83145da2b92786b7646a386c801fe9 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 19 Mar 2025 21:58:46 -0400 Subject: [PATCH 379/427] Rename gpt-4o-transcribe-latest to gpt-4o-transcribe in OpenAIRealtimeBetaLLMService --- CHANGELOG.md | 2 +- src/pipecat/services/openai_realtime_beta/events.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 660cf5c82..6825a62f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -130,7 +130,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added support in `OpenAIRealtimeBetaLLMService` for a slate of new features: - - The `'gpt-4o-transcribe-latest'` input audio transcription model, along + - The `'gpt-4o-transcribe'` input audio transcription model, along with new `language` and `prompt` options specific to that model. - The `input_audio_noise_reduction` session property. diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 88638b5fd..3a18833d2 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe-latest"] +InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"] class InputAudioTranscription(BaseModel): @@ -29,9 +29,9 @@ class InputAudioTranscription(BaseModel): prompt: Optional[str] = None, ): super().__init__(model=model, language=language, prompt=prompt) - if self.model != "gpt-4o-transcribe-latest" and (self.language or self.prompt): + if self.model != "gpt-4o-transcribe" and (self.language or self.prompt): raise ValueError( - "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe-latest'" + "Fields 'language' and 'prompt' are only supported when model is 'gpt-4o-transcribe'" ) From 70dbf0d6fc952e594b09506744eca9c57142172c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 11:48:06 -0400 Subject: [PATCH 380/427] Updated default models for OpenAISTTService and OpenAITTSService to gpt-4o based models --- CHANGELOG.md | 5 +++++ src/pipecat/services/openai.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6825a62f5..ac0a6ef3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,6 +169,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Updated `OpenAISTTService` to use `gpt-4o-transcribe` as the default + transcription model. + +- Updated `OpenAITTSService` to use `gpt-4o-mini-tts` as the default TTS model. + - Updated the default model for `CartesiaTTSService` and `CartesiaHttpTTSService` to `sonic-2`. diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index b5f5d9d54..4d3e1d6d7 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -409,13 +409,13 @@ class OpenAIImageGenService(ImageGenService): class OpenAISTTService(BaseWhisperSTTService): - """OpenAI Whisper speech-to-text service. + """OpenAI Speech-to-Text service that generates text from audio. - Uses OpenAI's Whisper API to convert audio to text. Requires an OpenAI API key + Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key set via the api_key parameter or OPENAI_API_KEY environment variable. Args: - model: Whisper model to use. Defaults to "whisper-1". + model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe". api_key: OpenAI API key. Defaults to None. base_url: API base URL. Defaults to None. language: Language of the audio input. Defaults to English. @@ -427,7 +427,7 @@ class OpenAISTTService(BaseWhisperSTTService): def __init__( self, *, - model: str = "whisper-1", + model: str = "gpt-4o-transcribe", api_key: Optional[str] = None, base_url: Optional[str] = None, language: Optional[Language] = Language.EN, @@ -472,7 +472,7 @@ class OpenAITTSService(TTSService): Args: api_key: OpenAI API key. Defaults to None. voice: Voice ID to use. Defaults to "alloy". - model: TTS model to use. Defaults to "tts-1". + model: TTS model to use. Defaults to "gpt-4o-mini-tts". sample_rate: Output audio sample rate in Hz. Defaults to None. **kwargs: Additional keyword arguments passed to TTSService. @@ -487,7 +487,7 @@ class OpenAITTSService(TTSService): *, api_key: Optional[str] = None, voice: str = "alloy", - model: str = "tts-1", + model: str = "gpt-4o-mini-tts", sample_rate: Optional[int] = None, **kwargs, ): From ada68f0699e09323f5d05b0ff5ce461754437f4e Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 19 Mar 2025 22:25:21 -0400 Subject: [PATCH 381/427] More robust handling of conversation item retrieval errors in OpenAIRealtimeBetaLLMService --- .../services/openai_realtime_beta/events.py | 1 + .../services/openai_realtime_beta/openai.py | 23 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 3a18833d2..f4133766b 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -122,6 +122,7 @@ class RealtimeError(BaseModel): code: Optional[str] = "" message: str param: Optional[str] = None + event_id: Optional[str] = None # diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index ab78a4451..324e5653a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -23,8 +23,6 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") -import re - from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, @@ -130,14 +128,20 @@ class OpenAIRealtimeBetaLLMService(LLMService): async def retrieve_conversation_item(self, item_id: str): future = self.get_event_loop().create_future() - retrieval_in_progress = False + retrieval_in_flight = False if not self._retrieve_conversation_item_futures.get(item_id): self._retrieve_conversation_item_futures[item_id] = [] else: - retrieval_in_progress = True + retrieval_in_flight = True self._retrieve_conversation_item_futures[item_id].append(future) - if not retrieval_in_progress: - await self.send_client_event(events.ConversationItemRetrieveEvent(item_id=item_id)) + if not retrieval_in_flight: + await self.send_client_event( + # Set event_id to "rci_{item_id}" so that we can identiy an + # error later if the retrieval fails. We don't need a UUID + # suffix to the event_id because we're ensuring only one + # in-flight retrieval per item_id + events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}") + ) return await future # @@ -543,11 +547,8 @@ class OpenAIRealtimeBetaLLMService(LLMService): Otherwise: - return false """ - match = re.match( - r"^Error retrieving item: the item with id '(.*)' does not exist\.$", evt.error.message - ) - if match: - item_id = match.group(1) + if evt.error.code == "item_retrieve_invalid_item_id": + item_id = evt.error.event_id.split("_", 1)[1] # event_id is of the form "rci_{item_id}" futures = self._retrieve_conversation_item_futures.pop(item_id, None) if futures: for future in futures: From 721ee75887441827c46819b4efff4e4895df93c1 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 12:41:40 -0400 Subject: [PATCH 382/427] Comment tweak --- src/pipecat/services/openai_realtime_beta/openai.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 324e5653a..c8f1f597a 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -136,10 +136,11 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._retrieve_conversation_item_futures[item_id].append(future) if not retrieval_in_flight: await self.send_client_event( - # Set event_id to "rci_{item_id}" so that we can identiy an + # Set event_id to "rci_{item_id}" so that we can identify an # error later if the retrieval fails. We don't need a UUID # suffix to the event_id because we're ensuring only one - # in-flight retrieval per item_id + # in-flight retrieval per item_id. (Note: "rci" = "retrieve + # conversation item") events.ConversationItemRetrieveEvent(item_id=item_id, event_id=f"rci_{item_id}") ) return await future From 44380bc8c0370dffe9685b3542f679b5d2b4ada0 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 13:51:16 -0400 Subject: [PATCH 383/427] Remove duplicate changelog entry due to rebase mistake --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0a6ef3c..065a5ced6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,9 +174,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `OpenAITTSService` to use `gpt-4o-mini-tts` as the default TTS model. -- Updated the default model for `CartesiaTTSService` and - `CartesiaHttpTTSService` to `sonic-2`. - - Function calls are now executed in tasks. This means that the pipeline will not be blocked while the function call is being executed. From efa5f133d747783c43c11ff5b1aad929a1f01484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 11:14:59 -0700 Subject: [PATCH 384/427] openai_realtime: fix and update function calling --- .../services/openai_realtime_beta/context.py | 72 +++---------------- .../services/openai_realtime_beta/openai.py | 4 +- 2 files changed, 11 insertions(+), 65 deletions(-) diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 31639dc6b..c8381976f 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -12,6 +12,7 @@ from loguru import logger from pipecat.frames.frames import ( Frame, + FunctionCallResultFrame, FunctionCallResultProperties, LLMMessagesUpdateFrame, LLMSetToolsFrame, @@ -174,67 +175,12 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): - async def push_aggregation(self): - # the only thing we implement here is function calling. in all other cases, messages - # are added to the context when we receive openai realtime api events - if not self._function_call_result: - return + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + await super().handle_function_call_result(frame) - properties: Optional[FunctionCallResultProperties] = None - - self.reset() - try: - run_llm = True - frame = self._function_call_result - properties = frame.properties - self._function_call_result = None - if frame.result: - # The "tool_call" message from the LLM that triggered the function call - self._context.add_message( - { - "role": "assistant", - "tool_calls": [ - { - "id": frame.tool_call_id, - "function": { - "name": frame.function_name, - "arguments": json.dumps(frame.arguments), - }, - "type": "function", - } - ], - } - ) - # The result of the function call. Need to add this both to our context here and to - # the openai realtime api context. - result_message = { - "role": "tool", - "content": json.dumps(frame.result), - "tool_call_id": frame.tool_call_id, - } - - self._context.add_message(result_message) - # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, - # so we didn't have a chance to add the result to the openai realtime api context. Let's push a - # special frame to do that. - await self.push_frame( - RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) - if properties and properties.run_llm is not None: - # If the tool call result has a run_llm property, use it - run_llm = properties.run_llm - else: - # Default behavior is to run the LLM if there are no function calls in progress - run_llm = not bool(self._function_calls_in_progress) - - if run_llm: - await self.push_context_frame(FrameDirection.UPSTREAM) - - # Emit the on_context_updated callback once the function call result is added to the context - if properties and properties.on_context_updated is not None: - await properties.on_context_updated() - - await self.push_context_frame() - - except Exception as e: - logger.error(f"Error processing frame: {e}") + # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, + # so we didn't have a chance to add the result to the openai realtime api context. Let's push a + # special frame to do that. + await self.push_frame( + RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index c8f1f597a..a1f4bc731 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -579,7 +579,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): arguments = json.loads(item.arguments) if self.has_function(function_name): run_llm = index == total_items - 1 - if function_name in self._callbacks.keys(): + if function_name in self._functions.keys(): await self.call_function( context=self._context, tool_call_id=tool_id, @@ -587,7 +587,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): arguments=arguments, run_llm=run_llm, ) - elif None in self._callbacks.keys(): + elif None in self._functions.keys(): await self.call_function( context=self._context, tool_call_id=tool_id, From 5a39f146f604d7349344d5d46913c2511d648702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 09:47:33 -0700 Subject: [PATCH 385/427] LLMUserContextAggregator: fix emulated user started/stopped speaking issues --- CHANGELOG.md | 7 ++- .../processors/aggregators/llm_response.py | 46 +++++++------------ src/pipecat/transports/base_input.py | 4 +- tests/test_context_aggregators.py | 7 +-- 4 files changed, 22 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 065a5ced6..0a3563229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -367,6 +367,9 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) ### Fixed +- Fixed an issue that would cause undesired interruptions via + `EmulateUserStartedSpeakingFrame`. + - Fixed a `GoogleLLMService` that was causing an exception when sending inline audio in some cases. @@ -383,10 +386,6 @@ stt = DeepgramSTTService(..., live_options=LiveOptions(model="nova-2-general")) - Fixed `match_endofsentence` support for ellipses. -- Fixed an issue that would cause undesired interruptions via - `EmulateUserStartedSpeakingFrame` when only interim transcriptions (i.e. no - final transcriptions) where received. - - Fixed an issue where `EndTaskFrame` was not triggering `on_client_disconnected` or closing the WebSocket in FastAPI. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 9267af364..75435a214 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -5,7 +5,6 @@ # import asyncio -import time from abc import abstractmethod from typing import Dict, List @@ -222,17 +221,15 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self, context: OpenAILLMContext, aggregation_timeout: float = 1.0, - bot_interruption_timeout: float = 5.0, **kwargs, ): super().__init__(context=context, role="user", **kwargs) self._aggregation_timeout = aggregation_timeout - self._bot_interruption_timeout = bot_interruption_timeout self._seen_interim_results = False self._user_speaking = False - self._last_user_speaking_time = 0 self._emulating_vad = False + self._waiting_for_aggregation = False self._aggregation_event = asyncio.Event() self._aggregation_task = None @@ -240,6 +237,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): def reset(self): super().reset() self._seen_interim_results = False + self._waiting_for_aggregation = False async def handle_aggregation(self, aggregation: str): self._context.add_message({"role": self.role, "content": self._aggregation}) @@ -285,14 +283,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" + self.reset() frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) - # Reset our accumulator state. - self.reset() - async def _start(self, frame: StartFrame): self._create_aggregation_task() @@ -303,12 +298,14 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await self._cancel_aggregation_task() async def _handle_user_started_speaking(self, _: UserStartedSpeakingFrame): - self._last_user_speaking_time = time.time() self._user_speaking = True + self._waiting_for_aggregation = True async def _handle_user_stopped_speaking(self, _: UserStoppedSpeakingFrame): - self._last_user_speaking_time = time.time() self._user_speaking = False + # We just stopped speaking. Let's see if there's some aggregation to + # push. If the last thing we saw is an interim transcription, let's wait + # pushing the aggregation as we will probably get a final transcription. if not self._seen_interim_results: await self.push_aggregation() @@ -361,18 +358,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): frame we might want to interrupt the bot. """ - if not self._user_speaking: - diff_time = time.time() - self._last_user_speaking_time - if diff_time > self._bot_interruption_timeout: - # If we reach this case we received a transcription but VAD was - # not able to detect voice (e.g. when you whisper a short - # utterance). So, we need to emulate VAD (i.e. user - # start/stopped speaking). - await self.push_frame(EmulateUserStartedSpeakingFrame(), FrameDirection.UPSTREAM) - self._emulating_vad = True - - # Reset time so we don't interrupt again right away. - self._last_user_speaking_time = time.time() + if not self._user_speaking and not self._waiting_for_aggregation: + # If we reach this case we received a transcription but VAD was not + # able to detect voice (e.g. when you whisper a short + # utterance). So, we need to emulate VAD (i.e. user start/stopped + # speaking). + await self.push_frame(EmulateUserStartedSpeakingFrame(), FrameDirection.UPSTREAM) + self._emulating_vad = True class LLMAssistantContextAggregator(LLMContextResponseAggregator): @@ -554,14 +546,11 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" + self.reset() frame = LLMMessagesFrame(self._context.messages) await self.push_frame(frame) - # Reset our accumulator state. - self.reset() - class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): def __init__(self, messages: List[dict] = [], **kwargs): @@ -573,10 +562,7 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): # Reset the aggregation. Reset it before pushing it down, otherwise # if the tasks gets cancelled we won't be able to clear things up. - self._aggregation = "" + self.reset() frame = LLMMessagesFrame(self._context.messages) await self.push_frame(frame) - - # Reset our accumulator state. - self.reset() diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 782ad1333..971dfe066 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -152,6 +152,7 @@ class BaseInputTransport(FrameProcessor): async def _handle_user_interruption(self, frame: Frame): if isinstance(frame, UserStartedSpeakingFrame): logger.debug("User started speaking") + await self.push_frame(frame) # Make sure we notify about interruptions quickly out-of-band. if self.interruptions_allowed: await self._start_interruption() @@ -161,12 +162,11 @@ class BaseInputTransport(FrameProcessor): await self.push_frame(StartInterruptionFrame()) elif isinstance(frame, UserStoppedSpeakingFrame): logger.debug("User stopped speaking") + await self.push_frame(frame) if self.interruptions_allowed: await self._stop_interruption() await self.push_frame(StopInterruptionFrame()) - await self.push_frame(frame) - # # Audio input # diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index 0c9b6d5e4..185725632 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -44,8 +44,6 @@ from pipecat.tests.utils import SleepFrame, run_test AGGREGATION_TIMEOUT = 0.1 AGGREGATION_SLEEP = 0.15 -BOT_INTERRUPTION_TIMEOUT = 0.2 -BOT_INTERRUPTION_SLEEP = 0.25 class BaseTestUserContextAggregator: @@ -388,14 +386,13 @@ class BaseTestUserContextAggregator: aggregator = self.AGGREGATOR_CLASS( context, aggregation_timeout=AGGREGATION_TIMEOUT, - bot_interruption_timeout=BOT_INTERRUPTION_TIMEOUT, ) frames_to_send = [ UserStartedSpeakingFrame(), InterimTranscriptionFrame(text="How ", user_id="cat", timestamp=""), SleepFrame(), UserStoppedSpeakingFrame(), - SleepFrame(BOT_INTERRUPTION_SLEEP), + SleepFrame(AGGREGATION_SLEEP), InterimTranscriptionFrame(text="are you?", user_id="cat", timestamp=""), TranscriptionFrame(text="How are you?", user_id="cat", timestamp=""), SleepFrame(sleep=AGGREGATION_SLEEP), @@ -405,12 +402,10 @@ class BaseTestUserContextAggregator: UserStoppedSpeakingFrame, *self.EXPECTED_CONTEXT_FRAMES, ] - expected_up_frames = [EmulateUserStartedSpeakingFrame, EmulateUserStoppedSpeakingFrame] await run_test( aggregator, frames_to_send=frames_to_send, expected_down_frames=expected_down_frames, - expected_up_frames=expected_up_frames, ) self.check_message_content(context, 0, "How are you?") From 08956e914a93146e7cdec4ad3c27d6375176a708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 10:06:45 -0700 Subject: [PATCH 386/427] livekit: remove unnecessary transport cleanup() function --- src/pipecat/transports/services/livekit.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 149ca4b7c..8ce5c885c 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -599,13 +599,6 @@ class LiveKitTransport(BaseTransport): ) await self._output.send_message(frame) - async def cleanup(self): - if self._input: - await self._input.cleanup() - if self._output: - await self._output.cleanup() - await self._client.disconnect() - async def on_room_event(self, event): # Handle room events pass From 66ba1116a40c9f599180f226b1929c6786621321 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 10:49:13 -0700 Subject: [PATCH 387/427] pyproject: rollback azure to 1.42.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 87a34f929..4e300bd91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ Website = "https://pipecat.ai" anthropic = [ "anthropic~=0.49.0" ] assemblyai = [ "assemblyai~=0.37.0" ] aws = [ "boto3~=1.37.16" ] -azure = [ "azure-cognitiveservices-speech~=1.43.0"] +azure = [ "azure-cognitiveservices-speech~=1.42.0"] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] From b20ce7d655adb5ab87cfd6c9075f6e2a2bf42498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 10:55:11 -0700 Subject: [PATCH 388/427] examples: move 07u-interruptible-neuphonic to 07v --- CHANGELOG.md | 10 ++++++++++ ...tible-ultravox.py => 07u-interruptible-ultravox.py} | 0 ...nic-http.py => 07v-interruptible-neuphonic-http.py} | 0 ...ble-neuphonic.py => 07v-interruptible-neuphonic.py} | 0 4 files changed, 10 insertions(+) rename examples/foundational/{07v-interruptible-ultravox.py => 07u-interruptible-ultravox.py} (100%) rename examples/foundational/{07u-interruptible-neuphonic-http.py => 07v-interruptible-neuphonic-http.py} (100%) rename examples/foundational/{07u-interruptible-neuphonic.py => 07v-interruptible-neuphonic.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a3563229..787be8afd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added new `sample_rate` constructor parameter to `TavusVideoService` to allow changing the output sample rate. +- Added new `NeuphonicTTSService`. + (see https://neuphonic.com) + - Added new `UltravoxSTTService`. (see https://github.com/fixie-ai/ultravox) @@ -269,6 +272,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add foundational example `07w-interruptible-fal.py`, showing `FalSTTService`. +- Added a new Ultravox example + `examples/foundational/07u-interruptible-ultravox.py`. + +- Added new Neuphonic examples + `examples/foundational/07v-interruptible-neuphonic.py` and + `examples/foundational/07v-interruptible-neuphonic-http.py`. + - Added a new example `examples/foundational/36-user-email-gathering.py` to show how to gather user emails. The example uses's Cartesia's `` tags and Rime `spell()` function to spell out the emails for confirmation. diff --git a/examples/foundational/07v-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py similarity index 100% rename from examples/foundational/07v-interruptible-ultravox.py rename to examples/foundational/07u-interruptible-ultravox.py diff --git a/examples/foundational/07u-interruptible-neuphonic-http.py b/examples/foundational/07v-interruptible-neuphonic-http.py similarity index 100% rename from examples/foundational/07u-interruptible-neuphonic-http.py rename to examples/foundational/07v-interruptible-neuphonic-http.py diff --git a/examples/foundational/07u-interruptible-neuphonic.py b/examples/foundational/07v-interruptible-neuphonic.py similarity index 100% rename from examples/foundational/07u-interruptible-neuphonic.py rename to examples/foundational/07v-interruptible-neuphonic.py From 2133152e5b98f63ff525ec820fd1358cf8816cc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 11:42:54 -0700 Subject: [PATCH 389/427] update CHANGELOG for 0.0.59 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 787be8afd..6d7b6f72e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.59] - 2025-03-20 ### Added From 8f6d92ce7d11a8107e72a5604f704a4156aec371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 13:47:15 -0700 Subject: [PATCH 390/427] update CHANGELOG with BaseOpenAILLMService `default_headers` --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7b6f72e..3a0524e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Added `default_headers` parameter to `BaseOpenAILLMService` constructor. + ## [0.0.59] - 2025-03-20 ### Added From 541a4b6063140e7d92c9f7982cbb492f4581a8be Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 15:12:43 -0400 Subject: [PATCH 391/427] Update InputAudioTranscription to use gpt-4o-transcribe model, update 19 examples to use FunctionSchema --- CHANGELOG.md | 10 +++++ .../foundational/19-openai-realtime-beta.py | 41 +++++++++--------- .../foundational/19a-azure-realtime-beta.py | 43 +++++++++---------- .../services/openai_realtime_beta/events.py | 13 ++++-- 4 files changed, 61 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a0524e29..63885acbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `default_headers` parameter to `BaseOpenAILLMService` constructor. +### Changed + +- Changed the default `InputAudioTranscription` model to `gpt-4o-transcribe` + for `OpenAIRealtimeBetaLLMService`. + +### Other + +- Update the `19-openai-realtime-beta.py` and `19a-azure-realtime-beta.py` + examples to use the FunctionSchema format. + ## [0.0.59] - 2025-03-20 ### Added diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 8796e0141..bb62466a7 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -14,6 +14,8 @@ from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -46,28 +48,25 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } -] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) async def main(): diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 14d034836..7ec1d195a 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -10,11 +10,12 @@ import sys from datetime import datetime import aiohttp -import websockets from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.adapters.schemas.function_schema import FunctionSchema +from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline @@ -47,28 +48,26 @@ async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context ) -tools = [ - { - "type": "function", - "name": "get_current_weather", - "description": "Get the current weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "format": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", - }, - }, - "required": ["location", "format"], +# Define weather function using standardized schema +weather_function = FunctionSchema( + name="get_current_weather", + description="Get the current weather", + properties={ + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", }, - } -] + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Infer this from the users location.", + }, + }, + required=["location", "format"], +) + +# Create tools schema +tools = ToolsSchema(standard_tools=[weather_function]) async def main(): diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index f4133766b..c2dcb7f09 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -14,17 +14,24 @@ from pydantic import BaseModel, Field # # session properties # -InputAudioTranscriptionModel = Literal["whisper-1", "gpt-4o-transcribe"] class InputAudioTranscription(BaseModel): - model: InputAudioTranscriptionModel + """Configuration for audio transcription settings. + + Attributes: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ + + model: str = "gpt-4o-transcribe" language: Optional[str] prompt: Optional[str] def __init__( self, - model: Optional[InputAudioTranscriptionModel] = "whisper-1", + model: Optional[str] = "gpt-4o-transcribe", language: Optional[str] = None, prompt: Optional[str] = None, ): From 41688205be87f443cbc73e08b215150befdbafa4 Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Thu, 20 Mar 2025 15:23:57 -0400 Subject: [PATCH 392/427] Provide new settings in OpenAI Realtime example --- examples/foundational/19-openai-realtime-beta.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index bb62466a7..f2349f6d2 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -26,7 +26,8 @@ from pipecat.services.openai_realtime_beta import ( InputAudioTranscription, OpenAIRealtimeBetaLLMService, SessionProperties, - TurnDetection, + SemanticTurnDetection, + InputAudioNoiseReduction ) from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -91,9 +92,10 @@ async def main(): input_audio_transcription=InputAudioTranscription(), # Set openai TurnDetection parameters. Not setting this at all will turn it # on by default - turn_detection=TurnDetection(silence_duration_ms=1000), + turn_detection=SemanticTurnDetection(), # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, + input_audio_noise_reduction=InputAudioNoiseReduction(type='near_field'), # tools=tools, instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. From 2ac8f2ec2d5acd26527a2e37510ddf17a521f5a0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 20 Mar 2025 15:50:51 -0400 Subject: [PATCH 393/427] Fix linting --- .../foundational/19-openai-realtime-beta.py | 6 +++--- .../foundational/19a-azure-realtime-beta.py | 1 - src/pipecat/services/openai.py | 19 ++++++++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index f2349f6d2..3aff14e65 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -23,11 +23,11 @@ from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai_realtime_beta import ( + InputAudioNoiseReduction, InputAudioTranscription, OpenAIRealtimeBetaLLMService, - SessionProperties, SemanticTurnDetection, - InputAudioNoiseReduction + SessionProperties, ) from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -95,7 +95,7 @@ async def main(): turn_detection=SemanticTurnDetection(), # Or set to False to disable openai turn detection and use transport VAD # turn_detection=False, - input_audio_noise_reduction=InputAudioNoiseReduction(type='near_field'), + input_audio_noise_reduction=InputAudioNoiseReduction(type="near_field"), # tools=tools, instructions="""Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 7ec1d195a..2eefd4ec9 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -26,7 +26,6 @@ from pipecat.services.openai_realtime_beta import ( AzureRealtimeBetaLLMService, InputAudioTranscription, SessionProperties, - TurnDetection, ) from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 48b108ad0..ff7bc0442 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -129,10 +129,23 @@ class BaseOpenAILLMService(LLMService): } self.set_model_name(model) self._client = self.create_client( - api_key=api_key, base_url=base_url, organization=organization, project=project, default_headers=default_headers, **kwargs + api_key=api_key, + base_url=base_url, + organization=organization, + project=project, + default_headers=default_headers, + **kwargs, ) - def create_client(self, api_key=None, base_url=None, organization=None, project=None, default_headers=None, **kwargs): + def create_client( + self, + api_key=None, + base_url=None, + organization=None, + project=None, + default_headers=None, + **kwargs, + ): return AsyncOpenAI( api_key=api_key, base_url=base_url, @@ -143,7 +156,7 @@ class BaseOpenAILLMService(LLMService): max_keepalive_connections=100, max_connections=1000, keepalive_expiry=None ) ), - default_headers=default_headers + default_headers=default_headers, ) def can_generate_metrics(self) -> bool: From 66e42ae4104b1e54806fb95ceab2824e2a75024e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 16:14:24 -0700 Subject: [PATCH 394/427] pyproject: rollback deepgram-sdk to 3.8.0 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63885acbb..7db8d9dc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Rollback to `deepgram-sdk` 3.8.0 since 3.10.1 was causing connections issues. + - Changed the default `InputAudioTranscription` model to `gpt-4o-transcribe` for `OpenAIRealtimeBetaLLMService`. diff --git a/pyproject.toml b/pyproject.toml index 4e300bd91..0d1e46ed9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] daily = [ "daily-python~=0.15.0" ] -deepgram = [ "deepgram-sdk~=3.10.1" ] +deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.9" ] fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] From f2b9789acfeb5d793f19a436bcf287043ca361c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 16:17:34 -0700 Subject: [PATCH 395/427] update CHANGELOG for 0.0.60 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7db8d9dc9..343efff17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.60] - 2025-03-20 ### Added From 3b4d91e1c17726667605a5e3e7f1abbd43db1443 Mon Sep 17 00:00:00 2001 From: milo157 <43028253+milo157@users.noreply.github.com> Date: Fri, 21 Mar 2025 03:55:43 +0200 Subject: [PATCH 396/427] Fixed ultravox service bugs (#1420) --- .../07u-interruptible-ultravox.py | 1 + src/pipecat/services/ultravox.py | 38 +++++++++++-------- tests/test_user_idle_processor.py | 6 +-- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/examples/foundational/07u-interruptible-ultravox.py b/examples/foundational/07u-interruptible-ultravox.py index 3ae4540f0..429e5a9fb 100644 --- a/examples/foundational/07u-interruptible-ultravox.py +++ b/examples/foundational/07u-interruptible-ultravox.py @@ -14,6 +14,7 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index 40029e673..a17ee3e85 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -17,6 +17,13 @@ from loguru import logger from pipecat.frames.frames import ( AudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + TranscriptionFrame, + TextFrame, + StartFrame, + EndFrame, CancelFrame, EndFrame, ErrorFrame, @@ -339,6 +346,12 @@ class UltravoxSTTService(AIService): # Concatenate audio frames - all should be int16 now audio_data = np.concatenate(audio_arrays) + audio_int16 = audio_data # Already in int16 format + # Save int16 audio + + # Convert int16 to float32 and normalize for model input + audio_float32 = audio_int16.astype(np.float32) / 32768.0 + # Generate text using the model if self._model: try: @@ -349,11 +362,11 @@ class UltravoxSTTService(AIService): await self.start_ttfb_metrics() await self.start_processing_metrics() - async for response in self._model.generate( + async for response in self.model.generate( messages=[{"role": "user", "content": "<|audio|>\n"}], - temperature=self._temperature, - max_tokens=self._max_tokens, - audio=audio_data, + temperature=self.temperature, + max_tokens=self.max_tokens, + audio=audio_float32, ): # Stop TTFB metrics after first response await self.stop_ttfb_metrics() @@ -369,18 +382,13 @@ class UltravoxSTTService(AIService): await self.stop_processing_metrics() logger.info(f"Generated text: {full_response}") - # Create a transcription frame with the generated text - transcription = full_response.strip() - if transcription: - yield TranscriptionFrame( - user_id="", - text=transcription, - timestamp=time_now_iso8601(), - ) - else: - logger.warning("Empty transcription result") - yield ErrorFrame("Empty transcription result") + yield LLMFullResponseStartFrame() + + text_frame = LLMTextFrame(text=full_response.strip()) + yield text_frame + + yield LLMFullResponseEndFrame() except Exception as e: logger.error(f"Error generating text from model: {e}") diff --git a/tests/test_user_idle_processor.py b/tests/test_user_idle_processor.py index a2f2fd386..7ea6f8744 100644 --- a/tests/test_user_idle_processor.py +++ b/tests/test_user_idle_processor.py @@ -86,9 +86,9 @@ class TestUserIdleProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_down_frames, ) - assert not callback_called.is_set(), ( - "Idle callback was called even though bot speaking frames reset the timer" - ) + assert ( + not callback_called.is_set() + ), "Idle callback was called even though bot speaking frames reset the timer" async def test_idle_retry_callback(self): """Test that retry count increases until user activity resets it.""" From d71b5201539209be80c165e003bd744eec4256b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 18:58:06 -0700 Subject: [PATCH 397/427] update CHANGELOG.md and fix formatting --- CHANGELOG.md | 7 +++++++ tests/test_user_idle_processor.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 343efff17..701b8a81c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Fixed an issue in `UltravoxSTTService` that caused improper audio processing + and incorrect LLM frame output. + ## [0.0.60] - 2025-03-20 ### Added diff --git a/tests/test_user_idle_processor.py b/tests/test_user_idle_processor.py index 7ea6f8744..a2f2fd386 100644 --- a/tests/test_user_idle_processor.py +++ b/tests/test_user_idle_processor.py @@ -86,9 +86,9 @@ class TestUserIdleProcessor(unittest.IsolatedAsyncioTestCase): expected_down_frames=expected_down_frames, ) - assert ( - not callback_called.is_set() - ), "Idle callback was called even though bot speaking frames reset the timer" + assert not callback_called.is_set(), ( + "Idle callback was called even though bot speaking frames reset the timer" + ) async def test_idle_retry_callback(self): """Test that retry count increases until user activity resets it.""" From fc78e6fc5a0062a266ec122a0e52297cb3a416da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 19:13:23 -0700 Subject: [PATCH 398/427] ElevenLabs: add support for a sample rate of 8000 --- CHANGELOG.md | 4 ++++ src/pipecat/services/elevenlabs.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 701b8a81c..c79c8f3b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- ElevenLabs TTS services now support a sample rate of 8000. + ### Fixed - Fixed an issue in `UltravoxSTTService` that caused improper audio processing diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 568b9eb64..68c71a144 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -102,6 +102,8 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: def output_format_from_sample_rate(sample_rate: int) -> str: match sample_rate: + case 8000: + return "pcm_8000" case 16000: return "pcm_16000" case 22050: @@ -113,7 +115,7 @@ def output_format_from_sample_rate(sample_rate: int) -> str: logger.warning( f"ElevenLabsTTSService: No output format available for {sample_rate} sample rate" ) - return "pcm_16000" + return "pcm_24000" def build_elevenlabs_voice_settings( From 442f18d47b323cb0494a37f876bb1cf01377251b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 19:16:56 -0700 Subject: [PATCH 399/427] ultravox: fix formatting --- src/pipecat/services/ultravox.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/ultravox.py b/src/pipecat/services/ultravox.py index a17ee3e85..52a5d05eb 100644 --- a/src/pipecat/services/ultravox.py +++ b/src/pipecat/services/ultravox.py @@ -17,25 +17,19 @@ from loguru import logger from pipecat.frames.frames import ( AudioRawFrame, - LLMFullResponseEndFrame, - LLMFullResponseStartFrame, - LLMTextFrame, - TranscriptionFrame, - TextFrame, - StartFrame, - EndFrame, CancelFrame, EndFrame, ErrorFrame, Frame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, StartFrame, - TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AIService -from pipecat.utils.time import time_now_iso8601 try: from transformers import AutoTokenizer From e77f7c84566dce84a5dd7b2b4ac9ac9bb6b11ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 20 Mar 2025 19:16:49 -0700 Subject: [PATCH 400/427] update ruff and pyright versions --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2b55adcbe..e65c2755c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,10 +3,10 @@ coverage~=7.6.12 grpcio-tools~=1.67.1 pip-tools~=7.4.1 pre-commit~=4.0.1 -pyright~=1.1.394 +pyright~=1.1.397 pytest~=8.3.4 pytest-asyncio~=0.25.3 -ruff~=0.9.7 +ruff~=0.11.1 setuptools~=70.0.0 setuptools_scm~=8.1.0 python-dotenv~=1.0.1 From 04d462ff022b92a98e59591f9300609da01c4322 Mon Sep 17 00:00:00 2001 From: allenmylath Date: Fri, 21 Mar 2025 10:09:09 +0530 Subject: [PATCH 401/427] Update requirements.txt example uses cartesia not elevenlabs --- examples/telnyx-chatbot/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/telnyx-chatbot/requirements.txt b/examples/telnyx-chatbot/requirements.txt index 630a98dd6..e103e438d 100644 --- a/examples/telnyx-chatbot/requirements.txt +++ b/examples/telnyx-chatbot/requirements.txt @@ -1,4 +1,4 @@ -pipecat-ai[openai,silero,deepgram,elevenlabs] +pipecat-ai[openai,silero,deepgram,cartesia] fastapi uvicorn python-dotenv From dd81048ddb8513f824fc796694da00b32e450f07 Mon Sep 17 00:00:00 2001 From: allenmylath Date: Fri, 21 Mar 2025 10:11:28 +0530 Subject: [PATCH 402/427] Update env.example EXAMPLE USES CARTESI NOT ELEVNE LABS --- examples/telnyx-chatbot/env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/telnyx-chatbot/env.example b/examples/telnyx-chatbot/env.example index 5de3bace2..1da398649 100644 --- a/examples/telnyx-chatbot/env.example +++ b/examples/telnyx-chatbot/env.example @@ -1,3 +1,3 @@ OPENAI_API_KEY= DEEPGRAM_API_KEY= -ELEVENLABS_API_KEY= +CARTESIA_API_KEY= From 3ed764a7699f8d290687f23d7ff30c7cc8f932d9 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Fri, 21 Mar 2025 12:56:05 -0300 Subject: [PATCH 403/427] Only checking the length if tools is a list. --- src/pipecat/processors/aggregators/openai_llm_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 2e5ade0a0..948e3e101 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -157,7 +157,7 @@ class OpenAILLMContext: self._tool_choice = tool_choice def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN): - if tools != NOT_GIVEN and len(tools) == 0: + if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0: tools = NOT_GIVEN self._tools = tools From 3311afc5819f1a59938999d9611ac0f3ae992cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sat, 22 Mar 2025 21:57:32 -0700 Subject: [PATCH 404/427] examples: add foundational 07x-interruptible-local.py --- CHANGELOG.md | 5 + .../foundational/07x-interruptible-local.py | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 examples/foundational/07x-interruptible-local.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c79c8f3b3..4749329ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed an issue in `UltravoxSTTService` that caused improper audio processing and incorrect LLM frame output. +### Other + +- Added `examples/foundational/07x-interruptible-local.py` to show how a local + transport can be used. + ## [0.0.60] - 2025-03-20 ### Added diff --git a/examples/foundational/07x-interruptible-local.py b/examples/foundational/07x-interruptible-local.py new file mode 100644 index 000000000..54942ad97 --- /dev/null +++ b/examples/foundational/07x-interruptible-local.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.local.audio import LocalAudioTransport, LocalAudioTransportParams + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + transport = LocalAudioTransport( + LocalAudioTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ) + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) From 48e8d3968a3680ee8c60782c0763f038b53033d4 Mon Sep 17 00:00:00 2001 From: "Thomas B." Date: Mon, 24 Mar 2025 03:29:52 +0100 Subject: [PATCH 405/427] fix: recognition language correctly set for Azure STT (#1436) --- src/pipecat/services/azure.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index c59cd29c2..9df1a8ef1 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -686,8 +686,11 @@ class AzureSTTService(STTService): ): super().__init__(sample_rate=sample_rate, **kwargs) - self._speech_config = SpeechConfig(subscription=api_key, region=region) - self._speech_config.speech_recognition_language = language + self._speech_config = SpeechConfig( + subscription=api_key, + region=region, + speech_recognition_language=language_to_azure_language(language), + ) self._audio_stream = None self._speech_recognizer = None From a9b1298f3bd6f89274a9e7c8bf2ab0bccb03e14f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 24 Mar 2025 10:25:31 -0400 Subject: [PATCH 406/427] Fix: RTVIObserver now outputs a single bot started and stopped speaking event per turn --- CHANGELOG.md | 3 +++ src/pipecat/processors/frameworks/rtvi.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c79c8f3b3..946d6bfa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue where the `RTVIObserver` would report two bot started and + stopped speaking events for each bot turn. + - Fixed an issue in `UltravoxSTTService` that caused improper audio processing and incorrect LLM frame output. diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index bb97c2098..f782e6ea8 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -440,7 +440,9 @@ class RTVIObserver(BaseObserver): if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): await self._handle_interruptions(frame) - elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)): + elif isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame)) and ( + direction == FrameDirection.UPSTREAM + ): await self._handle_bot_speaking(frame) elif isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)): await self._handle_user_transcriptions(frame) From 397bae29f70f8851ed528610edd465be577dceab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Mar 2025 15:39:35 -0700 Subject: [PATCH 407/427] LLMAssistantContextAggregator: create a task to run on_context_updated --- CHANGELOG.md | 3 +++ .../processors/aggregators/llm_response.py | 19 +++++++++++++++---- src/pipecat/services/ai_services.py | 2 +- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c06240bc..d7c8c5dd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that would cause `LLMAssistantContextAggregator` to block + processing more frames while processing a function call result. + - Fixed an issue where the `RTVIObserver` would report two bot started and stopped speaking events for each bot turn. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 75435a214..94be8b25b 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -6,7 +6,7 @@ import asyncio from abc import abstractmethod -from typing import Dict, List +from typing import Dict, List, Set from loguru import logger @@ -380,6 +380,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {} + self._context_updated_tasks: Set[asyncio.Task] = () async def handle_aggregation(self, aggregation: str): self._context.add_message({"role": "assistant", "content": aggregation}) @@ -486,10 +487,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): if run_llm: await self.push_context_frame(FrameDirection.UPSTREAM) - # Emit the on_context_updated callback once the function call - # result is added to the context + # Call the `on_context_updated` callback once the function call result + # is added to the context. Also, run this in a separate task to make + # sure we don't block the pipeline. if properties and properties.on_context_updated: - await properties.on_context_updated() + task = await self.create_task(properties.on_context_updated()) + self._context_updated_tasks.add(task) + task.add_done_callback(self._context_updated_task_finished) async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -535,6 +539,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): else: self._aggregation += frame.text + def _context_updated_task_finished(self, task: asyncio.Task): + self._context_updated_tasks.discard(task) + # The task is finished so this should exit immediately. We need to do + # this because otherwise the task manager would report a dangling task + # if we don't remove it. + asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) + class LLMUserResponseAggregator(LLMUserContextAggregator): def __init__(self, messages: List[dict] = [], **kwargs): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 9f9804e65..a78c268dd 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -369,7 +369,7 @@ class LLMService(AIService): if tuple_to_remove: self._function_call_tasks.discard(tuple_to_remove) # The task is finished so this should exit immediately. We need to - # do this because otherwise the task manager would have a dangling + # do this because otherwise the task manager would report a dangling # task if we don't remove it. asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) From f3b50bc3c4e243f2c39e39a51facc345e68259fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Mar 2025 15:40:26 -0700 Subject: [PATCH 408/427] Revert "LLMAssistantContextAggregator: create a task to run on_context_updated" This reverts commit 397bae29f70f8851ed528610edd465be577dceab. --- CHANGELOG.md | 3 --- .../processors/aggregators/llm_response.py | 19 ++++--------------- src/pipecat/services/ai_services.py | 2 +- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7c8c5dd8..1c06240bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Fixed an issue that would cause `LLMAssistantContextAggregator` to block - processing more frames while processing a function call result. - - Fixed an issue where the `RTVIObserver` would report two bot started and stopped speaking events for each bot turn. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 94be8b25b..75435a214 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -6,7 +6,7 @@ import asyncio from abc import abstractmethod -from typing import Dict, List, Set +from typing import Dict, List from loguru import logger @@ -380,7 +380,6 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {} - self._context_updated_tasks: Set[asyncio.Task] = () async def handle_aggregation(self, aggregation: str): self._context.add_message({"role": "assistant", "content": aggregation}) @@ -487,13 +486,10 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): if run_llm: await self.push_context_frame(FrameDirection.UPSTREAM) - # Call the `on_context_updated` callback once the function call result - # is added to the context. Also, run this in a separate task to make - # sure we don't block the pipeline. + # Emit the on_context_updated callback once the function call + # result is added to the context if properties and properties.on_context_updated: - task = await self.create_task(properties.on_context_updated()) - self._context_updated_tasks.add(task) - task.add_done_callback(self._context_updated_task_finished) + await properties.on_context_updated() async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -539,13 +535,6 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): else: self._aggregation += frame.text - def _context_updated_task_finished(self, task: asyncio.Task): - self._context_updated_tasks.discard(task) - # The task is finished so this should exit immediately. We need to do - # this because otherwise the task manager would report a dangling task - # if we don't remove it. - asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) - class LLMUserResponseAggregator(LLMUserContextAggregator): def __init__(self, messages: List[dict] = [], **kwargs): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index a78c268dd..9f9804e65 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -369,7 +369,7 @@ class LLMService(AIService): if tuple_to_remove: self._function_call_tasks.discard(tuple_to_remove) # The task is finished so this should exit immediately. We need to - # do this because otherwise the task manager would report a dangling + # do this because otherwise the task manager would have a dangling # task if we don't remove it. asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) From c99436b80eecd7e44be7804e0be16ddbbceeaf9f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Tue, 25 Mar 2025 17:26:31 -0400 Subject: [PATCH 409/427] Bump daily-python dependency to 0.16.0 to pick up support in `DailyTransport` for updating remote participants' `canReceive` permission via the `update_remote_participants()` method --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c06240bc..928ba88fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added support in `DailyTransport` for updating remote participants' + `canReceive` permission via the `update_remote_participants()` method, by + bumping the daily-python dependency to >= 0.16.0. + - ElevenLabs TTS services now support a sample rate of 8000. ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 0d1e46ed9..d2fa46dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.15.0" ] +daily = [ "daily-python~=0.16.0" ] deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.9" ] From 01458895c2b35e6d3af49def81159cab1d7aca30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 24 Mar 2025 15:39:35 -0700 Subject: [PATCH 410/427] LLMAssistantContextAggregator: create a task to run on_context_updated --- CHANGELOG.md | 3 +++ .../processors/aggregators/llm_response.py | 20 +++++++++++++++---- src/pipecat/processors/frame_processor.py | 7 +++++-- src/pipecat/services/ai_services.py | 2 +- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c06240bc..d7c8c5dd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that would cause `LLMAssistantContextAggregator` to block + processing more frames while processing a function call result. + - Fixed an issue where the `RTVIObserver` would report two bot started and stopped speaking events for each bot turn. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 75435a214..7e84f6376 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -6,7 +6,7 @@ import asyncio from abc import abstractmethod -from typing import Dict, List +from typing import Dict, List, Set from loguru import logger @@ -380,6 +380,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): self._started = 0 self._function_calls_in_progress: Dict[str, FunctionCallInProgressFrame] = {} + self._context_updated_tasks: Set[asyncio.Task] = set() async def handle_aggregation(self, aggregation: str): self._context.add_message({"role": "assistant", "content": aggregation}) @@ -486,10 +487,14 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): if run_llm: await self.push_context_frame(FrameDirection.UPSTREAM) - # Emit the on_context_updated callback once the function call - # result is added to the context + # Call the `on_context_updated` callback once the function call result + # is added to the context. Also, run this in a separate task to make + # sure we don't block the pipeline. if properties and properties.on_context_updated: - await properties.on_context_updated() + task_name = f"{frame.function_name}:{frame.tool_call_id}:on_context_updated" + task = self.create_task(properties.on_context_updated(), task_name) + self._context_updated_tasks.add(task) + task.add_done_callback(self._context_updated_task_finished) async def _handle_function_call_cancel(self, frame: FunctionCallCancelFrame): logger.debug( @@ -535,6 +540,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): else: self._aggregation += frame.text + def _context_updated_task_finished(self, task: asyncio.Task): + self._context_updated_tasks.discard(task) + # The task is finished so this should exit immediately. We need to do + # this because otherwise the task manager would report a dangling task + # if we don't remove it. + asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) + class LLMUserResponseAggregator(LLMUserContextAggregator): def __init__(self, messages: List[dict] = [], **kwargs): diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 847cdf175..590698e7f 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -147,10 +147,13 @@ class FrameProcessor(BaseObject): await self.stop_ttfb_metrics() await self.stop_processing_metrics() - def create_task(self, coroutine: Coroutine) -> asyncio.Task: + def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task: if not self._task_manager: raise Exception(f"{self} TaskManager is still not initialized.") - name = f"{self}::{coroutine.cr_code.co_name}" + if name: + name = f"{self}::{name}" + else: + name = f"{self}::{coroutine.cr_code.co_name}" return self._task_manager.create_task(coroutine, name) async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 9f9804e65..a78c268dd 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -369,7 +369,7 @@ class LLMService(AIService): if tuple_to_remove: self._function_call_tasks.discard(tuple_to_remove) # The task is finished so this should exit immediately. We need to - # do this because otherwise the task manager would have a dangling + # do this because otherwise the task manager would report a dangling # task if we don't remove it. asyncio.run_coroutine_threadsafe(self.wait_for_task(task), self.get_event_loop()) From 8aebf00c2d9006ac207165172e15f98a56f95e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Mar 2025 14:40:46 -0700 Subject: [PATCH 411/427] GoogleAssistantContextAggregator: function call result should be a JSON object --- CHANGELOG.md | 4 ++++ src/pipecat/frames/frames.py | 6 +++--- src/pipecat/services/anthropic.py | 2 +- src/pipecat/services/google/google.py | 24 ++++++++++-------------- src/pipecat/services/openai.py | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7c8c5dd8..f75d4eab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `GoogleAssistantContextAggregator` issue where function calls + placeholders where not being updated when then function call result was + different from a string. + - Fixed an issue that would cause `LLMAssistantContextAggregator` to block processing more frames while processing a function call result. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index c2a79461f..6452cbfe4 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -384,7 +384,7 @@ class FunctionCallResultFrame(DataFrame): function_name: str tool_call_id: str - arguments: str + arguments: Any result: Any properties: Optional[FunctionCallResultProperties] = None @@ -633,8 +633,8 @@ class FunctionCallInProgressFrame(SystemFrame): function_name: str tool_call_id: str - arguments: str - cancel_on_interruption: bool + arguments: Any + cancel_on_interruption: bool = False @dataclass diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 6a95d04e2..3e369075a 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -725,7 +725,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ) async def _update_function_call_result( - self, function_name: str, tool_call_id: str, result: str + self, function_name: str, tool_call_id: str, result: Any ): for message in self._context.messages: if message["role"] == "user": diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index bfddce46d..554d9cb6b 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -601,23 +601,18 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): async def handle_function_call_result(self, frame: FunctionCallResultFrame): if frame.result: - if not isinstance(frame.result, str): - return - - response = {"response": frame.result} - + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, frame.result + ) + else: + response = {"response": "COMPLETED"} await self._update_function_call_result( frame.function_name, frame.tool_call_id, response ) - else: - await self._update_function_call_result( - frame.function_name, frame.tool_call_id, "COMPLETED" - ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): - await self._update_function_call_result( - frame.function_name, frame.tool_call_id, "CANCELLED" - ) + response = {"response": "CANCELLED"} + await self._update_function_call_result(frame.function_name, frame.tool_call_id, response) async def _update_function_call_result( self, function_name: str, tool_call_id: str, result: Any @@ -626,11 +621,12 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if message.role == "user": for part in message.parts: if part.function_response and part.function_response.id == tool_call_id: - part.function_response.response = {"response": result} + part.function_response.response = result async def handle_user_image_frame(self, frame: UserImageRawFrame): + response = {"response": "COMPLETED"} await self._update_function_call_result( - frame.request.function_name, frame.request.tool_call_id, "COMPLETED" + frame.request.function_name, frame.request.tool_call_id, response ) self._context.add_image_frame_message( format=frame.format, diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index ff7bc0442..cb1edea72 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -613,7 +613,7 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ) async def _update_function_call_result( - self, function_name: str, tool_call_id: str, result: str + self, function_name: str, tool_call_id: str, result: Any ): for message in self._context.messages: if ( From 19b464ba23691c939ffd06f8f7704baef753e2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Mar 2025 14:41:33 -0700 Subject: [PATCH 412/427] tests: add assistant aggregator function call frame handling --- tests/test_context_aggregators.py | 97 ++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 13 deletions(-) diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index 185725632..baee3496f 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -4,13 +4,18 @@ # SPDX-License-Identifier: BSD 2-Clause License # +import json import unittest +from typing import Any import google.ai.generativelanguage as glm from pipecat.frames.frames import ( EmulateUserStartedSpeakingFrame, EmulateUserStoppedSpeakingFrame, + FunctionCallInProgressFrame, + FunctionCallResultFrame, + FunctionCallResultProperties, InterimTranscriptionFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, @@ -21,10 +26,7 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.llm_response import ( - LLMAssistantContextAggregator, - LLMUserContextAggregator, -) +from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -423,6 +425,9 @@ class BaseTestAssistantContextAggreagator: ): assert context.messages[index]["content"] == content + def check_function_call_result(self, context: OpenAILLMContext, index: int, content: str): + assert json.loads(context.messages[index]["content"]) == content + async def test_empty(self): assert self.CONTEXT_CLASS is not None, "CONTEXT_CLASS must be set in a subclass" assert self.AGGREGATOR_CLASS is not None, "AGGREGATOR_CLASS must be set in a subclass" @@ -556,9 +561,76 @@ class BaseTestAssistantContextAggreagator: self.check_message_multi_content(context, 0, 0, "Hello Pipecat.") self.check_message_multi_content(context, 0, 1, "How are you?") + async def test_function_call(self): + assert self.CONTEXT_CLASS is not None, "CONTEXT_CLASS must be set in a subclass" + assert self.AGGREGATOR_CLASS is not None, "AGGREGATOR_CLASS must be set in a subclass" + + context = self.CONTEXT_CLASS() + aggregator = self.AGGREGATOR_CLASS(context) + frames_to_send = [ + FunctionCallInProgressFrame( + function_name="get_weather", + tool_call_id="1", + arguments={"location": "Los Angeles"}, + cancel_on_interruption=False, + ), + SleepFrame(), + FunctionCallResultFrame( + function_name="get_weather", + tool_call_id="1", + arguments={"location": "Los Angeles"}, + result={"conditions": "Sunny"}, + ), + ] + expected_down_frames = [] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + self.check_function_call_result(context, -1, {"conditions": "Sunny"}) + + async def test_function_call_on_context_updated(self): + assert self.CONTEXT_CLASS is not None, "CONTEXT_CLASS must be set in a subclass" + assert self.AGGREGATOR_CLASS is not None, "AGGREGATOR_CLASS must be set in a subclass" + + context_updated = False + + async def on_context_updated(): + nonlocal context_updated + context_updated = True + + context = self.CONTEXT_CLASS() + aggregator = self.AGGREGATOR_CLASS(context) + frames_to_send = [ + FunctionCallInProgressFrame( + function_name="get_weather", + tool_call_id="1", + arguments={"location": "Los Angeles"}, + cancel_on_interruption=False, + ), + SleepFrame(), + FunctionCallResultFrame( + function_name="get_weather", + tool_call_id="1", + arguments={"location": "Los Angeles"}, + result={"conditions": "Sunny"}, + properties=FunctionCallResultProperties(on_context_updated=on_context_updated), + ), + SleepFrame(), + ] + expected_down_frames = [] + await run_test( + aggregator, + frames_to_send=frames_to_send, + expected_down_frames=expected_down_frames, + ) + self.check_function_call_result(context, -1, {"conditions": "Sunny"}) + assert context_updated + # -# LLMUserContextAggregator, LLMAssistantContextAggregator +# LLMUserContextAggregator # @@ -567,14 +639,6 @@ class TestLLMUserContextAggregator(BaseTestUserContextAggregator, unittest.Isola AGGREGATOR_CLASS = LLMUserContextAggregator -class TestLLMAssistantContextAggregator( - BaseTestAssistantContextAggreagator, unittest.IsolatedAsyncioTestCase -): - CONTEXT_CLASS = OpenAILLMContext - AGGREGATOR_CLASS = LLMAssistantContextAggregator - EXPECTED_CONTEXT_FRAMES = [OpenAILLMContextFrame, OpenAILLMContextAssistantTimestampFrame] - - # # OpenAI # @@ -626,6 +690,9 @@ class TestAnthropicAssistantContextAggregator( messages = context.messages[content_index] assert messages["content"][index]["text"] == content + def check_function_call_result(self, context: OpenAILLMContext, index: int, content: Any): + assert context.messages[index]["content"][0]["content"] == json.dumps(content) + # # Google @@ -665,3 +732,7 @@ class TestGoogleAssistantContextAggregator( ): obj = glm.Content.to_dict(context.messages[index]) assert obj["parts"][0]["text"] == content + + def check_function_call_result(self, context: OpenAILLMContext, index: int, content: Any): + obj = glm.Content.to_dict(context.messages[index]) + assert obj["parts"][0]["function_response"]["response"] == content From 077952b6584d6fc97971c76447ddd09c844623ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 25 Mar 2025 19:11:27 -0700 Subject: [PATCH 413/427] GoogleAssistantContextAggregator: allow any value as function call result --- src/pipecat/services/google/google.py | 13 ++++++------- tests/test_context_aggregators.py | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pipecat/services/google/google.py b/src/pipecat/services/google/google.py index 554d9cb6b..3d207a254 100644 --- a/src/pipecat/services/google/google.py +++ b/src/pipecat/services/google/google.py @@ -605,14 +605,14 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): frame.function_name, frame.tool_call_id, frame.result ) else: - response = {"response": "COMPLETED"} await self._update_function_call_result( - frame.function_name, frame.tool_call_id, response + frame.function_name, frame.tool_call_id, "COMPLETED" ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): - response = {"response": "CANCELLED"} - await self._update_function_call_result(frame.function_name, frame.tool_call_id, response) + await self._update_function_call_result( + frame.function_name, frame.tool_call_id, "CANCELLED" + ) async def _update_function_call_result( self, function_name: str, tool_call_id: str, result: Any @@ -621,12 +621,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if message.role == "user": for part in message.parts: if part.function_response and part.function_response.id == tool_call_id: - part.function_response.response = result + part.function_response.response = {"value": json.dumps(result)} async def handle_user_image_frame(self, frame: UserImageRawFrame): - response = {"response": "COMPLETED"} await self._update_function_call_result( - frame.request.function_name, frame.request.tool_call_id, response + frame.request.function_name, frame.request.tool_call_id, "COMPLETED" ) self._context.add_image_frame_message( format=frame.format, diff --git a/tests/test_context_aggregators.py b/tests/test_context_aggregators.py index baee3496f..df00f11e4 100644 --- a/tests/test_context_aggregators.py +++ b/tests/test_context_aggregators.py @@ -735,4 +735,4 @@ class TestGoogleAssistantContextAggregator( def check_function_call_result(self, context: OpenAILLMContext, index: int, content: Any): obj = glm.Content.to_dict(context.messages[index]) - assert obj["parts"][0]["function_response"]["response"] == content + assert obj["parts"][0]["function_response"]["response"]["value"] == json.dumps(content) From ce9f75a8515e8741075abb87ac05611543f47c4b Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 26 Mar 2025 08:17:50 -0300 Subject: [PATCH 414/427] Fixing the tool choice extra type to be a dict instead of string. --- src/pipecat/frames/frames.py | 2 +- src/pipecat/processors/aggregators/llm_response.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 39e25fc11..11cd17801 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -367,7 +367,7 @@ class LLMSetToolsFrame(DataFrame): class LLMSetToolChoiceFrame(DataFrame): """A frame containing a tool choice for an LLM to use for function calling.""" - tool_choice: Literal["none", "auto", "required"] | str + tool_choice: Literal["none", "auto", "required"] | dict @dataclass diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index c0ef65f12..af8bf1a2e 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -210,7 +210,7 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): def set_tools(self, tools: List): self._context.set_tools(tools) - def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | str): + def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict): self._context.set_tool_choice(tool_choice) def reset(self): From aeac40312e6f67e1c3c0dc420a448bde409f3ed0 Mon Sep 17 00:00:00 2001 From: Filipi Fuchter Date: Wed, 26 Mar 2025 09:06:29 -0300 Subject: [PATCH 415/427] Added the feature to change dynamically the tool choice to the changelog. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47c2560e..e88155896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a new frame, `LLMSetToolChoiceFrame`, which provides a mechanism + for modifying the `tool_choice` in the context. + - Added support in `DailyTransport` for updating remote participants' `canReceive` permission via the `update_remote_participants()` method, by bumping the daily-python dependency to >= 0.16.0. From 72d373e5655710f9a95a869c3478b1775d921ac9 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:25:16 +0100 Subject: [PATCH 416/427] feature/support instructions in OpenAITTSService --- src/pipecat/services/openai.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index cb1edea72..3be1df243 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -391,6 +391,7 @@ class OpenAIImageGenService(ImageGenService): self, *, api_key: str, + base_url: Optional[str] = None, aiohttp_session: aiohttp.ClientSession, image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"], model: str = "dall-e-3", @@ -398,7 +399,7 @@ class OpenAIImageGenService(ImageGenService): super().__init__() self.set_model_name(model) self._image_size = image_size - self._client = AsyncOpenAI(api_key=api_key) + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) self._aiohttp_session = aiohttp_session async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: @@ -501,9 +502,12 @@ class OpenAITTSService(TTSService): self, *, api_key: Optional[str] = None, + base_url: Optional[str] = None, + websocket_base_url: Optional[str] = None, voice: str = "alloy", model: str = "gpt-4o-mini-tts", sample_rate: Optional[int] = None, + instructions: Optional[str] = None, **kwargs, ): if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: @@ -515,8 +519,8 @@ class OpenAITTSService(TTSService): self.set_model_name(model) self.set_voice(voice) - - self._client = AsyncOpenAI(api_key=api_key) + self._instructions = instructions + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url) def can_generate_metrics(self) -> bool: return True @@ -538,11 +542,17 @@ class OpenAITTSService(TTSService): try: await self.start_ttfb_metrics() + # Setup extra body parameters + extra_body = {} + if self._instructions: + extra_body["instructions"] = self._instructions + async with self._client.audio.speech.with_streaming_response.create( input=text or " ", # Text must contain at least one character model=self.model_name, voice=VALID_VOICES[self._voice_id], response_format="pcm", + extra_body=extra_body, ) as r: if r.status_code != 200: error = await r.text() From d982fc35d8aebadaadb6449201917f7e741661f8 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Tue, 25 Mar 2025 10:35:53 +0100 Subject: [PATCH 417/427] fix: formatter --- src/pipecat/services/openai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 3be1df243..c631fd8c0 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -520,7 +520,9 @@ class OpenAITTSService(TTSService): self.set_model_name(model) self.set_voice(voice) self._instructions = instructions - self._client = AsyncOpenAI(api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url) + self._client = AsyncOpenAI( + api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url + ) def can_generate_metrics(self) -> bool: return True From dc2ee2bf0a96393bf6e01f85d3671df0bcdf5119 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Wed, 26 Mar 2025 15:41:37 +0100 Subject: [PATCH 418/427] review: remove websocket_base_url --- src/pipecat/services/openai.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index c631fd8c0..3f85d917c 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -503,7 +503,6 @@ class OpenAITTSService(TTSService): *, api_key: Optional[str] = None, base_url: Optional[str] = None, - websocket_base_url: Optional[str] = None, voice: str = "alloy", model: str = "gpt-4o-mini-tts", sample_rate: Optional[int] = None, @@ -520,9 +519,7 @@ class OpenAITTSService(TTSService): self.set_model_name(model) self.set_voice(voice) self._instructions = instructions - self._client = AsyncOpenAI( - api_key=api_key, base_url=base_url, websocket_base_url=websocket_base_url - ) + self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) def can_generate_metrics(self) -> bool: return True From e6e339a02ecf6afe7fca531be0670ae8170aa22f Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 26 Mar 2025 11:22:23 -0400 Subject: [PATCH 419/427] Bump daily-python dependency to 0.16.1 to pick up a bugfix --- CHANGELOG.md | 5 ++++- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e88155896..105e6d891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added a new frame, `LLMSetToolChoiceFrame`, which provides a mechanism +- Added a new frame, `LLMSetToolChoiceFrame`, which provides a mechanism for modifying the `tool_choice` in the context. - Added support in `DailyTransport` for updating remote participants' @@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue in Daily involving switching virtual devices, by bumping the + daily-python dependency to >= 0.16.1. + - Fixed a `GoogleAssistantContextAggregator` issue where function calls placeholders where not being updated when then function call result was different from a string. diff --git a/pyproject.toml b/pyproject.toml index d2fa46dcf..c3fbea3e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ] neuphonic = [ "pyneuphonic~=1.5.13", "websockets~=13.1" ] cerebras = [] deepseek = [] -daily = [ "daily-python~=0.16.0" ] +daily = [ "daily-python~=0.16.1" ] deepgram = [ "deepgram-sdk~=3.8.0" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.5.9" ] From 499e69846d0918d1b42bd295d4ab07aa67553e89 Mon Sep 17 00:00:00 2001 From: Nico <105345946+nicougou@users.noreply.github.com> Date: Wed, 26 Mar 2025 17:13:30 +0100 Subject: [PATCH 420/427] review: add changelog entries --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e88155896..fa272561d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ElevenLabs TTS services now support a sample rate of 8000. +- Added support for `instructions` in `OpenAITTSService` + +- Added support for `base_url` in `OpenAIImageGenService` and `OpenAITTSService` + ### Fixed - Fixed a `GoogleAssistantContextAggregator` issue where function calls From 060bb4c26bf616b372eab086bb58bfcbc14be231 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Tue, 25 Mar 2025 18:47:01 -0700 Subject: [PATCH 421/427] wip --- .../foundational/07y-interruptible-groq.py | 101 ++++++++++++++++++ pyproject.toml | 2 +- src/pipecat/services/groq.py | 78 +++++++++++++- 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 examples/foundational/07y-interruptible-groq.py diff --git a/examples/foundational/07y-interruptible-groq.py b/examples/foundational/07y-interruptible-groq.py new file mode 100644 index 000000000..48d0eb700 --- /dev/null +++ b/examples/foundational/07y-interruptible-groq.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.groq import GroqLLMService, GroqSTTService, GroqTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + # transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY")) + + llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") + + tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"), voice_id="Atlas-PlayAI") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_participant_left") + async def on_participant_left(transport, participant, reason): + await task.cancel() + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index d2fa46dcf..da9c18dd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] google = [ "google-cloud-speech~=2.31.1", "google-cloud-texttospeech~=2.25.1", "google-genai~=1.7.0", "google-generativeai~=0.8.4" ] grok = [] -groq = [] +groq = [ "groq~=0.20.0" ] gstreamer = [ "pygobject~=3.50.0" ] fireworks = [] krisp = [ "pipecat-ai-krisp~=0.3.0" ] diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index 66cc9357f..1b3570abb 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -5,10 +5,14 @@ # -from typing import Optional +from typing import AsyncGenerator, Optional +from groq import AsyncGroq from loguru import logger +from pydantic import BaseModel +from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame +from pipecat.services.ai_services import InterruptibleTTSService from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription from pipecat.services.openai import OpenAILLMService from pipecat.transcriptions.language import Language @@ -98,3 +102,75 @@ class GroqSTTService(BaseWhisperSTTService): kwargs["temperature"] = self._temperature return await self._client.audio.transcriptions.create(**kwargs) + + +class GroqTTSService(InterruptibleTTSService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + speed: Optional[float] = 1.0 + seed: Optional[int] = None + + def __init__( + self, + *, + api_key: str, + output_format: str = "wav", + params: InputParams = InputParams(), + model_name: str = "playai-tts", + voice_id: str = "Atlas-PlayAI", + **kwargs, + ): + super().__init__( + pause_frame_processing=True, + **kwargs, + ) + + self._api_key = api_key + self._model_name = model_name + self._output_format = output_format + self._voice_id = voice_id + self._params = params + + self._client = AsyncGroq(api_key=self._api_key) + + def can_generate_metrics(self) -> bool: + return True + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"{self}: Generating TTS [{text}]") + measuring_ttfb = True + await self.start_ttfb_metrics() + yield TTSStartedFrame() + + response = await self._client.audio.speech.create( + model=self._model_name, + voice=self._voice_id, + response_format=self._output_format, + input=text, + ) + + async for data in response.iter_bytes(4096): + if measuring_ttfb: + await self.stop_ttfb_metrics() + measuring_ttfb = False + # remove wav header if present + if data.startswith(b"RIFF"): + continue + yield TTSAudioRawFrame(data, 48000, 1) + + yield TTSStoppedFrame() + + async def _connect(self) -> None: + pass + + async def _disconnect(self) -> None: + pass + + async def _connect_websocket(self) -> None: + pass + + async def _disconnect_websocket(self) -> None: + pass + + async def _receive_messages(self) -> None: + pass From 406f5a395b56859727ae4df72c890ce584e4e3db Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Wed, 26 Mar 2025 07:54:57 -0700 Subject: [PATCH 422/427] fix class heirarchy and audio chunking --- .../foundational/07y-interruptible-groq.py | 2 +- src/pipecat/services/groq.py | 27 +++++-------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/examples/foundational/07y-interruptible-groq.py b/examples/foundational/07y-interruptible-groq.py index 48d0eb700..9e5719c21 100644 --- a/examples/foundational/07y-interruptible-groq.py +++ b/examples/foundational/07y-interruptible-groq.py @@ -48,7 +48,7 @@ async def main(): llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile") - tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"), voice_id="Atlas-PlayAI") + tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY")) messages = [ { diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index 1b3570abb..9345f9be1 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -12,7 +12,7 @@ from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import Frame, TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame -from pipecat.services.ai_services import InterruptibleTTSService +from pipecat.services.ai_services import TTSService from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription from pipecat.services.openai import OpenAILLMService from pipecat.transcriptions.language import Language @@ -104,7 +104,7 @@ class GroqSTTService(BaseWhisperSTTService): return await self._client.audio.transcriptions.create(**kwargs) -class GroqTTSService(InterruptibleTTSService): +class GroqTTSService(TTSService): class InputParams(BaseModel): language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -117,7 +117,7 @@ class GroqTTSService(InterruptibleTTSService): output_format: str = "wav", params: InputParams = InputParams(), model_name: str = "playai-tts", - voice_id: str = "Atlas-PlayAI", + voice_id: str = "Celeste-PlayAI", **kwargs, ): super().__init__( @@ -149,28 +149,15 @@ class GroqTTSService(InterruptibleTTSService): input=text, ) - async for data in response.iter_bytes(4096): + async for data in response.iter_bytes(): if measuring_ttfb: await self.stop_ttfb_metrics() measuring_ttfb = False # remove wav header if present if data.startswith(b"RIFF"): - continue + data = data[44:] + if len(data) == 0: + continue yield TTSAudioRawFrame(data, 48000, 1) yield TTSStoppedFrame() - - async def _connect(self) -> None: - pass - - async def _disconnect(self) -> None: - pass - - async def _connect_websocket(self) -> None: - pass - - async def _disconnect_websocket(self) -> None: - pass - - async def _receive_messages(self) -> None: - pass From e087f6ec5d7b5c7faaa99ea95709431de1180dd5 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Wed, 26 Mar 2025 08:57:42 -0700 Subject: [PATCH 423/427] GroqTTSService added to CHANGELOG.md --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e88155896..877387b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added a new frame, `LLMSetToolChoiceFrame`, which provides a mechanism +- Added a new frame, `LLMSetToolChoiceFrame`, which provides a mechanism for modifying the `tool_choice` in the context. +- Added `GroqTTSService` which provides text-to-speech functionality using + Groq's API. + - Added support in `DailyTransport` for updating remote participants' `canReceive` permission via the `update_remote_participants()` method, by bumping the daily-python dependency to >= 0.16.0. From f5d49fea8137067272da01af892872e9c316b32a Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Wed, 26 Mar 2025 09:13:03 -0700 Subject: [PATCH 424/427] try/catch import of groq SDK --- src/pipecat/services/groq.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index 9345f9be1..9d4b09455 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -7,7 +7,6 @@ from typing import AsyncGenerator, Optional -from groq import AsyncGroq from loguru import logger from pydantic import BaseModel @@ -17,6 +16,15 @@ from pipecat.services.base_whisper import BaseWhisperSTTService, Transcription from pipecat.services.openai import OpenAILLMService from pipecat.transcriptions.language import Language +try: + from groq import AsyncGroq +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Groq, you need to `pip install pipecat-ai[groq]`. Also, set a `GROQ_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + class GroqLLMService(OpenAILLMService): """A service for interacting with Groq's API using the OpenAI-compatible interface. From 887c197bce272f49c31034641d5f9b054dcfe9ce Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 26 Mar 2025 12:26:11 -0400 Subject: [PATCH 425/427] Add sample_rate to the constructor --- examples/foundational/07e-interruptible-playht.py | 2 +- src/pipecat/services/groq.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 5ccb96c15..c402c09db 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -48,7 +48,7 @@ async def main(): tts = PlayHTTTSService( user_id=os.getenv("PLAYHT_USER_ID"), api_key=os.getenv("PLAYHT_API_KEY"), - voice_url="s3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json", + voice_url="s3://voice-cloning-zero-shot/e46b4027-b38d-4d24-b292-38fbca2be0ef/original/manifest.json", params=PlayHTTTSService.InputParams(language=Language.EN), ) diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index 9d4b09455..bf0304df2 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -118,6 +118,8 @@ class GroqTTSService(TTSService): speed: Optional[float] = 1.0 seed: Optional[int] = None + GROQ_SAMPLE_RATE = 48000 # Groq TTS only supports 48kHz sample rate + def __init__( self, *, @@ -126,10 +128,14 @@ class GroqTTSService(TTSService): params: InputParams = InputParams(), model_name: str = "playai-tts", voice_id: str = "Celeste-PlayAI", + sample_rate: Optional[int] = GROQ_SAMPLE_RATE, **kwargs, ): + if sample_rate != self.GROQ_SAMPLE_RATE: + logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ") super().__init__( pause_frame_processing=True, + sample_rate=sample_rate, **kwargs, ) @@ -166,6 +172,6 @@ class GroqTTSService(TTSService): data = data[44:] if len(data) == 0: continue - yield TTSAudioRawFrame(data, 48000, 1) + yield TTSAudioRawFrame(data, self.sample_rate, 1) yield TTSStoppedFrame() From 4ef4dcefce6dc22ddff92a45d0eb3a793ce7fe9a Mon Sep 17 00:00:00 2001 From: Paul Kompfner Date: Wed, 26 Mar 2025 13:06:31 -0400 Subject: [PATCH 426/427] Update CHANGELOG for 0.0.61 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c521d0fcf..43c180778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.61] - 2025-03-26 ### Added From b414077a07c7c28aa43fd3d345c15af1e00fe594 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 26 Mar 2025 13:55:42 -0400 Subject: [PATCH 427/427] Fix: Resolve an issue where Google LLM context messages were causing a TypeError --- CHANGELOG.md | 4 ++++ src/pipecat/processors/frameworks/rtvi.py | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0504bd7ae..a8278cfdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue in `RTVIObserver` that prevented handling of Google LLM + context messages. The observer now processes both OpenAI-style and + Google-style contexts. + - Fixed an issue in Daily involving switching virtual devices, by bumping the daily-python dependency to >= 0.16.1. diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index f782e6ea8..eec07a29f 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -540,10 +540,23 @@ class RTVIObserver(BaseObserver): await self.push_transport_message_urgent(message) async def _handle_context(self, frame: OpenAILLMContextFrame): + """Process LLM context frames to extract user messages for the RTVI client.""" try: messages = frame.context.messages - if len(messages) > 0: - message = messages[-1] + if not messages: + return + + message = messages[-1] + + # Handle Google LLM format (protobuf objects with attributes) + if hasattr(message, "role") and message.role == "user" and hasattr(message, "parts"): + text = "".join(part.text for part in message.parts if hasattr(part, "text")) + if text: + rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text)) + await self.push_transport_message_urgent(rtvi_message) + + # Handle OpenAI format (original implementation) + elif isinstance(message, dict): if message["role"] == "user": content = message["content"] if isinstance(content, list): @@ -552,7 +565,8 @@ class RTVIObserver(BaseObserver): text = content rtvi_message = RTVIUserLLMTextMessage(data=RTVITextMessageData(text=text)) await self.push_transport_message_urgent(rtvi_message) - except TypeError as e: + + except Exception as e: logger.warning(f"Caught an error while trying to handle context: {e}") async def _handle_metrics(self, frame: MetricsFrame):