From 4232cca5b67cb5fad36e702c5922ff9a3fb03f9d Mon Sep 17 00:00:00 2001
From: Matt Kim <MaCaki@users.noreply.github.com>
Date: Wed, 30 Apr 2025 16:44:42 -0700
Subject: [PATCH 01/21] [Rime] Add phonemizeBetweenBrackets and
 pauseBetweenBrackets to RimeTTSService (ws)

There is a fix incoming in
---
 src/pipecat/services/rime/tts.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py
index e6d1d400a..57ade7484 100644
--- a/src/pipecat/services/rime/tts.py
+++ b/src/pipecat/services/rime/tts.py
@@ -68,6 +68,8 @@ class RimeTTSService(AudioContextWordTTSService):
         language: Optional[Language] = Language.EN
         speed_alpha: Optional[float] = 1.0
         reduce_latency: Optional[bool] = False
+        pause_between_brackets: Optional[bool] = False
+        phonemize_between_brackets: Optional[bool] = False
 
     def __init__(
         self,
@@ -117,6 +119,8 @@ class RimeTTSService(AudioContextWordTTSService):
             else "eng",
             "speedAlpha": params.speed_alpha,
             "reduceLatency": params.reduce_latency,
+            "pauseBetweenBrackets": json.dumps(params.pause_between_brackets),
+            "phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets),
         }
 
         # State tracking

From 2bb1b0b343bbb769d4617c034930bd772505356d Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 16:09:50 -0400
Subject: [PATCH 02/21] Add Changelog entry for PR 1707

---
 CHANGELOG.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 079142b9a..5954e6baf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,7 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added support for cross-platform local smart turn detection. You can use 
+- Added two new input parameters to `RimeTTSService`: `pause_between_brackets`
+  and `phonemize_between_brackets`.
+
+- Added support for cross-platform local smart turn detection. You can use
   `LocalSmartTurnAnalyzer` for on-device inference using Torch.
 
 - `BaseOutputTransport` now allows multiple destinations if the transport

From eeaa9f67a1b4ff8bc7f288a3b4359d25741df2e1 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 16:26:23 -0400
Subject: [PATCH 03/21] Fix: SimliVideoService was continuously emitting audio,
 preventing BotStoppedSpeakingFrame from being sent

---
 CHANGELOG.md                            |  3 +++
 examples/foundational/27-simli-layer.py |  1 +
 src/pipecat/services/simli/video.py     | 17 ++++++++++-------
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5954e6baf..baa07ba6f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -136,6 +136,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fixed an issue with `SimliVideoService` where the bot was continuously outputting
+  audio, which prevents the `BotStoppedSpeakingFrame` from being emitted.
+
 - Fixed an issue where `OpenAIRealtimeBetaLLMService` would add two assistant
   messages to the context.
 
diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py
index 513bbc239..38721e50e 100644
--- a/examples/foundational/27-simli-layer.py
+++ b/examples/foundational/27-simli-layer.py
@@ -36,6 +36,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
             audio_in_enabled=True,
             audio_out_enabled=True,
             video_out_enabled=True,
+            video_out_is_live=True,
             video_out_width=512,
             video_out_height=512,
             vad_analyzer=SileroVADAnalyzer(),
diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py
index 0fb490914..121801e4b 100644
--- a/src/pipecat/services/simli/video.py
+++ b/src/pipecat/services/simli/video.py
@@ -64,13 +64,16 @@ class SimliVideoService(FrameProcessor):
         async for audio_frame in self._simli_client.getAudioStreamIterator():
             resampled_frames = self._pipecat_resampler.resample(audio_frame)
             for resampled_frame in resampled_frames:
-                await self.push_frame(
-                    TTSAudioRawFrame(
-                        audio=resampled_frame.to_ndarray().tobytes(),
-                        sample_rate=self._pipecat_resampler.rate,
-                        num_channels=1,
-                    ),
-                )
+                audio_array = resampled_frame.to_ndarray()
+                # Only push frame is there is audio (e.g. not silence)
+                if audio_array.any():
+                    await self.push_frame(
+                        TTSAudioRawFrame(
+                            audio=audio_array.tobytes(),
+                            sample_rate=self._pipecat_resampler.rate,
+                            num_channels=1,
+                        ),
+                    )
 
     async def _consume_and_process_video(self):
         await self._pipecat_resampler_event.wait()

From 93c40b87dcd12fc5a47d3db3b5246e957bdcacc9 Mon Sep 17 00:00:00 2001
From: Kwindla Hultman Kramer <kwindla@daily.co>
Date: Thu, 1 May 2025 17:07:00 -0700
Subject: [PATCH 04/21] small groq updates

---
 examples/foundational/07l-interruptible-groq.py    | 9 +++++++--
 examples/foundational/14f-function-calling-groq.py | 9 +++++++--
 pyproject.toml                                     | 2 +-
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/examples/foundational/07l-interruptible-groq.py b/examples/foundational/07l-interruptible-groq.py
index b6826d49c..869548274 100644
--- a/examples/foundational/07l-interruptible-groq.py
+++ b/examples/foundational/07l-interruptible-groq.py
@@ -14,6 +14,7 @@ from pipecat.audio.vad.silero import SileroVADAnalyzer
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.groq.llm import GroqLLMService
 from pipecat.services.groq.stt import GroqSTTService
@@ -39,7 +40,9 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
 
     stt = GroqSTTService(api_key=os.getenv("GROQ_API_KEY"))
 
-    llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile")
+    llm = GroqLLMService(
+        api_key=os.getenv("GROQ_API_KEY"), model="meta-llama/llama-4-maverick-17b-128e-instruct"
+    )
 
     tts = GroqTTSService(api_key=os.getenv("GROQ_API_KEY"))
 
@@ -51,7 +54,9 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
     ]
 
     context = OpenAILLMContext(messages)
-    context_aggregator = llm.create_context_aggregator(context)
+    context_aggregator = llm.create_context_aggregator(
+        context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05)
+    )
 
     pipeline = Pipeline(
         [
diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py
index f71a43fa2..ee6a9a855 100644
--- a/examples/foundational/14f-function-calling-groq.py
+++ b/examples/foundational/14f-function-calling-groq.py
@@ -17,6 +17,7 @@ from pipecat.frames.frames import TTSSpeakFrame
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_response import LLMUserAggregatorParams
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.cartesia.tts import CartesiaTTSService
 from pipecat.services.groq.llm import GroqLLMService
@@ -53,7 +54,9 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
         voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
     )
 
-    llm = GroqLLMService(api_key=os.getenv("GROQ_API_KEY"), model="llama-3.3-70b-versatile")
+    llm = GroqLLMService(
+        api_key=os.getenv("GROQ_API_KEY"), model="meta-llama/llama-4-maverick-17b-128e-instruct"
+    )
     # You can also register a function_name of None to get all functions
     # sent to the same callback with an additional function_name parameter.
     llm.register_function("get_current_weather", fetch_weather_from_api)
@@ -83,7 +86,9 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
     ]
 
     context = OpenAILLMContext(messages, tools)
-    context_aggregator = llm.create_context_aggregator(context)
+    context_aggregator = llm.create_context_aggregator(
+        context, user_params=LLMUserAggregatorParams(aggregation_timeout=0.05)
+    )
 
     pipeline = Pipeline(
         [
diff --git a/pyproject.toml b/pyproject.toml
index f7db97ea0..6d1cf840c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,7 +56,7 @@ fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ]
 gladia = [ "websockets~=13.1" ]
 google = [ "google-cloud-speech~=2.31.1", "google-cloud-texttospeech~=2.25.1", "google-genai~=1.7.0", "google-generativeai~=0.8.4", "websockets~=13.1" ]
 grok = []
-groq = [ "groq~=0.20.0" ]
+groq = [ "groq~=0.23.0" ]
 gstreamer = [ "pygobject~=3.50.0" ]
 krisp = [ "pipecat-ai-krisp~=0.3.0" ]
 koala = [ "pvkoala~=2.0.3" ]

From 15cbd18acce55771967d642bd6d4bf48e53a9b0e Mon Sep 17 00:00:00 2001
From: Matt Kim <MaCaki@users.noreply.github.com>
Date: Wed, 30 Apr 2025 16:44:42 -0700
Subject: [PATCH 05/21] [Rime] Add phonemizeBetweenBrackets and
 pauseBetweenBrackets to RimeTTSService (ws)

There is a fix incoming in
---
 src/pipecat/services/rime/tts.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py
index e6d1d400a..57ade7484 100644
--- a/src/pipecat/services/rime/tts.py
+++ b/src/pipecat/services/rime/tts.py
@@ -68,6 +68,8 @@ class RimeTTSService(AudioContextWordTTSService):
         language: Optional[Language] = Language.EN
         speed_alpha: Optional[float] = 1.0
         reduce_latency: Optional[bool] = False
+        pause_between_brackets: Optional[bool] = False
+        phonemize_between_brackets: Optional[bool] = False
 
     def __init__(
         self,
@@ -117,6 +119,8 @@ class RimeTTSService(AudioContextWordTTSService):
             else "eng",
             "speedAlpha": params.speed_alpha,
             "reduceLatency": params.reduce_latency,
+            "pauseBetweenBrackets": json.dumps(params.pause_between_brackets),
+            "phonemizeBetweenBrackets": json.dumps(params.phonemize_between_brackets),
         }
 
         # State tracking

From 02c07755b0c5bbb922a03ae057d621bea4e732df Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 16:09:50 -0400
Subject: [PATCH 06/21] Add Changelog entry for PR 1707

---
 CHANGELOG.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 079142b9a..5954e6baf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,7 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
-- Added support for cross-platform local smart turn detection. You can use 
+- Added two new input parameters to `RimeTTSService`: `pause_between_brackets`
+  and `phonemize_between_brackets`.
+
+- Added support for cross-platform local smart turn detection. You can use
   `LocalSmartTurnAnalyzer` for on-device inference using Torch.
 
 - `BaseOutputTransport` now allows multiple destinations if the transport

From 63a65627a29ad29dfd554ea3139977b4eb10f511 Mon Sep 17 00:00:00 2001
From: vipyne <vipyne@gmail.com>
Date: Wed, 30 Apr 2025 11:25:10 -0500
Subject: [PATCH 07/21] Riva Service: add magpie-tts-multilingual model

---
 .../07r-interruptible-riva-nim.py             |   8 +-
 src/pipecat/services/riva/stt.py              | 204 +++++++++++++++++-
 src/pipecat/services/riva/tts.py              |  54 ++++-
 3 files changed, 251 insertions(+), 15 deletions(-)

diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py
index 915beda51..ca74dfd7a 100644
--- a/examples/foundational/07r-interruptible-riva-nim.py
+++ b/examples/foundational/07r-interruptible-riva-nim.py
@@ -16,8 +16,8 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.nim.llm import NimLLMService
-from pipecat.services.riva.stt import ParakeetSTTService
-from pipecat.services.riva.tts import FastPitchTTSService
+from pipecat.services.riva.stt import ParakeetSTTService, RivaOfflineSTTService
+from pipecat.services.riva.tts import RivaTTSService, FastPitchTTSService
 from pipecat.transports.base_transport import TransportParams
 from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
 from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
@@ -37,11 +37,13 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
         ),
     )
 
-    stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    stt = RivaOfflineSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    # stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
 
     tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
+    # tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     messages = [
         {
diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py
index 6328bcb65..3676befa3 100644
--- a/src/pipecat/services/riva/stt.py
+++ b/src/pipecat/services/riva/stt.py
@@ -13,11 +13,13 @@ from pydantic import BaseModel
 from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
+    ErrorFrame,
     Frame,
     InterimTranscriptionFrame,
     StartFrame,
     TranscriptionFrame,
 )
+from pipecat.services.stt_service import SegmentedSTTService
 from pipecat.services.stt_service import STTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
@@ -27,20 +29,21 @@ try:
 
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
-    logger.error("In order to use NVIDIA Riva STT, you need to `pip install pipecat-ai[riva]`.")
+    logger.error("In order to use NVIDIA Riva STT, you need to `pip install pipecat-ai[riva]`. Also set NVIDIA_API_KEY env var.")
     raise Exception(f"Missing module: {e}")
 
 
-class ParakeetSTTService(STTService):
+class RivaSTTService(STTService):
     class InputParams(BaseModel):
         language: Optional[Language] = Language.EN_US
 
     def __init__(
         self,
         *,
-        api_key: str,
+        api_key: str = None,
         server: str = "grpc.nvcf.nvidia.com:443",
         function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
+        model_name: str = "parakeet-ctc-1.1b-asr",
         sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
@@ -61,7 +64,7 @@ class ParakeetSTTService(STTService):
         self._stop_threshold_eou = -1.0
         self._custom_configuration = ""
 
-        self.set_model_name("parakeet-ctc-1.1b-asr")
+        self.set_model_name(model_name)
 
         metadata = [
             ["function-id", function_id],
@@ -196,3 +199,196 @@ class ParakeetSTTService(STTService):
 
     def __iter__(self):
         return self
+
+class RivaOfflineSTTService(SegmentedSTTService):
+    """Speech-to-text service using Fal's Wizper API.
+
+    This service uses Fal's Wizper API to perform speech-to-text transcription on audio
+    segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
+
+    Args:
+        api_key: NVIDIA_API_KEY.
+        sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
+        params: Configuration parameters for Riva.
+        **kwargs: Additional arguments passed to SegmentedSTTService.
+    """
+
+    class InputParams(BaseModel):
+        """Configuration parameters for Fal's Wizper API.
+
+        Attributes:
+            language: Language of the audio input. Defaults to English.
+            task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'.
+            chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
+            version: Version of Wizper model to use. Defaults to '3'.
+        """
+
+        language: Optional[Language] = Language.EN
+        task: str = "transcribe"
+        chunk_level: str = "segment"
+        version: str = "3"
+
+    def __init__(
+        self,
+        *,
+        api_key: str = None,
+        server: str = "grpc.nvcf.nvidia.com:443",
+        function_id: str = "ee8dc628-76de-4acc-8595-1836e7e857bd",
+        model_name: str = "canary-1b-asr",
+        sample_rate: Optional[int] = None,
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
+        super().__init__(sample_rate=sample_rate, **kwargs)
+        self._api_key = api_key
+        self._profanity_filter = False
+        self._automatic_punctuation = False
+        self._no_verbatim_transcripts = False
+        self._language_code = params.language
+        self._boosted_lm_words = None
+        self._boosted_lm_score = 4.0
+        self._start_history = -1
+        self._start_threshold = -1.0
+        self._stop_history = -1
+        self._stop_threshold = -1.0
+        self._stop_history_eou = -1
+        self._stop_threshold_eou = -1.0
+        self._custom_configuration = ""
+
+        self.set_model_name(model_name)
+
+        metadata = [
+            ["function-id", function_id],
+            ["authorization", f"Bearer {api_key}"],
+        ]
+        auth = riva.client.Auth(None, True, server, metadata)
+
+        self._asr_service = riva.client.ASRService(auth)
+
+        self._queue = asyncio.Queue()
+        self._config = None
+        self._thread_task = None
+        self._response_task = None
+
+    def can_generate_metrics(self) -> bool:
+        return False
+
+    async def start(self, frame: StartFrame):
+        await super().start(frame)
+
+        if self._config:
+            return
+
+        # config = riva.client.StreamingRecognitionConfig(
+        config=riva.client.RecognitionConfig(
+            # encoding=riva.client.AudioEncoding.LINEAR_PCM,
+            language_code=self._language_code,
+            # model="",
+            max_alternatives=1,
+            profanity_filter=self._profanity_filter,
+            enable_automatic_punctuation=self._automatic_punctuation,
+            verbatim_transcripts=not self._no_verbatim_transcripts,
+                # sample_rate_hertz=self.sample_rate,
+                # audio_channel_count=1,
+                # enable_word_time_offsets=args.word_time_offsets or args.speaker_diarization,??
+            # ),
+            # interim_results=True,
+        )
+
+        riva.client.add_word_boosting_to_config(
+            config, self._boosted_lm_words, self._boosted_lm_score
+        )
+
+        riva.client.add_endpoint_parameters_to_config(
+            config,
+            self._start_history,
+            self._start_threshold,
+            self._stop_history,
+            self._stop_history_eou,
+            self._stop_threshold,
+            self._stop_threshold_eou,
+        )
+        riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
+
+        self._config = config
+
+
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Transcribe an audio segment
+
+        Args:
+            audio: Raw audio bytes in WAV format (already converted by base class).
+
+        Yields:
+            Frame: TranscriptionFrame containing the transcribed text.
+
+        Note:
+            The audio is already in WAV format from the SegmentedSTTService.
+            Only non-empty transcriptions are yielded.
+        """
+        try:
+            response = self._asr_service.offline_recognize(audio, self._config)
+            # response = riva.client.print_offline(response=self._asr_service.offline_recognize(audio, self._config))
+            print(f"_____stt.py * response: {response}")
+            # # Send to Fal directly (audio is already in WAV format from base class)
+            # data_uri = fal_client.encode(audio, "audio/x-wav")
+            # response = await self._fal_client.run(
+            #     "fal-ai/wizper",
+            #     arguments={"audio_url": data_uri, **self._settings},
+            # )
+
+            if response and "text" in response:
+                text = response["text"].strip()
+                if text:  # Only yield non-empty text
+                    logger.debug(f"Transcription: [{text}]")
+                    yield TranscriptionFrame(
+                        text, "", time_now_iso8601(), Language(self._settings["language"])
+                    )
+
+        except Exception as e:
+            logger.error(f"Riva Offline STT error: {e}")
+            yield ErrorFrame(f"Riva Offline STT error: {str(e)}")
+
+    def __next__(self) -> bytes:
+        if not self._thread_running:
+            raise StopIteration
+        future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop())
+        return future.result()
+
+    def __iter__(self):
+        return self
+
+class ParakeetSTTService(RivaSTTService):
+    class InputParams(BaseModel):
+        language: Optional[Language] = Language.EN_US
+
+    def __init__(
+        self,
+        *,
+        api_key: str = None,
+        server: str = "grpc.nvcf.nvidia.com:443",
+        function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
+        model_name: str = "parakeet-ctc-1.1b-asr",
+        sample_rate: Optional[int] = None,
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
+        super().__init__(
+            api_key=api_key,
+            server=server,
+            function_id=function_id,
+            model_name=model_name,
+            sample_rate=sample_rate,
+            params=params,
+            **kwargs,
+        )
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "`ParakeetSTTService` is deprecated, use `RivaSTTService` instead.",
+                DeprecationWarning,
+            )
+
diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py
index 0fd0c3ce0..f37064acb 100644
--- a/src/pipecat/services/riva/tts.py
+++ b/src/pipecat/services/riva/tts.py
@@ -27,10 +27,10 @@ except ModuleNotFoundError as e:
     logger.error("In order to use NVIDIA Riva TTS, you need to `pip install pipecat-ai[riva]`.")
     raise Exception(f"Missing module: {e}")
 
-FASTPITCH_TIMEOUT_SECS = 5
+RIVA_TTS_TIMEOUT_SECS = 5
 
 
-class FastPitchTTSService(TTSService):
+class RivaTTSService(TTSService):
     class InputParams(BaseModel):
         language: Optional[Language] = Language.EN_US
         quality: Optional[int] = 20
@@ -38,11 +38,12 @@ class FastPitchTTSService(TTSService):
     def __init__(
         self,
         *,
-        api_key: str,
+        api_key: str = None,
         server: str = "grpc.nvcf.nvidia.com:443",
-        voice_id: str = "English-US.Female-1",
+        voice_id: str = "Magpie-Multilingual.EN-US.Male.Male-1",
         sample_rate: Optional[int] = None,
-        function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
+        function_id: str = "877104f7-e885-42b9-8de8-f6e4c6303969",
+        model_name: str = "magpie-tts-multilingual",
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -52,7 +53,7 @@ class FastPitchTTSService(TTSService):
         self._language_code = params.language
         self._quality = params.quality
 
-        self.set_model_name("fastpitch-hifigan-tts")
+        self.set_model_name(model_name)
         self.set_voice(voice_id)
 
         metadata = [
@@ -100,7 +101,7 @@ class FastPitchTTSService(TTSService):
             await asyncio.to_thread(read_audio_responses, queue)
 
             # Wait for the thread to start.
-            resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS)
+            resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS)
             while resp:
                 await self.stop_ttfb_metrics()
                 frame = TTSAudioRawFrame(
@@ -109,9 +110,46 @@ class FastPitchTTSService(TTSService):
                     num_channels=1,
                 )
                 yield frame
-                resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS)
+                resp = await asyncio.wait_for(queue.get(), RIVA_TTS_TIMEOUT_SECS)
         except asyncio.TimeoutError:
             logger.error(f"{self} timeout waiting for audio response")
 
         await self.start_tts_usage_metrics(text)
         yield TTSStoppedFrame()
+
+
+class FastPitchTTSService(RivaTTSService):
+    class InputParams(BaseModel):
+        language: Optional[Language] = Language.EN_US
+        quality: Optional[int] = 20
+
+    def __init__(
+        self,
+        *,
+        api_key: str = None,
+        server: str = "grpc.nvcf.nvidia.com:443",
+        voice_id: str = "English-US.Female-1",
+        sample_rate: Optional[int] = None,
+        function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
+        model_name: str = "fastpitch-hifigan-tts",
+        params: InputParams = InputParams(),
+        **kwargs,
+    ):
+        super().__init__(
+            api_key=api_key,
+            voice_id=voice_id,
+            sample_rate=sample_rate,
+            function_id=function_id,
+            model_name=model_name,
+            params=params,
+            **kwargs,
+        )
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "`FastPitchTTSService` is deprecated, use `RivaTTSService` instead.",
+                DeprecationWarning,
+            )
+

From daf9d47e58269a9f1338415843294462ed7eaf07 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 09:52:23 -0400
Subject: [PATCH 08/21] Update RivaSegmentedSTTService

---
 .../07r-interruptible-riva-nim.py             |  10 +-
 src/pipecat/services/riva/stt.py              | 306 ++++++++++++------
 src/pipecat/services/riva/tts.py              |   5 +-
 3 files changed, 219 insertions(+), 102 deletions(-)

diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py
index ca74dfd7a..070467844 100644
--- a/examples/foundational/07r-interruptible-riva-nim.py
+++ b/examples/foundational/07r-interruptible-riva-nim.py
@@ -16,8 +16,12 @@ from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
 from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
 from pipecat.services.nim.llm import NimLLMService
-from pipecat.services.riva.stt import ParakeetSTTService, RivaOfflineSTTService
-from pipecat.services.riva.tts import RivaTTSService, FastPitchTTSService
+from pipecat.services.riva.stt import (
+    ParakeetSTTService,
+    RivaSegmentedSTTService,
+    RivaSTTService,
+)
+from pipecat.services.riva.tts import FastPitchTTSService, RivaTTSService
 from pipecat.transports.base_transport import TransportParams
 from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
 from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
@@ -37,7 +41,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
         ),
     )
 
-    stt = RivaOfflineSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    stt = RivaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
     # stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py
index 3676befa3..d3b826060 100644
--- a/src/pipecat/services/riva/stt.py
+++ b/src/pipecat/services/riva/stt.py
@@ -5,7 +5,7 @@
 #
 
 import asyncio
-from typing import AsyncGenerator, Optional
+from typing import AsyncGenerator, List, Optional
 
 from loguru import logger
 from pydantic import BaseModel
@@ -19,8 +19,7 @@ from pipecat.frames.frames import (
     StartFrame,
     TranscriptionFrame,
 )
-from pipecat.services.stt_service import SegmentedSTTService
-from pipecat.services.stt_service import STTService
+from pipecat.services.stt_service import SegmentedSTTService, STTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
 
@@ -29,10 +28,62 @@ try:
 
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
-    logger.error("In order to use NVIDIA Riva STT, you need to `pip install pipecat-ai[riva]`. Also set NVIDIA_API_KEY env var.")
+    logger.error("In order to use NVIDIA Riva STT, you need to `pip install pipecat-ai[riva]`.")
     raise Exception(f"Missing module: {e}")
 
 
+def language_to_riva_language(language: Language) -> Optional[str]:
+    """Maps Language enum to Riva ASR language codes.
+
+    Source:
+    https://docs.nvidia.com/deeplearning/riva/user-guide/docs/asr/asr-riva-build-table.html?highlight=fr%20fr
+
+    Args:
+        language: Language enum value.
+
+    Returns:
+        Optional[str]: Riva language code or None if not supported.
+    """
+    language_map = {
+        # Arabic
+        Language.AR: "ar-AR",
+        # English
+        Language.EN: "en-US",  # Default to US
+        Language.EN_US: "en-US",
+        Language.EN_GB: "en-GB",
+        # French
+        Language.FR: "fr-FR",
+        Language.FR_FR: "fr-FR",
+        # German
+        Language.DE: "de-DE",
+        Language.DE_DE: "de-DE",
+        # Hindi
+        Language.HI: "hi-IN",
+        Language.HI_IN: "hi-IN",
+        # Italian
+        Language.IT: "it-IT",
+        Language.IT_IT: "it-IT",
+        # Japanese
+        Language.JA: "ja-JP",
+        Language.JA_JP: "ja-JP",
+        # Korean
+        Language.KO: "ko-KR",
+        Language.KO_KR: "ko-KR",
+        # Portuguese
+        Language.PT: "pt-BR",  # Default to Brazilian
+        Language.PT_BR: "pt-BR",
+        # Russian
+        Language.RU: "ru-RU",
+        Language.RU_RU: "ru-RU",
+        # Spanish
+        Language.ES: "es-ES",  # Default to Spain
+        Language.ES_ES: "es-ES",
+        Language.ES_US: "es-US",  # US Spanish
+    }
+
+    return language_map.get(language)
+
+
 class RivaSTTService(STTService):
     class InputParams(BaseModel):
         language: Optional[Language] = Language.EN_US
@@ -200,38 +251,38 @@ class RivaSTTService(STTService):
     def __iter__(self):
         return self
 
-class RivaOfflineSTTService(SegmentedSTTService):
-    """Speech-to-text service using Fal's Wizper API.
 
-    This service uses Fal's Wizper API to perform speech-to-text transcription on audio
-    segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
+class RivaSegmentedSTTService(SegmentedSTTService):
+    """Speech-to-text service using NVIDIA Riva Canary ASR API.
+
+    This service uses NVIDIA's Riva Canary ASR API to perform speech-to-text
+    transcription on audio segments. It inherits from SegmentedSTTService to handle
+    audio buffering and speech detection.
 
     Args:
-        api_key: NVIDIA_API_KEY.
-        sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
-        params: Configuration parameters for Riva.
-        **kwargs: Additional arguments passed to SegmentedSTTService.
+        api_key: NVIDIA API key for authentication
+        server: Riva server address (defaults to NVIDIA Cloud Function endpoint)
+        function_id: NVIDIA Cloud Function ID for the Canary ASR service
+        model_name: Name of the Canary ASR model to use
+        sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate
+        params: Additional configuration parameters for Riva
+        **kwargs: Additional arguments passed to SegmentedSTTService
     """
 
     class InputParams(BaseModel):
-        """Configuration parameters for Fal's Wizper API.
+        """Configuration parameters for Riva Canary ASR API."""
 
-        Attributes:
-            language: Language of the audio input. Defaults to English.
-            task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'.
-            chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
-            version: Version of Wizper model to use. Defaults to '3'.
-        """
-
-        language: Optional[Language] = Language.EN
-        task: str = "transcribe"
-        chunk_level: str = "segment"
-        version: str = "3"
+        language: Optional[Language] = Language.EN_US
+        profanity_filter: bool = False
+        automatic_punctuation: bool = True
+        verbatim_transcripts: bool = False
+        boosted_lm_words: Optional[List[str]] = None
+        boosted_lm_score: float = 4.0
 
     def __init__(
         self,
         *,
-        api_key: str = None,
+        api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
         function_id: str = "ee8dc628-76de-4acc-8595-1836e7e857bd",
         model_name: str = "canary-1b-asr",
@@ -240,13 +291,27 @@ class RivaOfflineSTTService(SegmentedSTTService):
         **kwargs,
     ):
         super().__init__(sample_rate=sample_rate, **kwargs)
+
+        # Set model name
+        self.set_model_name(model_name)
+
+        # Initialize Riva settings
         self._api_key = api_key
-        self._profanity_filter = False
-        self._automatic_punctuation = False
-        self._no_verbatim_transcripts = False
-        self._language_code = params.language
-        self._boosted_lm_words = None
-        self._boosted_lm_score = 4.0
+        self._server = server
+        self._function_id = function_id
+
+        # Store the language as a Language enum and as a string
+        self._language_enum = params.language or Language.EN_US
+        self._language = self.language_to_service_language(self._language_enum) or "en-US"
+
+        # Configure transcription parameters
+        self._profanity_filter = params.profanity_filter
+        self._automatic_punctuation = params.automatic_punctuation
+        self._verbatim_transcripts = params.verbatim_transcripts
+        self._boosted_lm_words = params.boosted_lm_words
+        self._boosted_lm_score = params.boosted_lm_score
+
+        # Voice activity detection thresholds (use Riva defaults)
         self._start_history = -1
         self._start_threshold = -1.0
         self._stop_history = -1
@@ -255,50 +320,50 @@ class RivaOfflineSTTService(SegmentedSTTService):
         self._stop_threshold_eou = -1.0
         self._custom_configuration = ""
 
-        self.set_model_name(model_name)
-
-        metadata = [
-            ["function-id", function_id],
-            ["authorization", f"Bearer {api_key}"],
-        ]
-        auth = riva.client.Auth(None, True, server, metadata)
-
-        self._asr_service = riva.client.ASRService(auth)
-
-        self._queue = asyncio.Queue()
+        # Create Riva client
         self._config = None
-        self._thread_task = None
-        self._response_task = None
+        self._asr_service = None
+        self._settings = {"language": self._language_enum}
 
-    def can_generate_metrics(self) -> bool:
-        return False
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert pipecat Language enum to Riva's language code."""
+        return language_to_riva_language(language)
 
-    async def start(self, frame: StartFrame):
-        await super().start(frame)
-
-        if self._config:
+    def _initialize_client(self):
+        """Initialize the Riva ASR client with authentication metadata."""
+        if self._asr_service is not None:
             return
 
-        # config = riva.client.StreamingRecognitionConfig(
-        config=riva.client.RecognitionConfig(
-            # encoding=riva.client.AudioEncoding.LINEAR_PCM,
-            language_code=self._language_code,
-            # model="",
+        # Set up authentication metadata for NVIDIA Cloud Functions
+        metadata = [
+            ["function-id", self._function_id],
+            ["authorization", f"Bearer {self._api_key}"],
+        ]
+
+        # Create authenticated client
+        auth = riva.client.Auth(None, True, self._server, metadata)
+        self._asr_service = riva.client.ASRService(auth)
+
+        logger.info(f"Initialized Riva Canary ASR service with model: {self.model_name}")
+
+    def _create_recognition_config(self):
+        """Create the Riva ASR recognition configuration."""
+        # Create base configuration
+        config = riva.client.RecognitionConfig(
+            language_code=self._language,  # Now using the string, not a tuple
             max_alternatives=1,
             profanity_filter=self._profanity_filter,
             enable_automatic_punctuation=self._automatic_punctuation,
-            verbatim_transcripts=not self._no_verbatim_transcripts,
-                # sample_rate_hertz=self.sample_rate,
-                # audio_channel_count=1,
-                # enable_word_time_offsets=args.word_time_offsets or args.speaker_diarization,??
-            # ),
-            # interim_results=True,
+            verbatim_transcripts=self._verbatim_transcripts,
         )
 
-        riva.client.add_word_boosting_to_config(
-            config, self._boosted_lm_words, self._boosted_lm_score
-        )
+        # Add word boosting if specified
+        if self._boosted_lm_words:
+            riva.client.add_word_boosting_to_config(
+                config, self._boosted_lm_words, self._boosted_lm_score
+            )
 
+        # Add voice activity detection parameters
         riva.client.add_endpoint_parameters_to_config(
             config,
             self._start_history,
@@ -308,56 +373,102 @@ class RivaOfflineSTTService(SegmentedSTTService):
             self._stop_threshold,
             self._stop_threshold_eou,
         )
-        riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
 
-        self._config = config
+        # Add any custom configuration
+        if self._custom_configuration:
+            riva.client.add_custom_configuration_to_config(config, self._custom_configuration)
 
+        return config
 
+    def can_generate_metrics(self) -> bool:
+        """Indicates whether this service can generate processing metrics."""
+        return True
+
+    async def start(self, frame: StartFrame):
+        """Initialize the service when the pipeline starts."""
+        await super().start(frame)
+        self._initialize_client()
+        self._config = self._create_recognition_config()
+
+    async def set_language(self, language: Language):
+        """Set the language for the STT service."""
+        logger.info(f"Switching STT language to: [{language}]")
+        self._language_enum = language
+        self._language = self.language_to_service_language(language) or "en-US"
+        self._settings["language"] = language
+
+        # Update configuration with new language
+        if self._config:
+            self._config.language_code = self._language
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        """Transcribe an audio segment
+        """Transcribe an audio segment using Riva Canary ASR.
 
         Args:
             audio: Raw audio bytes in WAV format (already converted by base class).
 
         Yields:
             Frame: TranscriptionFrame containing the transcribed text.
-
-        Note:
-            The audio is already in WAV format from the SegmentedSTTService.
-            Only non-empty transcriptions are yielded.
         """
         try:
-            response = self._asr_service.offline_recognize(audio, self._config)
-            # response = riva.client.print_offline(response=self._asr_service.offline_recognize(audio, self._config))
-            print(f"_____stt.py * response: {response}")
-            # # Send to Fal directly (audio is already in WAV format from base class)
-            # data_uri = fal_client.encode(audio, "audio/x-wav")
-            # response = await self._fal_client.run(
-            #     "fal-ai/wizper",
-            #     arguments={"audio_url": data_uri, **self._settings},
-            # )
+            await self.start_processing_metrics()
+            await self.start_ttfb_metrics()
 
-            if response and "text" in response:
-                text = response["text"].strip()
-                if text:  # Only yield non-empty text
-                    logger.debug(f"Transcription: [{text}]")
-                    yield TranscriptionFrame(
-                        text, "", time_now_iso8601(), Language(self._settings["language"])
-                    )
+            # Make sure the client is initialized
+            if self._asr_service is None:
+                self._initialize_client()
+
+            # Make sure the config is created
+            if self._config is None:
+                self._config = self._create_recognition_config()
+
+            # Type assertion to satisfy the IDE
+            assert self._asr_service is not None, "ASR service not initialized"
+            assert self._config is not None, "Recognition config not created"
+
+            # Process audio with Riva ASR - explicitly request non-future response
+            raw_response = self._asr_service.offline_recognize(audio, self._config, future=False)
+
+            await self.stop_ttfb_metrics()
+            await self.stop_processing_metrics()
+
+            # Process the response - handle different possible return types
+            try:
+                # If it's a future-like object, get the result
+                if hasattr(raw_response, "result"):
+                    response = raw_response.result()
+                else:
+                    response = raw_response
+
+                # Process transcription results
+                transcription_found = False
+
+                # Now we can safely check results
+                # Type hint for the IDE
+                results = getattr(response, "results", [])
+
+                for result in results:
+                    alternatives = getattr(result, "alternatives", [])
+                    if alternatives:
+                        text = alternatives[0].transcript.strip()
+                        if text:
+                            logger.debug(f"Transcription: [{text}]")
+                            yield TranscriptionFrame(
+                                text, "", time_now_iso8601(), self._language_enum
+                            )
+                            transcription_found = True
+
+                if not transcription_found:
+                    logger.debug("No transcription results found in Riva response")
+
+            except AttributeError as ae:
+                logger.error(f"Unexpected response structure from Riva: {ae}")
+                yield ErrorFrame(f"Unexpected Riva response format: {str(ae)}")
 
         except Exception as e:
-            logger.error(f"Riva Offline STT error: {e}")
-            yield ErrorFrame(f"Riva Offline STT error: {str(e)}")
+            logger.exception(f"Riva Canary ASR error: {e}")
+            yield ErrorFrame(f"Riva Canary ASR error: {str(e)}")
 
-    def __next__(self) -> bytes:
-        if not self._thread_running:
-            raise StopIteration
-        future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop())
-        return future.result()
-
-    def __iter__(self):
-        return self
 
 class ParakeetSTTService(RivaSTTService):
     class InputParams(BaseModel):
@@ -391,4 +502,3 @@ class ParakeetSTTService(RivaSTTService):
                 "`ParakeetSTTService` is deprecated, use `RivaSTTService` instead.",
                 DeprecationWarning,
             )
-
diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py
index f37064acb..bfa732678 100644
--- a/src/pipecat/services/riva/tts.py
+++ b/src/pipecat/services/riva/tts.py
@@ -5,8 +5,12 @@
 #
 
 import asyncio
+import os
 from typing import AsyncGenerator, Optional
 
+# Suppress gRPC fork warnings
+os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
+
 from loguru import logger
 from pydantic import BaseModel
 
@@ -152,4 +156,3 @@ class FastPitchTTSService(RivaTTSService):
                 "`FastPitchTTSService` is deprecated, use `RivaTTSService` instead.",
                 DeprecationWarning,
             )
-

From 725ab5ec219888e42f5d090a2fb976f3ccce1099 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 09:57:39 -0400
Subject: [PATCH 09/21] Small fixes: No default api_key of None,
 ParakeetSTTService uses RivaSTTService.InputParams

---
 src/pipecat/services/riva/stt.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py
index d3b826060..0ee7b3e5e 100644
--- a/src/pipecat/services/riva/stt.py
+++ b/src/pipecat/services/riva/stt.py
@@ -91,7 +91,7 @@ class RivaSTTService(STTService):
     def __init__(
         self,
         *,
-        api_key: str = None,
+        api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
         function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
         model_name: str = "parakeet-ctc-1.1b-asr",
@@ -102,7 +102,7 @@ class RivaSTTService(STTService):
         super().__init__(sample_rate=sample_rate, **kwargs)
         self._api_key = api_key
         self._profanity_filter = False
-        self._automatic_punctuation = False
+        self._automatic_punctuation = True
         self._no_verbatim_transcripts = False
         self._language_code = params.language
         self._boosted_lm_words = None
@@ -471,18 +471,17 @@ class RivaSegmentedSTTService(SegmentedSTTService):
 
 
 class ParakeetSTTService(RivaSTTService):
-    class InputParams(BaseModel):
-        language: Optional[Language] = Language.EN_US
+    """Deprecated: Use RivaSTTService instead."""
 
     def __init__(
         self,
         *,
-        api_key: str = None,
+        api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
         function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
         model_name: str = "parakeet-ctc-1.1b-asr",
         sample_rate: Optional[int] = None,
-        params: InputParams = InputParams(),
+        params: RivaSTTService.InputParams,  # Use parent class's type
         **kwargs,
     ):
         super().__init__(

From c14406a3b96a9d14b3cc30648676b657aaadec32 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 09:59:01 -0400
Subject: [PATCH 10/21] Demos use the latest services

---
 examples/foundational/07r-interruptible-riva-nim.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py
index 070467844..7b82871ab 100644
--- a/examples/foundational/07r-interruptible-riva-nim.py
+++ b/examples/foundational/07r-interruptible-riva-nim.py
@@ -41,13 +41,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
         ),
     )
 
-    stt = RivaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    # stt = RivaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
     # stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
 
-    tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
-    # tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
+    # tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
+    tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     messages = [
         {

From da60e7069be5db30776365ea8c929d57f92a5764 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 10:08:49 -0400
Subject: [PATCH 11/21] Update pyproject.toml to use nvidia-riva-client 2.19.1

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 6d1cf840c..30d72f161 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,7 +78,7 @@ perplexity = []
 playht = [ "pyht~=0.1.12", "websockets~=13.1" ]
 qwen = []
 rime = [ "websockets~=13.1" ]
-riva = [ "nvidia-riva-client~=2.19.0" ]
+riva = [ "nvidia-riva-client~=2.19.1" ]
 sentry = [ "sentry-sdk~=2.23.1" ]
 local-smart-turn = [ "coremltools>=8.0", "transformers", "torch==2.5.0", "torchaudio==2.5.0" ]
 remote-smart-turn = []

From e7d889a1435eef3cf6bd187b9237f448bf5bba79 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 10:49:35 -0400
Subject: [PATCH 12/21] Update RivaSTTService to use  by default

---
 src/pipecat/services/riva/stt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py
index 0ee7b3e5e..0d15ab0af 100644
--- a/src/pipecat/services/riva/stt.py
+++ b/src/pipecat/services/riva/stt.py
@@ -94,7 +94,7 @@ class RivaSTTService(STTService):
         api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
         function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
-        model_name: str = "parakeet-ctc-1.1b-asr",
+        model_name: str = "parakeet-1.1b-rnnt-multilingual-asr",
         sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,

From 6c428c303bda85b75c2bb53977f3323033c73697 Mon Sep 17 00:00:00 2001
From: vipyne <vipyne@gmail.com>
Date: Fri, 2 May 2025 10:27:50 -0500
Subject: [PATCH 13/21] update magpie voice

---
 src/pipecat/services/riva/tts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py
index bfa732678..5ab0a0eaa 100644
--- a/src/pipecat/services/riva/tts.py
+++ b/src/pipecat/services/riva/tts.py
@@ -44,7 +44,7 @@ class RivaTTSService(TTSService):
         *,
         api_key: str = None,
         server: str = "grpc.nvcf.nvidia.com:443",
-        voice_id: str = "Magpie-Multilingual.EN-US.Male.Male-1",
+        voice_id: str = "Magpie-Multilingual.EN-US.Ray",
         sample_rate: Optional[int] = None,
         function_id: str = "877104f7-e885-42b9-8de8-f6e4c6303969",
         model_name: str = "magpie-tts-multilingual",

From b2754117c859417e513c3227f8da3ee46981d44b Mon Sep 17 00:00:00 2001
From: vipyne <vipyne@gmail.com>
Date: Fri, 2 May 2025 11:34:40 -0500
Subject: [PATCH 14/21] Riva: refactor function_id and model_name

---
 .../07r-interruptible-riva-nim.py             |  8 +--
 src/pipecat/services/riva/stt.py              | 62 ++++++++++++-------
 src/pipecat/services/riva/tts.py              | 29 ++++++---
 3 files changed, 64 insertions(+), 35 deletions(-)

diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py
index 7b82871ab..efe9b5479 100644
--- a/examples/foundational/07r-interruptible-riva-nim.py
+++ b/examples/foundational/07r-interruptible-riva-nim.py
@@ -41,14 +41,14 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
         ),
     )
 
-    stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
-    # stt = RivaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    # stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) # parakeet og
+    stt = RivaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY"))  # canary
     # stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
 
-    # tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
-    tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
+    tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
+    # tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     messages = [
         {
diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py
index 0d15ab0af..2b263f03e 100644
--- a/src/pipecat/services/riva/stt.py
+++ b/src/pipecat/services/riva/stt.py
@@ -5,7 +5,7 @@
 #
 
 import asyncio
-from typing import AsyncGenerator, List, Optional
+from typing import AsyncGenerator, List, Mapping, Optional
 
 from loguru import logger
 from pydantic import BaseModel
@@ -93,8 +93,10 @@ class RivaSTTService(STTService):
         *,
         api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
-        function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
-        model_name: str = "parakeet-1.1b-rnnt-multilingual-asr",
+        model_function_map: Mapping[str, str] = {
+            "function_id": "1598d209-5e27-4d3c-8079-4751568b1081",
+            "model_name": "parakeet-ctc-1.1b-asr",
+        },
         sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
@@ -114,11 +116,12 @@ class RivaSTTService(STTService):
         self._stop_history_eou = -1
         self._stop_threshold_eou = -1.0
         self._custom_configuration = ""
+        self._function_id = model_function_map.get("function_id")
 
-        self.set_model_name(model_name)
+        self.set_model_name(model_function_map.get("model_name"))
 
         metadata = [
-            ["function-id", function_id],
+            ["function-id", self._function_id],
             ["authorization", f"Bearer {api_key}"],
         ]
         auth = riva.client.Auth(None, True, server, metadata)
@@ -133,6 +136,13 @@ class RivaSTTService(STTService):
     def can_generate_metrics(self) -> bool:
         return False
 
+    async def set_model(self, model: str):
+        logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
+        example = {"function_id": "<UUID>", "model_name": "<model_name>"}
+        logger.warning(
+            f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
+        )
+
     async def start(self, frame: StartFrame):
         await super().start(frame)
 
@@ -253,25 +263,22 @@ class RivaSTTService(STTService):
 
 
 class RivaSegmentedSTTService(SegmentedSTTService):
-    """Speech-to-text service using NVIDIA Riva Canary ASR API.
+    """Speech-to-text service using NVIDIA Riva's offline/batch models.
 
-    This service uses NVIDIA's Riva Canary ASR API to perform speech-to-text
+    By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text
     transcription on audio segments. It inherits from SegmentedSTTService to handle
     audio buffering and speech detection.
 
     Args:
         api_key: NVIDIA API key for authentication
         server: Riva server address (defaults to NVIDIA Cloud Function endpoint)
-        function_id: NVIDIA Cloud Function ID for the Canary ASR service
-        model_name: Name of the Canary ASR model to use
+        model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID
         sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate
         params: Additional configuration parameters for Riva
         **kwargs: Additional arguments passed to SegmentedSTTService
     """
 
     class InputParams(BaseModel):
-        """Configuration parameters for Riva Canary ASR API."""
-
         language: Optional[Language] = Language.EN_US
         profanity_filter: bool = False
         automatic_punctuation: bool = True
@@ -284,8 +291,10 @@ class RivaSegmentedSTTService(SegmentedSTTService):
         *,
         api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
-        function_id: str = "ee8dc628-76de-4acc-8595-1836e7e857bd",
-        model_name: str = "canary-1b-asr",
+        model_function_map: Mapping[str, str] = {
+            "function_id": "ee8dc628-76de-4acc-8595-1836e7e857bd",
+            "model_name": "canary-1b-asr",
+        },
         sample_rate: Optional[int] = None,
         params: InputParams = InputParams(),
         **kwargs,
@@ -293,12 +302,13 @@ class RivaSegmentedSTTService(SegmentedSTTService):
         super().__init__(sample_rate=sample_rate, **kwargs)
 
         # Set model name
-        self.set_model_name(model_name)
+        self.set_model_name(model_function_map.get("model_name"))
 
         # Initialize Riva settings
         self._api_key = api_key
         self._server = server
-        self._function_id = function_id
+        self._function_id = model_function_map.get("function_id")
+        self._model_name = model_function_map.get("model_name")
 
         # Store the language as a Language enum and as a string
         self._language_enum = params.language or Language.EN_US
@@ -344,7 +354,7 @@ class RivaSegmentedSTTService(SegmentedSTTService):
         auth = riva.client.Auth(None, True, self._server, metadata)
         self._asr_service = riva.client.ASRService(auth)
 
-        logger.info(f"Initialized Riva Canary ASR service with model: {self.model_name}")
+        logger.info(f"Initialized RivaSegmentedSTTService with model: {self.model_name}")
 
     def _create_recognition_config(self):
         """Create the Riva ASR recognition configuration."""
@@ -384,6 +394,13 @@ class RivaSegmentedSTTService(SegmentedSTTService):
         """Indicates whether this service can generate processing metrics."""
         return True
 
+    async def set_model(self, model: str):
+        logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
+        example = {"function_id": "<UUID>", "model_name": "<model_name>"}
+        logger.warning(
+            f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
+        )
+
     async def start(self, frame: StartFrame):
         """Initialize the service when the pipeline starts."""
         await super().start(frame)
@@ -402,7 +419,7 @@ class RivaSegmentedSTTService(SegmentedSTTService):
             self._config.language_code = self._language
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        """Transcribe an audio segment using Riva Canary ASR.
+        """Transcribe an audio segment.
 
         Args:
             audio: Raw audio bytes in WAV format (already converted by base class).
@@ -478,17 +495,18 @@ class ParakeetSTTService(RivaSTTService):
         *,
         api_key: str,
         server: str = "grpc.nvcf.nvidia.com:443",
-        function_id: str = "1598d209-5e27-4d3c-8079-4751568b1081",
-        model_name: str = "parakeet-ctc-1.1b-asr",
+        model_function_map: Mapping[str, str] = {
+            "function_id": "1598d209-5e27-4d3c-8079-4751568b1081",
+            "model_name": "parakeet-ctc-1.1b-asr",
+        },
         sample_rate: Optional[int] = None,
-        params: RivaSTTService.InputParams,  # Use parent class's type
+        params: RivaSTTService.InputParams = RivaSTTService.InputParams(),  # Use parent class's type
         **kwargs,
     ):
         super().__init__(
             api_key=api_key,
             server=server,
-            function_id=function_id,
-            model_name=model_name,
+            model_function_map=model_function_map,
             sample_rate=sample_rate,
             params=params,
             **kwargs,
diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py
index 5ab0a0eaa..4fc8b8489 100644
--- a/src/pipecat/services/riva/tts.py
+++ b/src/pipecat/services/riva/tts.py
@@ -6,7 +6,7 @@
 
 import asyncio
 import os
-from typing import AsyncGenerator, Optional
+from typing import AsyncGenerator, Mapping, Optional
 
 # Suppress gRPC fork warnings
 os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false"
@@ -46,8 +46,10 @@ class RivaTTSService(TTSService):
         server: str = "grpc.nvcf.nvidia.com:443",
         voice_id: str = "Magpie-Multilingual.EN-US.Ray",
         sample_rate: Optional[int] = None,
-        function_id: str = "877104f7-e885-42b9-8de8-f6e4c6303969",
-        model_name: str = "magpie-tts-multilingual",
+        model_function_map: Mapping[str, str] = {
+            "function_id": "877104f7-e885-42b9-8de8-f6e4c6303969",
+            "model_name": "magpie-tts-multilingual",
+        },
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -56,12 +58,13 @@ class RivaTTSService(TTSService):
         self._voice_id = voice_id
         self._language_code = params.language
         self._quality = params.quality
+        self._function_id = model_function_map.get("function_id")
 
-        self.set_model_name(model_name)
+        self.set_model_name(model_function_map.get("model_name"))
         self.set_voice(voice_id)
 
         metadata = [
-            ["function-id", function_id],
+            ["function-id", self._function_id],
             ["authorization", f"Bearer {api_key}"],
         ]
         auth = riva.client.Auth(None, True, server, metadata)
@@ -73,6 +76,13 @@ class RivaTTSService(TTSService):
             riva.client.proto.riva_tts_pb2.RivaSynthesisConfigRequest()
         )
 
+    async def set_model(self, model: str):
+        logger.warning(f"Cannot set model after initialization. Set model and function id like so:")
+        example = {"function_id": "<UUID>", "model_name": "<model_name>"}
+        logger.warning(
+            f"{self.__class__.__name__}(api_key=<api_key>, model_function_map={example})"
+        )
+
     async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
         def read_audio_responses(queue: asyncio.Queue):
             def add_response(r):
@@ -134,8 +144,10 @@ class FastPitchTTSService(RivaTTSService):
         server: str = "grpc.nvcf.nvidia.com:443",
         voice_id: str = "English-US.Female-1",
         sample_rate: Optional[int] = None,
-        function_id: str = "0149dedb-2be8-4195-b9a0-e57e0e14f972",
-        model_name: str = "fastpitch-hifigan-tts",
+        model_function_map: Mapping[str, str] = {
+            "function_id": "0149dedb-2be8-4195-b9a0-e57e0e14f972",
+            "model_name": "fastpitch-hifigan-tts",
+        },
         params: InputParams = InputParams(),
         **kwargs,
     ):
@@ -143,8 +155,7 @@ class FastPitchTTSService(RivaTTSService):
             api_key=api_key,
             voice_id=voice_id,
             sample_rate=sample_rate,
-            function_id=function_id,
-            model_name=model_name,
+            model_function_map=model_function_map,
             params=params,
             **kwargs,
         )

From b2ecd83706de83f6e65236cc0d4844d46ab85b89 Mon Sep 17 00:00:00 2001
From: vipyne <vipyne@gmail.com>
Date: Fri, 2 May 2025 14:04:32 -0500
Subject: [PATCH 15/21] update CHANGELOG with Riva details

---
 CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5954e6baf..1f4811184 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -119,6 +119,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   case there's no need to push audio to the rest of the pipeline, but this is
   not a very common case.
 
+- Added `RivaSegmentedSTTService`, which allows Riva offline/batch models, such
+  as  to be "canary-1b-asr" used in Pipecat.
+
 ### Deprecated
 
 - Function calls with parameters
@@ -134,6 +137,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `TransportParams.vad_audio_passthrough` parameter is now deprecated, use
   `TransportParams.audio_in_passthrough` instead.
 
+- `ParakeetSTTService` is now deprecated, use `RivaSTTService` instead, which uses
+  the model "parakeet-ctc-1.1b-asr" by default.
+
+- `FastPitchTTSService` is now deprecated, use `RivaTTSService` instead, which uses
+  the model "magpie-tts-multilingual" by default.
+
 ### Fixed
 
 - Fixed an issue where `OpenAIRealtimeBetaLLMService` would add two assistant

From 667bd32e6a5471a47eae30c139007fd271f15675 Mon Sep 17 00:00:00 2001
From: vipyne <vipyne@gmail.com>
Date: Fri, 2 May 2025 15:32:33 -0500
Subject: [PATCH 16/21] Riva: remove deprecated lines in example

---
 examples/foundational/07r-interruptible-riva-nim.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py
index efe9b5479..ddb80181c 100644
--- a/examples/foundational/07r-interruptible-riva-nim.py
+++ b/examples/foundational/07r-interruptible-riva-nim.py
@@ -41,14 +41,11 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespac
         ),
     )
 
-    # stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY")) # parakeet og
-    stt = RivaSegmentedSTTService(api_key=os.getenv("NVIDIA_API_KEY"))  # canary
-    # stt = ParakeetSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
+    stt = RivaSTTService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     llm = NimLLMService(api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct")
 
-    tts = FastPitchTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
-    # tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
+    tts = RivaTTSService(api_key=os.getenv("NVIDIA_API_KEY"))
 
     messages = [
         {

From 814e7509e17d0b66914651b7b552b8440e5b4e44 Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 18 Apr 2025 08:47:32 -0400
Subject: [PATCH 17/21] demo: Restructure storytelling-chatbot directory,
 update README steps, link to vercel demo

---
 examples/storytelling-chatbot/.dockerignore   |   2 -
 examples/storytelling-chatbot/README.md       |  86 ++++++++++++------
 .../{frontend => client}/.eslintrc.json       |   0
 .../{frontend => client}/.gitignore           |   0
 .../{frontend => client}/README.md            |   0
 .../{frontend => client}/app/favicon.ico      | Bin
 .../{frontend => client}/app/globals.css      |   0
 .../{frontend => client}/app/layout.tsx       |   0
 .../app/opengraph-image.png                   | Bin
 .../{frontend => client}/app/page.tsx         |   0
 .../app/twitter-image.png                     | Bin
 .../{frontend => client}/app/utils.ts         |   0
 .../{frontend => client}/components.json      |   0
 .../{frontend => client}/components/App.tsx   |   0
 .../components/AudioIndicator/index.tsx       |   0
 .../components/DevicePicker/index.tsx         |   0
 .../components/MicToggle/index.tsx            |   0
 .../{frontend => client}/components/Setup.tsx |   0
 .../{frontend => client}/components/Story.tsx |   0
 .../StoryTranscript.module.css                |   0
 .../components/StoryTranscript/index.tsx      |   0
 .../UserInputIndicator.module.css             |   0
 .../components/UserInputIndicator/index.tsx   |   0
 .../components/VideoTile/VideoTile.module.css |   0
 .../components/VideoTile/index.tsx            |   0
 .../components/WaveText/WaveText.module.css   |   0
 .../components/WaveText/index.tsx             |   0
 .../components/ui/button.tsx                  |   0
 .../components/ui/select.tsx                  |   0
 .../components/ui/typewriter.tsx              |   0
 .../{frontend => client}/env.example          |   0
 .../{frontend => client}/next.config.mjs      |   0
 .../{frontend => client}/package-lock.json    |   4 +-
 .../{frontend => client}/package.json         |   2 +-
 .../{frontend => client}/postcss.config.js    |   0
 .../public/alpha-mask.gif                     | Bin
 .../{frontend => client}/public/bg.jpg        | Bin
 .../{frontend => client}/tailwind.config.ts   |   0
 .../{frontend => client}/tsconfig.json        |   0
 .../storytelling-chatbot/server/.dockerignore |   2 +
 .../{ => server}/Dockerfile                   |  10 +-
 .../{src => server}/assets/book1.png          | Bin
 .../{src => server}/assets/book2.png          | Bin
 .../{src => server}/assets/ding.wav           | Bin
 .../{src => server}/assets/listening.wav      | Bin
 .../{src => server}/assets/talking.wav        | Bin
 .../{src => server}/bot.py                    |   0
 .../{src => server}/bot_runner.py             |   4 +-
 .../{ => server}/env.example                  |   0
 .../{src => server}/processors.py             |   0
 .../{src => server}/prompts.py                |   0
 .../{ => server}/requirements.txt             |   0
 .../{src => server}/utils/helpers.py          |   0
 53 files changed, 69 insertions(+), 41 deletions(-)
 delete mode 100644 examples/storytelling-chatbot/.dockerignore
 rename examples/storytelling-chatbot/{frontend => client}/.eslintrc.json (100%)
 rename examples/storytelling-chatbot/{frontend => client}/.gitignore (100%)
 rename examples/storytelling-chatbot/{frontend => client}/README.md (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/favicon.ico (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/globals.css (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/layout.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/opengraph-image.png (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/page.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/twitter-image.png (100%)
 rename examples/storytelling-chatbot/{frontend => client}/app/utils.ts (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components.json (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/App.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/AudioIndicator/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/DevicePicker/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/MicToggle/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/Setup.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/Story.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/StoryTranscript/StoryTranscript.module.css (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/StoryTranscript/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/UserInputIndicator/UserInputIndicator.module.css (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/UserInputIndicator/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/VideoTile/VideoTile.module.css (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/VideoTile/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/WaveText/WaveText.module.css (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/WaveText/index.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/ui/button.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/ui/select.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/components/ui/typewriter.tsx (100%)
 rename examples/storytelling-chatbot/{frontend => client}/env.example (100%)
 rename examples/storytelling-chatbot/{frontend => client}/next.config.mjs (100%)
 rename examples/storytelling-chatbot/{frontend => client}/package-lock.json (99%)
 rename examples/storytelling-chatbot/{frontend => client}/package.json (97%)
 rename examples/storytelling-chatbot/{frontend => client}/postcss.config.js (100%)
 rename examples/storytelling-chatbot/{frontend => client}/public/alpha-mask.gif (100%)
 rename examples/storytelling-chatbot/{frontend => client}/public/bg.jpg (100%)
 rename examples/storytelling-chatbot/{frontend => client}/tailwind.config.ts (100%)
 rename examples/storytelling-chatbot/{frontend => client}/tsconfig.json (100%)
 create mode 100644 examples/storytelling-chatbot/server/.dockerignore
 rename examples/storytelling-chatbot/{ => server}/Dockerfile (87%)
 rename examples/storytelling-chatbot/{src => server}/assets/book1.png (100%)
 rename examples/storytelling-chatbot/{src => server}/assets/book2.png (100%)
 rename examples/storytelling-chatbot/{src => server}/assets/ding.wav (100%)
 rename examples/storytelling-chatbot/{src => server}/assets/listening.wav (100%)
 rename examples/storytelling-chatbot/{src => server}/assets/talking.wav (100%)
 rename examples/storytelling-chatbot/{src => server}/bot.py (100%)
 rename examples/storytelling-chatbot/{src => server}/bot_runner.py (98%)
 rename examples/storytelling-chatbot/{ => server}/env.example (100%)
 rename examples/storytelling-chatbot/{src => server}/processors.py (100%)
 rename examples/storytelling-chatbot/{src => server}/prompts.py (100%)
 rename examples/storytelling-chatbot/{ => server}/requirements.txt (100%)
 rename examples/storytelling-chatbot/{src => server}/utils/helpers.py (100%)

diff --git a/examples/storytelling-chatbot/.dockerignore b/examples/storytelling-chatbot/.dockerignore
deleted file mode 100644
index a6b5f4231..000000000
--- a/examples/storytelling-chatbot/.dockerignore
+++ /dev/null
@@ -1,2 +0,0 @@
-frontend/node_modules
-frontend/out
\ No newline at end of file
diff --git a/examples/storytelling-chatbot/README.md b/examples/storytelling-chatbot/README.md
index 08da91044..d10620eef 100644
--- a/examples/storytelling-chatbot/README.md
+++ b/examples/storytelling-chatbot/README.md
@@ -1,4 +1,4 @@
-[![Try](https://img.shields.io/badge/try_it-here-blue)](https://storytelling-chatbot.fly.dev)
+[![Try](https://img.shields.io/badge/try_it-here-blue)](https://gemini-storybot.vercel.app/)
 
 # Storytelling Chatbot
 
@@ -9,7 +9,6 @@ It periodically prompts the user for input for a 'choose your own adventure' sty
 
 We use Gemini 2.0 for creating the story and image prompts, and we add visual elements to the story by generating images using Google's Imagen.
 
-
 ---
 
 ### It uses the following AI services:
@@ -20,7 +19,7 @@ Transcribes inbound participant voice media to text.
 
 **Google Gemini 2.0 - LLM**
 
-Our creative writer LLM. You can see the context used to prompt it [here](src/prompts.py)
+Our creative writer LLM. You can see the context used to prompt it [here](server/prompts.py)
 
 **ElevenLabs - Text-to-Speech**
 
@@ -34,47 +33,76 @@ Adds pictures to our story. Prompting is quite key for style consistency, so we
 
 ## Setup
 
-**Install requirements**
+### Client
 
-```shell
-python3 -m venv venv
-source venv/bin/activate
-pip install -r requirements.txt
-```
+1. Navigate to the client directory:
 
-**Create environment file and set variables:**
+   ```shell
+   cd client
+   ```
 
-```shell
-mv env.example .env
-```
+2. Install dependencies:
 
-When deploying to production, to ensure only this app can spawn a new bot, set your `ENV` to `production`
+   ```shell
+   npm install
+   ```
 
-**Build the frontend:**
+3. Build the client:
 
-This project uses a custom frontend, which needs to built. Note: this is done automatically as part of the Docker deployment.
+   ```shell
+   npm run build
+   ```
 
-```shell
-cd frontend/
-npm install
-npm run build
-```
+### Server
 
-The build UI files can be found in `frontend/out`
+1. Navigate to the server directory
 
-## Running it locally
+   ```shell
+   cd ../server
+   ```
 
-Start the API / bot manager:
+2. Set up your virtual environment and install requirements
 
-`python src/bot_runner.py --host localhost`
+   ```shell
+   python3 -m venv venv
+   source venv/bin/activate
+   pip install -r requirements.txt
+   ```
 
-If you'd like to run a custom domain or port:
+3. Create environment file and set variables
 
-`python src/bot_runner.py --host somehost --p someport`
+   ```shell
+   mv env.example .env
+   ```
 
-➡️ Open the host URL in your browser `http://localhost:7860`
+   You'll need API keys for:
 
-If you've run previous versions of the demo, make sure to set `ENV=dev`, and remove the `RUN_AS_VM` line from the .env file.
+   - DAILY_API_KEY
+   - ELEVENLABS_API_KEY
+   - ELEVENLABS_VOICE_ID
+   - GOOGLE_API_KEY
+
+4. (Optional) Deployment:
+
+   When deploying to production, to ensure only this app can spawn new bot processes, set your `ENV` to `production`
+
+## Run it locally
+
+1. Navigate back to the demo's root directory:
+
+   ```shell
+   cd ..
+   ```
+
+2. Run the application:
+
+   ```shell
+   python server/bot_runner.py --host localhost
+   ```
+
+   You can run with a custom domain or port using: `python server/bot_runner.py --host somehost --p someport`
+
+3. ➡️ Open the host URL in your browser: http://localhost:7860
 
 ---
 
diff --git a/examples/storytelling-chatbot/frontend/.eslintrc.json b/examples/storytelling-chatbot/client/.eslintrc.json
similarity index 100%
rename from examples/storytelling-chatbot/frontend/.eslintrc.json
rename to examples/storytelling-chatbot/client/.eslintrc.json
diff --git a/examples/storytelling-chatbot/frontend/.gitignore b/examples/storytelling-chatbot/client/.gitignore
similarity index 100%
rename from examples/storytelling-chatbot/frontend/.gitignore
rename to examples/storytelling-chatbot/client/.gitignore
diff --git a/examples/storytelling-chatbot/frontend/README.md b/examples/storytelling-chatbot/client/README.md
similarity index 100%
rename from examples/storytelling-chatbot/frontend/README.md
rename to examples/storytelling-chatbot/client/README.md
diff --git a/examples/storytelling-chatbot/frontend/app/favicon.ico b/examples/storytelling-chatbot/client/app/favicon.ico
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/favicon.ico
rename to examples/storytelling-chatbot/client/app/favicon.ico
diff --git a/examples/storytelling-chatbot/frontend/app/globals.css b/examples/storytelling-chatbot/client/app/globals.css
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/globals.css
rename to examples/storytelling-chatbot/client/app/globals.css
diff --git a/examples/storytelling-chatbot/frontend/app/layout.tsx b/examples/storytelling-chatbot/client/app/layout.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/layout.tsx
rename to examples/storytelling-chatbot/client/app/layout.tsx
diff --git a/examples/storytelling-chatbot/frontend/app/opengraph-image.png b/examples/storytelling-chatbot/client/app/opengraph-image.png
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/opengraph-image.png
rename to examples/storytelling-chatbot/client/app/opengraph-image.png
diff --git a/examples/storytelling-chatbot/frontend/app/page.tsx b/examples/storytelling-chatbot/client/app/page.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/page.tsx
rename to examples/storytelling-chatbot/client/app/page.tsx
diff --git a/examples/storytelling-chatbot/frontend/app/twitter-image.png b/examples/storytelling-chatbot/client/app/twitter-image.png
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/twitter-image.png
rename to examples/storytelling-chatbot/client/app/twitter-image.png
diff --git a/examples/storytelling-chatbot/frontend/app/utils.ts b/examples/storytelling-chatbot/client/app/utils.ts
similarity index 100%
rename from examples/storytelling-chatbot/frontend/app/utils.ts
rename to examples/storytelling-chatbot/client/app/utils.ts
diff --git a/examples/storytelling-chatbot/frontend/components.json b/examples/storytelling-chatbot/client/components.json
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components.json
rename to examples/storytelling-chatbot/client/components.json
diff --git a/examples/storytelling-chatbot/frontend/components/App.tsx b/examples/storytelling-chatbot/client/components/App.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/App.tsx
rename to examples/storytelling-chatbot/client/components/App.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/AudioIndicator/index.tsx b/examples/storytelling-chatbot/client/components/AudioIndicator/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/AudioIndicator/index.tsx
rename to examples/storytelling-chatbot/client/components/AudioIndicator/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/DevicePicker/index.tsx b/examples/storytelling-chatbot/client/components/DevicePicker/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/DevicePicker/index.tsx
rename to examples/storytelling-chatbot/client/components/DevicePicker/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/MicToggle/index.tsx b/examples/storytelling-chatbot/client/components/MicToggle/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/MicToggle/index.tsx
rename to examples/storytelling-chatbot/client/components/MicToggle/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/Setup.tsx b/examples/storytelling-chatbot/client/components/Setup.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/Setup.tsx
rename to examples/storytelling-chatbot/client/components/Setup.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/Story.tsx b/examples/storytelling-chatbot/client/components/Story.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/Story.tsx
rename to examples/storytelling-chatbot/client/components/Story.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/StoryTranscript/StoryTranscript.module.css b/examples/storytelling-chatbot/client/components/StoryTranscript/StoryTranscript.module.css
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/StoryTranscript/StoryTranscript.module.css
rename to examples/storytelling-chatbot/client/components/StoryTranscript/StoryTranscript.module.css
diff --git a/examples/storytelling-chatbot/frontend/components/StoryTranscript/index.tsx b/examples/storytelling-chatbot/client/components/StoryTranscript/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/StoryTranscript/index.tsx
rename to examples/storytelling-chatbot/client/components/StoryTranscript/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css b/examples/storytelling-chatbot/client/components/UserInputIndicator/UserInputIndicator.module.css
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/UserInputIndicator/UserInputIndicator.module.css
rename to examples/storytelling-chatbot/client/components/UserInputIndicator/UserInputIndicator.module.css
diff --git a/examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx b/examples/storytelling-chatbot/client/components/UserInputIndicator/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/UserInputIndicator/index.tsx
rename to examples/storytelling-chatbot/client/components/UserInputIndicator/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css b/examples/storytelling-chatbot/client/components/VideoTile/VideoTile.module.css
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/VideoTile/VideoTile.module.css
rename to examples/storytelling-chatbot/client/components/VideoTile/VideoTile.module.css
diff --git a/examples/storytelling-chatbot/frontend/components/VideoTile/index.tsx b/examples/storytelling-chatbot/client/components/VideoTile/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/VideoTile/index.tsx
rename to examples/storytelling-chatbot/client/components/VideoTile/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/WaveText/WaveText.module.css b/examples/storytelling-chatbot/client/components/WaveText/WaveText.module.css
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/WaveText/WaveText.module.css
rename to examples/storytelling-chatbot/client/components/WaveText/WaveText.module.css
diff --git a/examples/storytelling-chatbot/frontend/components/WaveText/index.tsx b/examples/storytelling-chatbot/client/components/WaveText/index.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/WaveText/index.tsx
rename to examples/storytelling-chatbot/client/components/WaveText/index.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/ui/button.tsx b/examples/storytelling-chatbot/client/components/ui/button.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/ui/button.tsx
rename to examples/storytelling-chatbot/client/components/ui/button.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/ui/select.tsx b/examples/storytelling-chatbot/client/components/ui/select.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/ui/select.tsx
rename to examples/storytelling-chatbot/client/components/ui/select.tsx
diff --git a/examples/storytelling-chatbot/frontend/components/ui/typewriter.tsx b/examples/storytelling-chatbot/client/components/ui/typewriter.tsx
similarity index 100%
rename from examples/storytelling-chatbot/frontend/components/ui/typewriter.tsx
rename to examples/storytelling-chatbot/client/components/ui/typewriter.tsx
diff --git a/examples/storytelling-chatbot/frontend/env.example b/examples/storytelling-chatbot/client/env.example
similarity index 100%
rename from examples/storytelling-chatbot/frontend/env.example
rename to examples/storytelling-chatbot/client/env.example
diff --git a/examples/storytelling-chatbot/frontend/next.config.mjs b/examples/storytelling-chatbot/client/next.config.mjs
similarity index 100%
rename from examples/storytelling-chatbot/frontend/next.config.mjs
rename to examples/storytelling-chatbot/client/next.config.mjs
diff --git a/examples/storytelling-chatbot/frontend/package-lock.json b/examples/storytelling-chatbot/client/package-lock.json
similarity index 99%
rename from examples/storytelling-chatbot/frontend/package-lock.json
rename to examples/storytelling-chatbot/client/package-lock.json
index f94e74f85..57761664d 100644
--- a/examples/storytelling-chatbot/frontend/package-lock.json
+++ b/examples/storytelling-chatbot/client/package-lock.json
@@ -1,11 +1,11 @@
 {
-  "name": "frontend",
+  "name": "client",
   "version": "0.1.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "frontend",
+      "name": "client",
       "version": "0.1.0",
       "dependencies": {
         "@daily-co/daily-js": "^0.62.0",
diff --git a/examples/storytelling-chatbot/frontend/package.json b/examples/storytelling-chatbot/client/package.json
similarity index 97%
rename from examples/storytelling-chatbot/frontend/package.json
rename to examples/storytelling-chatbot/client/package.json
index b68568b4f..4b34d1059 100644
--- a/examples/storytelling-chatbot/frontend/package.json
+++ b/examples/storytelling-chatbot/client/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "frontend",
+  "name": "client",
   "version": "0.1.0",
   "private": true,
   "scripts": {
diff --git a/examples/storytelling-chatbot/frontend/postcss.config.js b/examples/storytelling-chatbot/client/postcss.config.js
similarity index 100%
rename from examples/storytelling-chatbot/frontend/postcss.config.js
rename to examples/storytelling-chatbot/client/postcss.config.js
diff --git a/examples/storytelling-chatbot/frontend/public/alpha-mask.gif b/examples/storytelling-chatbot/client/public/alpha-mask.gif
similarity index 100%
rename from examples/storytelling-chatbot/frontend/public/alpha-mask.gif
rename to examples/storytelling-chatbot/client/public/alpha-mask.gif
diff --git a/examples/storytelling-chatbot/frontend/public/bg.jpg b/examples/storytelling-chatbot/client/public/bg.jpg
similarity index 100%
rename from examples/storytelling-chatbot/frontend/public/bg.jpg
rename to examples/storytelling-chatbot/client/public/bg.jpg
diff --git a/examples/storytelling-chatbot/frontend/tailwind.config.ts b/examples/storytelling-chatbot/client/tailwind.config.ts
similarity index 100%
rename from examples/storytelling-chatbot/frontend/tailwind.config.ts
rename to examples/storytelling-chatbot/client/tailwind.config.ts
diff --git a/examples/storytelling-chatbot/frontend/tsconfig.json b/examples/storytelling-chatbot/client/tsconfig.json
similarity index 100%
rename from examples/storytelling-chatbot/frontend/tsconfig.json
rename to examples/storytelling-chatbot/client/tsconfig.json
diff --git a/examples/storytelling-chatbot/server/.dockerignore b/examples/storytelling-chatbot/server/.dockerignore
new file mode 100644
index 000000000..4c6d90950
--- /dev/null
+++ b/examples/storytelling-chatbot/server/.dockerignore
@@ -0,0 +1,2 @@
+client/node_modules
+client/out
\ No newline at end of file
diff --git a/examples/storytelling-chatbot/Dockerfile b/examples/storytelling-chatbot/server/Dockerfile
similarity index 87%
rename from examples/storytelling-chatbot/Dockerfile
rename to examples/storytelling-chatbot/server/Dockerfile
index 5d762cd14..46da11124 100644
--- a/examples/storytelling-chatbot/Dockerfile
+++ b/examples/storytelling-chatbot/server/Dockerfile
@@ -44,11 +44,11 @@ COPY ./requirements.txt requirements.txt
 RUN pip3 install --no-cache-dir --upgrade -r requirements.txt
 
 # Copy everything else
-COPY --chown=user ./src/ src/
+COPY --chown=user ./server/ server/
 
-# Copy frontend app and build
-COPY --chown=user ./frontend/ frontend/
-RUN cd frontend && npm install && npm run build
+# Copy client app and build
+COPY --chown=user ./client/ client/
+RUN cd client && npm install && npm run build
 
 # Start the FastAPI server
-CMD python3 src/bot_runner.py --port ${FAST_API_PORT}
\ No newline at end of file
+CMD python3 server/bot_runner.py --port ${FAST_API_PORT}
\ No newline at end of file
diff --git a/examples/storytelling-chatbot/src/assets/book1.png b/examples/storytelling-chatbot/server/assets/book1.png
similarity index 100%
rename from examples/storytelling-chatbot/src/assets/book1.png
rename to examples/storytelling-chatbot/server/assets/book1.png
diff --git a/examples/storytelling-chatbot/src/assets/book2.png b/examples/storytelling-chatbot/server/assets/book2.png
similarity index 100%
rename from examples/storytelling-chatbot/src/assets/book2.png
rename to examples/storytelling-chatbot/server/assets/book2.png
diff --git a/examples/storytelling-chatbot/src/assets/ding.wav b/examples/storytelling-chatbot/server/assets/ding.wav
similarity index 100%
rename from examples/storytelling-chatbot/src/assets/ding.wav
rename to examples/storytelling-chatbot/server/assets/ding.wav
diff --git a/examples/storytelling-chatbot/src/assets/listening.wav b/examples/storytelling-chatbot/server/assets/listening.wav
similarity index 100%
rename from examples/storytelling-chatbot/src/assets/listening.wav
rename to examples/storytelling-chatbot/server/assets/listening.wav
diff --git a/examples/storytelling-chatbot/src/assets/talking.wav b/examples/storytelling-chatbot/server/assets/talking.wav
similarity index 100%
rename from examples/storytelling-chatbot/src/assets/talking.wav
rename to examples/storytelling-chatbot/server/assets/talking.wav
diff --git a/examples/storytelling-chatbot/src/bot.py b/examples/storytelling-chatbot/server/bot.py
similarity index 100%
rename from examples/storytelling-chatbot/src/bot.py
rename to examples/storytelling-chatbot/server/bot.py
diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/server/bot_runner.py
similarity index 98%
rename from examples/storytelling-chatbot/src/bot_runner.py
rename to examples/storytelling-chatbot/server/bot_runner.py
index 591b2d598..f09b12fac 100644
--- a/examples/storytelling-chatbot/src/bot_runner.py
+++ b/examples/storytelling-chatbot/server/bot_runner.py
@@ -57,7 +57,7 @@ app.add_middleware(
 )
 
 # Mount the static directory
-STATIC_DIR = "frontend/out"
+STATIC_DIR = "client/out"
 
 
 # ------------ Fast API Routes ------------ #
@@ -175,7 +175,7 @@ async def virtualize_bot(room_url: str, token: str):
             image = data[0]["config"]["image"]
 
         # Machine configuration
-        cmd = f"python src/bot.py -u {room_url} -t {token}"
+        cmd = f"python server/bot.py -u {room_url} -t {token}"
         cmd = cmd.split()
         worker_props = {
             "config": {
diff --git a/examples/storytelling-chatbot/env.example b/examples/storytelling-chatbot/server/env.example
similarity index 100%
rename from examples/storytelling-chatbot/env.example
rename to examples/storytelling-chatbot/server/env.example
diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/server/processors.py
similarity index 100%
rename from examples/storytelling-chatbot/src/processors.py
rename to examples/storytelling-chatbot/server/processors.py
diff --git a/examples/storytelling-chatbot/src/prompts.py b/examples/storytelling-chatbot/server/prompts.py
similarity index 100%
rename from examples/storytelling-chatbot/src/prompts.py
rename to examples/storytelling-chatbot/server/prompts.py
diff --git a/examples/storytelling-chatbot/requirements.txt b/examples/storytelling-chatbot/server/requirements.txt
similarity index 100%
rename from examples/storytelling-chatbot/requirements.txt
rename to examples/storytelling-chatbot/server/requirements.txt
diff --git a/examples/storytelling-chatbot/src/utils/helpers.py b/examples/storytelling-chatbot/server/utils/helpers.py
similarity index 100%
rename from examples/storytelling-chatbot/src/utils/helpers.py
rename to examples/storytelling-chatbot/server/utils/helpers.py

From 7152faafb20b13d27640e6685821a44f9eed1627 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Fri, 2 May 2025 14:23:15 -0700
Subject: [PATCH 18/21] BaseOutputTransport: always initialize audio task

We also use the audio task to also send synchronized images with audio.
---
 src/pipecat/transports/base_output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py
index 31a86f92d..b679a50c1 100644
--- a/src/pipecat/transports/base_output.py
+++ b/src/pipecat/transports/base_output.py
@@ -369,7 +369,7 @@ class BaseOutputTransport(FrameProcessor):
         #
 
         def _create_audio_task(self):
-            if not self._audio_task and self._params.audio_out_enabled:
+            if not self._audio_task:
                 self._audio_queue = asyncio.Queue()
                 self._audio_task = self._transport.create_task(self._audio_task_handler())
 

From 872204b795aaf9e72ddcd1d4ca3f929884b7babc Mon Sep 17 00:00:00 2001
From: Mark Backman <mark@daily.co>
Date: Fri, 2 May 2025 17:23:41 -0400
Subject: [PATCH 19/21] Only display the destination in the bot started/stopped
 speaking log when there is a desintation

---
 src/pipecat/transports/base_output.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py
index 31a86f92d..c567025e3 100644
--- a/src/pipecat/transports/base_output.py
+++ b/src/pipecat/transports/base_output.py
@@ -380,7 +380,9 @@ class BaseOutputTransport(FrameProcessor):
 
         async def _bot_started_speaking(self):
             if not self._bot_speaking:
-                logger.debug(f"Bot [{self._destination}] started speaking")
+                logger.debug(
+                    f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking"
+                )
 
                 downstream_frame = BotStartedSpeakingFrame()
                 downstream_frame.transport_destination = self._destination
@@ -393,7 +395,9 @@ class BaseOutputTransport(FrameProcessor):
 
         async def _bot_stopped_speaking(self):
             if self._bot_speaking:
-                logger.debug(f"Bot [{self._destination}] stopped speaking")
+                logger.debug(
+                    f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking"
+                )
 
                 downstream_frame = BotStoppedSpeakingFrame()
                 downstream_frame.transport_destination = self._destination

From 54971a07357376dd3a5c6c4602dc3583db31335f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Fri, 2 May 2025 17:47:44 -0700
Subject: [PATCH 20/21] update to daily-python 0.18.1

---
 pyproject.toml                           |  2 +-
 src/pipecat/transports/services/daily.py | 20 +++++++++++---------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 30d72f161..ecddb0902 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,7 +47,7 @@ canonical = [ "aiofiles~=24.1.0" ]
 cartesia = [ "cartesia~=1.4.0", "websockets~=13.1" ]
 cerebras = []
 deepseek = []
-daily = [ "daily-python~=0.18.0" ]
+daily = [ "daily-python~=0.18.1" ]
 deepgram = [ "deepgram-sdk~=3.8.0" ]
 elevenlabs = [ "websockets~=13.1" ]
 fal = [ "fal-client~=0.5.9" ]
diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py
index 08a2a6030..1684d572a 100644
--- a/src/pipecat/transports/services/daily.py
+++ b/src/pipecat/transports/services/daily.py
@@ -11,14 +11,6 @@ from dataclasses import dataclass
 from typing import Any, Awaitable, Callable, Dict, Mapping, Optional
 
 import aiohttp
-from daily import (
-    AudioData,
-    CustomAudioSource,
-    VideoFrame,
-    VirtualCameraDevice,
-    VirtualMicrophoneDevice,
-    VirtualSpeakerDevice,
-)
 from loguru import logger
 from pydantic import BaseModel
 
@@ -50,7 +42,17 @@ from pipecat.transports.base_transport import BaseTransport, TransportParams
 from pipecat.utils.asyncio import BaseTaskManager
 
 try:
-    from daily import CallClient, Daily, EventHandler
+    from daily import (
+        AudioData,
+        CallClient,
+        CustomAudioSource,
+        Daily,
+        EventHandler,
+        VideoFrame,
+        VirtualCameraDevice,
+        VirtualMicrophoneDevice,
+        VirtualSpeakerDevice,
+    )
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
     logger.error(

From f720d795d0a0af137de1e2a02d7ac5191a2a1790 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Fri, 2 May 2025 20:29:51 -0700
Subject: [PATCH 21/21] update CHANGELOG for pipecat 0.0.66

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11cfca91f..21abadaee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.0.66] - 2025-05-02
 
 ### Added