From 4f093f11dbe537d14c217a04eefe4a3618841873 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 18 Dec 2024 11:38:57 -0500 Subject: [PATCH 1/2] Add CerebrasLLMService and foundational example --- CHANGELOG.md | 3 + README.md | 25 +-- .../14k-function-calling-cerebras.py | 148 ++++++++++++++++++ pyproject.toml | 1 + src/pipecat/services/cerebras.py | 38 +++++ 5 files changed, 203 insertions(+), 12 deletions(-) create mode 100644 examples/foundational/14k-function-calling-cerebras.py create mode 100644 src/pipecat/services/cerebras.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c58234019..998066fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Noise Suppression. (see https://picovoice.ai/platform/koala/) +- Added `CerebrasLLMService` for Cerebras integration with an OpenAI-compatible + interface. Added foundational example `14k-function-calling-cerebras.py`. + - Pipecat now supports Python 3.13. We had a dependency on the `audioop` package which was deprecated and now removed on Python 3.13. We are now using `audioop-lts` (https://github.com/AbstractUmbra/audioop) to provide the same diff --git a/README.md b/README.md index 6c7965e45..b4366125b 100644 --- a/README.md +++ b/README.md @@ -55,17 +55,17 @@ pip install "pipecat-ai[option,...]" Available options include: -| Category | Services | Install Command Example | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | -| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | -| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | -| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | +| Category | Services | Install Command Example | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | +| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | +| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) @@ -223,7 +223,8 @@ Install the ### PyCharm -`ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings: +`ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings: + 1. **Name**: `Ruff formatter` 2. **File type**: `Python` 3. **Working directory**: `$ContentRoot$` diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py new file mode 100644 index 000000000..2b75ad366 --- /dev/null +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.cerebras import CerebrasLLMService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + # note: we can't push a frame to the LLM here. the bot + # can interrupt itself and/or cause audio overlapping glitches. + # possible question for Aleix and Chad about what the right way + # to trigger speech is, now, with the new queues/async/sync refactors. + # await llm.push_frame(TextFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather for a specific location. You MUST use this function whenever asked about weather.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Use fahrenheit for US locations, celsius for others.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] + messages = [ + { + "role": "system", + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + +You have one functions available: + +1. get_current_weather is used to get current weather information. + +Infer whether to use Fahrenheit or Celsius automatically based on the location, unless the user specifies a preference. + +Start by asking me for my location. Then, use 'get_weather_current' to give me a forecast. + + Respond to what the user said in a creative and helpful way.""", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index be9eab02f..75889d51c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ aws = [ "boto3~=1.35.27" ] azure = [ "azure-cognitiveservices-speech~=1.41.1", "openai~=1.57.2" ] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.0.13", "websockets~=13.1" ] +cerebras = [ "openai~=1.57.2" ] daily = [ "daily-python~=0.13.0" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py new file mode 100644 index 000000000..a89b17990 --- /dev/null +++ b/src/pipecat/services/cerebras.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from loguru import logger + +from pipecat.services.openai import OpenAILLMService + + +class CerebrasLLMService(OpenAILLMService): + """A service for interacting with Cerebras's API using the OpenAI-compatible interface. + + This service extends OpenAILLMService to connect to Cerebras's API endpoint while + maintaining full compatibility with OpenAI's interface and functionality. + + Args: + api_key (str): The API key for accessing Cerebras's API + base_url (str, optional): The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1" + model (str, optional): The model identifier to use. Defaults to "llama-3.3-70b" + **kwargs: Additional keyword arguments passed to OpenAILLMService + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.cerebras.ai/v1", + model: str = "llama-3.3-70b", + **kwargs, + ): + super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + + def create_client(self, api_key=None, base_url=None, **kwargs): + """Create OpenAI-compatible client for Cerebras API endpoint.""" + logger.debug(f"Creating Cerebras client with api {base_url}") + return super().create_client(api_key, base_url, **kwargs) From c9dd9060576438ec1b5621e2cf84ae282633c25d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 18 Dec 2024 12:21:55 -0500 Subject: [PATCH 2/2] Tailor chat completion inputs to Cerebras API --- src/pipecat/services/cerebras.py | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py index a89b17990..a0cc81803 100644 --- a/src/pipecat/services/cerebras.py +++ b/src/pipecat/services/cerebras.py @@ -4,10 +4,25 @@ # SPDX-License-Identifier: BSD 2-Clause License # +from typing import List + from loguru import logger +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService +try: + from openai import ( + AsyncStream, + ) + from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Fireworks, you need to `pip install pipecat-ai[cerebras]`. Also, set `CEREBRAS_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + class CerebrasLLMService(OpenAILLMService): """A service for interacting with Cerebras's API using the OpenAI-compatible interface. @@ -36,3 +51,35 @@ class CerebrasLLMService(OpenAILLMService): """Create OpenAI-compatible client for Cerebras API endpoint.""" logger.debug(f"Creating Cerebras client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) + + async def get_chat_completions( + self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] + ) -> AsyncStream[ChatCompletionChunk]: + """Create a streaming chat completion using Cerebras's API. + + Args: + context (OpenAILLMContext): The context object containing tools configuration + and other settings for the chat completion. + messages (List[ChatCompletionMessageParam]): The list of messages comprising + the conversation history and current request. + + Returns: + AsyncStream[ChatCompletionChunk]: A streaming response of chat completion + chunks that can be processed asynchronously. + """ + params = { + "model": self.model_name, + "stream": True, + "messages": messages, + "tools": context.tools, + "tool_choice": context.tool_choice, + "seed": self._settings["seed"], + "temperature": self._settings["temperature"], + "top_p": self._settings["top_p"], + "max_completion_tokens": self._settings["max_completion_tokens"], + } + + params.update(self._settings["extra"]) + + chunks = await self._client.chat.completions.create(**params) + return chunks