diff --git a/examples/foundational/39-mcp-stdio.py b/examples/foundational/39-mcp-stdio.py index b88ee30b1..7127d831f 100644 --- a/examples/foundational/39-mcp-stdio.py +++ b/examples/foundational/39-mcp-stdio.py @@ -155,7 +155,7 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. diff --git a/examples/foundational/39a-mcp-run-sse.py b/examples/foundational/39a-mcp-run-sse.py deleted file mode 100644 index 328af6ce4..000000000 --- a/examples/foundational/39a-mcp-run-sse.py +++ /dev/null @@ -1,159 +0,0 @@ -# -# Copyright (c) 2024–2025, Daily -# -# SPDX-License-Identifier: BSD 2-Clause License -# - - -import os - -from dotenv import load_dotenv -from loguru import logger -from mcp.client.session_group import SseServerParameters - -from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams -from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3 -from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.audio.vad.vad_analyzer import VADParams -from pipecat.frames.frames import LLMRunFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.llm_context import LLMContext -from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair -from pipecat.runner.types import RunnerArguments -from pipecat.runner.utils import create_transport -from pipecat.services.anthropic.llm import AnthropicLLMService -from pipecat.services.cartesia.tts import CartesiaTTSService -from pipecat.services.deepgram.stt import DeepgramSTTService -from pipecat.services.mcp_service import MCPClient -from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.transports.daily.transport import DailyParams -from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams - -load_dotenv(override=True) - -# We store functions so objects (e.g. SileroVADAnalyzer) don't get -# instantiated. The function will be called when the desired transport gets -# selected. -transport_params = { - "daily": lambda: DailyParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "twilio": lambda: FastAPIWebsocketParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), - "webrtc": lambda: TransportParams( - audio_in_enabled=True, - audio_out_enabled=True, - vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), - turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()), - ), -} - - -async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): - logger.info(f"Starting bot") - - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - - tts = CartesiaTTSService( - api_key=os.getenv("CARTESIA_API_KEY"), - voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady - ) - - llm = AnthropicLLMService( - api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-7-sonnet-latest" - ) - - try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - mcp = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) - except Exception as e: - logger.error(f"error setting up mcp") - logger.exception("error trace:") - - tools = {} - try: - tools = await mcp.register_tools(llm) - except Exception as e: - logger.error(f"error registering tools") - logger.exception("error trace:") - - system = f""" - You are a helpful LLM in a WebRTC call. - Your goal is to demonstrate your capabilities in a succinct way. - You have access to a number of tools provided by mcp.run. Use any and all tools to help users. - Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. - Respond to what the user said in a creative and helpful way. - When asked for today's date, use 'https://www.datetoday.net/'. - Don't overexplain what you are doing. - Just respond with short sentences when you are carrying out tool calls. - """ - - messages = [{"role": "system", "content": system}] - - context = LLMContext(messages, tools) - context_aggregator = LLMContextAggregatorPair(context) - - pipeline = Pipeline( - [ - transport.input(), # Transport user input - stt, - context_aggregator.user(), # User spoken responses - llm, # LLM - tts, # TTS - transport.output(), # Transport bot output - context_aggregator.assistant(), # Assistant spoken responses and tool context - ] - ) - - task = PipelineTask( - pipeline, - params=PipelineParams( - enable_metrics=True, - enable_usage_metrics=True, - ), - idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, - ) - - @transport.event_handler("on_client_connected") - async def on_client_connected(transport, client): - logger.info(f"Client connected: {client}") - # Kick off the conversation. - await task.queue_frames([LLMRunFrame()]) - - @transport.event_handler("on_client_disconnected") - async def on_client_disconnected(transport, client): - logger.info(f"Client disconnected") - await task.cancel() - - runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) - - await runner.run(task) - - -async def bot(runner_args: RunnerArguments): - """Main bot entry point compatible with Pipecat Cloud.""" - transport = await create_transport(runner_args, transport_params) - await run_bot(transport, runner_args) - - -if __name__ == "__main__": - if not os.getenv("MCP_RUN_SSE_URL"): - logger.error( - f"Please set MCP_RUN_SSE_URL environment variable for this example. See https://mcp.run" - ) - import sys - - sys.exit(1) - - from pipecat.runner.run import main - - main() diff --git a/examples/foundational/39b-multiple-mcp.py b/examples/foundational/39b-multiple-mcp.py index dad059208..88fdc3610 100644 --- a/examples/foundational/39b-multiple-mcp.py +++ b/examples/foundational/39b-multiple-mcp.py @@ -7,6 +7,7 @@ import asyncio import io +import json import os import re import shutil @@ -15,7 +16,7 @@ import aiohttp from dotenv import load_dotenv from loguru import logger from mcp import StdioServerParameters -from mcp.client.session_group import SseServerParameters +from mcp.client.session_group import StreamableHttpParameters from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema @@ -66,10 +67,12 @@ class UrlToImageProcessor(FrameProcessor): await self.push_frame(frame, direction) def extract_url(self, text: str): - pattern = r"!\[[^\]]*\]\((https?://[^)]+\.(png|jpg|jpeg|PNG|JPG|JPEG|gif))\)" - match = re.search(pattern, text) - if match: - return match.group(1) + data = json.loads(text) + if "artObject" in data: + return data["artObject"]["webImage"]["url"] + if "artworks" in data and len(data["artworks"]): + return data["artworks"][0]["webImage"]["url"] + return None async def run_image_process(self, image_url: str): @@ -132,10 +135,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): system = f""" You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. - You have access to tools to search the Rijksmuseum collection. - Offer, for example, to show the earliest Rembrandt work from the museum. Use the `search_artwork` tool. + You have access to tools to search the Rijksmuseum collection and the user's GitHub repositories and account. + Offer, for example, to show a floral still life, use the `search_artwork` tool. The tool may respond with a JSON object with an `artworks` array. Choose the art from that array. Once the tool has responded, tell the user the title and use the `open_image_in_browser` tool. + You can also offer to answer users questions about their GitHub repositories and account. Your output will be spoken aloud, so avoid special characters that can't easily be spoken, such as emojis or bullet points. Respond to what the user said in a creative and helpful way. Don't overexplain what you are doing. @@ -145,11 +149,11 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): messages = [{"role": "system", "content": system}] try: - mcp = MCPClient( + rijksmuseum_mcp = MCPClient( server_params=StdioServerParameters( command=shutil.which("npx"), # https://github.com/r-huijts/rijksmuseum-mcp - args=["-y", "mcp-server-error setting up mcp"], + args=["-y", "mcp-server-rijksmuseum"], env={"RIJKSMUSEUM_API_KEY": os.getenv("RIJKSMUSEUM_API_KEY")}, ) ) @@ -157,24 +161,32 @@ async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.error(f"error setting up rijksmuseum mcp") logger.exception("error trace:") try: - # https://docs.mcp.run/integrating/tutorials/mcp-run-sse-openai-agents/ - # ie. "https://www.mcp.run/api/mcp/sse?..." - # ensure the profile has a tool or few installed - mcp_run = MCPClient(server_params=SseServerParameters(url=os.getenv("MCP_RUN_SSE_URL"))) + # Github MCP docs: https://github.com/github/github-mcp-server + # Enable Github Copilot on your GitHub account. Free tier is ok. (https://github.com/settings/copilot) + # Generate a personal access token. It must be a Fine-grained token, classic tokens are not supported. (https://github.com/settings/personal-access-tokens) + # Set permissions you want to use (eg. "all repositories", "profile: read/write", etc) + github_mcp = MCPClient( + server_params=StreamableHttpParameters( + url="https://api.githubcopilot.com/mcp/", + headers={ + "Authorization": f"Bearer {os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')}" + }, + ) + ) except Exception as e: logger.error(f"error setting up mcp.run") logger.exception("error trace:") - tools = {} - run_tools = {} + rijksmuseum_tools = {} + github_tools = {} try: - tools = await mcp.register_tools(llm) - run_tools = await mcp_run.register_tools(llm) + rijksmuseum_tools = await rijksmuseum_mcp.register_tools(llm) + github_tools = await github_mcp.register_tools(llm) except Exception as e: logger.error(f"error registering tools") logger.exception("error trace:") - all_standard_tools = run_tools.standard_tools + tools.standard_tools + all_standard_tools = rijksmuseum_tools.standard_tools + github_tools.standard_tools all_tools = ToolsSchema(standard_tools=all_standard_tools) context = LLMContext(messages, all_tools) @@ -226,9 +238,9 @@ async def bot(runner_args: RunnerArguments): if __name__ == "__main__": - if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("MCP_RUN_SSE_URL"): + if not os.getenv("RIJKSMUSEUM_API_KEY") or not os.getenv("GITHUB_PERSONAL_ACCESS_TOKEN"): logger.error( - f"Please set RIJKSMUSEUM_API_KEY and MCP_RUN_SSE_URL environment variables. See https://github.com/r-huijts/rijksmuseum-mcp and https://mcp.run" + f"Please set `RIJKSMUSEUM_API_KEY` and `GITHUB_PERSONAL_ACCESS_TOKEN` environment variables. See https://github.com/r-huijts/rijksmuseum-mcp." ) import sys