wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more updates

Browse Source
This commit is contained in:
Nikita Gamolsky
2024-11-02 13:46:28 -07:00
parent c4cdb2d809
commit abd486025b
1 changed files with 38 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
examples/foundational/99-anthropic-hackathon.py
View File

@@ -57,11 +57,18 @@ class TranscriptFrameCatcher(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
logger.debug(f"TranscriptLogger: {frame}") logger.debug(
f"TranscriptLogger: {frame}, num frames: {len(recent_image_frames)}, anthropic context: {anthropic_context}"
)
if anthropic_context:
add_message_with_images(
anthropic_context, frame.text, frames=list(recent_image_frames)
)
async def main(): async def main():
global llm global llm
global anthropic_context
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session) (room_url, token) = await configure(session)
@@ -112,18 +119,20 @@ Your response will be turned into speech so use only simple words and punctuatio
] ]
context = OpenAILLMContext(messages) context = OpenAILLMContext(messages)
anthropic_context = AnthropicLLMContext.upgrade_to_anthropic(context)
context_aggregator = llm.create_context_aggregator(context) context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
ImageFrameCatcher(), ImageFrameCatcher(),
TranscriptFrameCatcher(),
context_aggregator.user(), # User speech to text context_aggregator.user(), # User speech to text
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
transport.output(), # Transport bot output transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses and tool context context_aggregator.assistant(), # Assistant spoken responses and tool context
] ],
) )
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True)) task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True))
@@ -141,8 +150,6 @@ Your response will be turned into speech so use only simple words and punctuatio
@transport.event_handler("on_app_message") @transport.event_handler("on_app_message")
async def on_app_message(transport, message, sender): async def on_app_message(transport, message, sender):
global anthropic_context
anthropic_context = AnthropicLLMContext.upgrade_to_anthropic(context)
logger.debug(f"Received app message: {message} - {context}") logger.debug(f"Received app message: {message} - {context}")
if not recent_image_frames: if not recent_image_frames:
logger.debug("No image frames to send") logger.debug("No image frames to send")
@@ -157,7 +164,8 @@ Your response will be turned into speech so use only simple words and punctuatio
runner = PipelineRunner() runner = PipelineRunner()
await runner.run(task) await runner.run(task)
def add_message_with_images(c, message, frames=None):
def add_message_with_images(c, message, frames=None):
if frames is None: if frames is None:
frames = list(recent_image_frames) frames = list(recent_image_frames)