wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more updates

Browse Source
This commit is contained in:
Nikita Gamolsky
2024-11-02 13:46:28 -07:00
parent c4cdb2d809
commit abd486025b
1 changed files with 38 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
examples/foundational/99-anthropic-hackathon.py
View File

@@ -57,11 +57,18 @@ class TranscriptFrameCatcher(FrameProcessor):
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
logger.debug(f"TranscriptLogger: {frame}") logger.debug(
f"TranscriptLogger: {frame}, num frames: {len(recent_image_frames)}, anthropic context: {anthropic_context}"
)
if anthropic_context:
add_message_with_images(
anthropic_context, frame.text, frames=list(recent_image_frames)
)
async def main(): async def main():
global llm global llm
global anthropic_context
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
(room_url, token) = await configure(session) (room_url, token) = await configure(session)
@@ -112,18 +119,20 @@ Your response will be turned into speech so use only simple words and punctuatio
] ]
context = OpenAILLMContext(messages) context = OpenAILLMContext(messages)
anthropic_context = AnthropicLLMContext.upgrade_to_anthropic(context)
context_aggregator = llm.create_context_aggregator(context) context_aggregator = llm.create_context_aggregator(context)
pipeline = Pipeline( pipeline = Pipeline(
[ [
transport.input(), # Transport user input transport.input(), # Transport user input
ImageFrameCatcher(), ImageFrameCatcher(),
TranscriptFrameCatcher(),
context_aggregator.user(), # User speech to text context_aggregator.user(), # User speech to text
llm, # LLM llm, # LLM
tts, # TTS tts, # TTS
transport.output(), # Transport bot output transport.output(), # Transport bot output
context_aggregator.assistant(), # Assistant spoken responses and tool context context_aggregator.assistant(), # Assistant spoken responses and tool context
] ],
) )
task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True)) task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True))
@@ -141,8 +150,6 @@ Your response will be turned into speech so use only simple words and punctuatio
@transport.event_handler("on_app_message") @transport.event_handler("on_app_message")
async def on_app_message(transport, message, sender): async def on_app_message(transport, message, sender):
global anthropic_context
anthropic_context = AnthropicLLMContext.upgrade_to_anthropic(context)
logger.debug(f"Received app message: {message} - {context}") logger.debug(f"Received app message: {message} - {context}")
if not recent_image_frames: if not recent_image_frames:
logger.debug("No image frames to send") logger.debug("No image frames to send")
@@ -157,37 +164,38 @@ Your response will be turned into speech so use only simple words and punctuatio
runner = PipelineRunner() runner = PipelineRunner()
await runner.run(task) await runner.run(task)
def add_message_with_images(c, message, frames=None):
if frames is None:
frames = list(recent_image_frames)
if not frames: def add_message_with_images(c, message, frames=None):
logger.debug("No image frames to send") if frames is None:
return frames = list(recent_image_frames)
# Create content list starting with all images if not frames:
content = [] logger.debug("No image frames to send")
for frame in frames: return
buffer = io.BytesIO()
Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
content.append( # Create content list starting with all images
{ content = []
"type": "image", for frame in frames:
"source": { buffer = io.BytesIO()
"type": "base64", Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
"media_type": "image/jpeg", encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
"data": encoded_image,
},
}
)
# Add text message at the end if provided content.append(
if message: {
content.append({"type": "text", "text": message}) "type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": encoded_image,
},
}
)
c.add_message({"role": "user", "content": content}) # Add text message at the end if provided
if message:
content.append({"type": "text", "text": message})
c.add_message({"role": "user", "content": content})
if __name__ == "__main__": if __name__ == "__main__":