From d803737843ad1fab82263838c439d8ce6f8b6bef Mon Sep 17 00:00:00 2001 From: James Hush Date: Fri, 22 Nov 2024 14:48:29 +0800 Subject: [PATCH] Add messages ahead of time --- examples/foundational/bigly.py | 74 +++++++++---------- src/pipecat/services/openai.py | 128 ++++++++++++++++++++++++++++++--- 2 files changed, 156 insertions(+), 46 deletions(-) diff --git a/examples/foundational/bigly.py b/examples/foundational/bigly.py index 82fd6b5da..f08c7ea02 100644 --- a/examples/foundational/bigly.py +++ b/examples/foundational/bigly.py @@ -359,7 +359,9 @@ async def main(): "description": """"" Used to find information from the knowledge base. Use this tool in the following scenarios: - When the user asks questions about the company. - - If user is not showing interest and you need to convince them by using information about the company + - If you need to convince the user to purchase solar panels. + + IMPORTANT: ALWAYS call this tool after EVERY question the user asks. """, "strict": True, # type: ignore[typeddict-unknown-key] "parameters": { @@ -367,7 +369,7 @@ async def main(): "properties": { "question": { "type": "string", - "description": "User questions or concerns about the Company", + "description": "The question the user asked.", }, "reason": { "type": "string", @@ -479,38 +481,38 @@ async def main(): ), ) - @transport.event_handler("on_call_state_updated") - async def on_call_state_updated(transport, state: str) -> None: - logger.info("Call State Updated, state: {state}", state=state) + # @transport.event_handler("on_call_state_updated") + # async def on_call_state_updated(transport, state: str) -> None: + # logger.info("Call State Updated, state: {state}", state=state) - async def _dialout_retry_handler() -> None: - try: - for i in range(3): - logger.info("Attempting a Dial-Out, Attempt: {attempt}", attempt=i + 1) - await transport.start_dialout( - {"phoneNumber": TO, "video": False} - # {"phoneNumber": TO, "callerId": FROM_DAILY_CALLER_ID, "video": False} - ) - await asyncio.sleep(15) - current_participant_count = transport.participant_counts() - if current_participant_count["present"] >= 2: - return - raise Exception("Unable to perform a dial-out for Daily-Co ROOM") - except Exception as e: - raise e + # async def _dialout_retry_handler() -> None: + # try: + # for i in range(3): + # logger.info("Attempting a Dial-Out, Attempt: {attempt}", attempt=i + 1) + # await transport.start_dialout( + # {"phoneNumber": TO, "video": False} + # # {"phoneNumber": TO, "callerId": FROM_DAILY_CALLER_ID, "video": False} + # ) + # await asyncio.sleep(15) + # current_participant_count = transport.participant_counts() + # if current_participant_count["present"] >= 2: + # return + # raise Exception("Unable to perform a dial-out for Daily-Co ROOM") + # except Exception as e: + # raise e - async def _dialout_task_exception(task: asyncio.Task) -> None: - if task.exception(): - await task.queue_frames([EndFrame()]) - else: - logger.info("Dial-out completed successfully.") + # async def _dialout_task_exception(task: asyncio.Task) -> None: + # if task.exception(): + # await task.queue_frames([EndFrame()]) + # else: + # logger.info("Dial-out completed successfully.") - def _handle_dialout_completion(task: asyncio.Task) -> None: - asyncio.create_task(_dialout_task_exception(task)) + # def _handle_dialout_completion(task: asyncio.Task) -> None: + # asyncio.create_task(_dialout_task_exception(task)) - if state == "joined": - task = transport.input().get_event_loop().create_task(_dialout_retry_handler()) - task.add_done_callback(_handle_dialout_completion) + # if state == "joined": + # task = transport.input().get_event_loop().create_task(_dialout_retry_handler()) + # task.add_done_callback(_handle_dialout_completion) # Event handler for on_first_participant_joined @transport.event_handler("on_first_participant_joined") @@ -519,8 +521,8 @@ async def main(): "First Participant Joined with ID {participant_id}", participant_id=participant["id"], ) - PARTICIPANT_ID[0] = participant["id"] - await transport.capture_participant_transcription(participant["id"]) + # PARTICIPANT_ID[0] = participant["id"] + # await transport.capture_participant_transcription(participant["id"]) @transport.event_handler("on_dialout_error") async def on_dialout_error(transport, cdata) -> None: @@ -528,10 +530,10 @@ async def main(): await task.queue_frames([LLMMessagesFrame(messages)]) # Event handler for on_dialout_answered - @transport.event_handler("on_dialout_answered") - async def on_dialout_answered(transport, cdata) -> None: - logger.info("Dial-Out Answered with data as follow: {data}", data=cdata) - await task.queue_frames([LLMMessagesFrame(messages)]) + # @transport.event_handler("on_dialout_answered") + # async def on_dialout_answered(transport, cdata) -> None: + # logger.info("Dial-Out Answered with data as follow: {data}", data=cdata) + # await task.queue_frames([LLMMessagesFrame(messages)]) # Event handler for on_participant_left @transport.event_handler("on_participant_left") diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index b228f9f4b..318bab113 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -48,13 +48,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import ImageGenService, LLMService, TTSService try: - from openai import ( - NOT_GIVEN, - AsyncOpenAI, - AsyncStream, - BadRequestError, - DefaultAsyncHttpxClient, - ) + from openai import NOT_GIVEN, AsyncOpenAI, AsyncStream, BadRequestError, DefaultAsyncHttpxClient from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -165,6 +159,118 @@ class BaseOpenAILLMService(LLMService): params.update(self._settings["extra"]) + # test_params {'model': 'gpt-4o', 'stream': True, 'messages': [{...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, ...], 'tools': [{...}, {...}], 'tool_choice': NOT_GIVEN, 'stream_options': {'include_usage': True}, 'frequency_penalty': NOT_GIVEN, 'presence_penalty': NOT_GIVEN, 'seed': NOT_GIVEN, 'temperature': NOT_GIVEN, 'top_p': NOT_GIVEN, 'max_tokens': NOT_GIVEN, 'max_completion_tokens': NOT_GIVEN} + + default_messages = [ + { + "role": "system", + "content": "You are a friendly sales person for a solar panel company. Your responses will be converted to audio. Please do not include any special characters in your response other than '!' or '?'. ", + }, + {"role": "user", "content": "Hey there."}, + {"role": "assistant", "content": "Hi! How can I assist you today?"}, + {"role": "user", "content": "I want to learn more about your company."}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_BPxFAYW2i8Ke4iAjS2ODZjwS", + "function": { + "name": "get_knowledge_base", + "arguments": '{"question": "Tell me about your company", "reason": "GENERAL_QUESTION"}', + }, + "type": "function", + } + ], + }, + { + "role": "tool", + "content": '"Sure! Here are some random facts that could be associated with a solar panel company: 1. **History**: The company was founded in 2008 by a group of renewable energy enthusiasts who wanted to make solar power more accessible to homeowners and businesses. 2. **Headquarters**: Their headquarters is a net-zero energy building powered entirely by their own solar panels, showcasing their commitment to sustainability. 3. **Products**: They produce three main types of solar panels: monocrystalline, polycrystalline, and thin-film, catering to different customer needs and budgets. 4. **Innovation**: The company holds patents for advanced solar cell technology that increases efficiency by 20% compared to industry standards. 5. **Global Reach**: They have installed solar systems in over 40 countries and have manufacturing plants on three continents.6. **Community Impact**: For every 100 solar panels sold, they donate a panel to a school or community center in underprivileged areas. 7. **Workforce**: The company employs over 5,000 people, 40% of whom are in research and development roles. 8. **Recognition**: They won the \\u201cGreen Energy Innovator of the Year\\u201d award in 2022 for their work on solar panels made from recycled materials. 9. **Sustainability**: Their panels are designed to last 25+ years and are 95% recyclable at the end of their life cycle. 10. **Customer Perks**: They offer a 25-year warranty and real-time monitoring systems that allow users to track energy production via an app. 11. **Mission Statement**: \\"Empowering the world with clean energy, one panel at a time.\\" 12. **Energy Production**: The combined output of all their installations generates enough electricity to power over 2 million homes annually. 13. **R&D Efforts**: They are actively working on integrating solar panels into everyday items like backpacks and electric vehicles. 14. **Solar Farms**: The company has partnered with governments to develop large-scale solar farms, including one that spans over 10,000 acres. 15. **Future Goals**: By 2030, they aim to make solar power the most affordable energy source worldwide. Would you like these customized for a specific scenario?"', + "tool_call_id": "call_BPxFAYW2i8Ke4iAjS2ODZjwS", + }, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + {"role": "assistant", "content": "Sure! Our company was founded in"}, + {"role": "user", "content": "Hi. Can you tell me more about the company?"}, + ] + + params['messages'] = default_messages + params['messages'] + + # print(f"_____openai.py get_chat_completions * params: {params['messages']}") try: chunks = await self._client.chat.completions.create(**params) except aiohttp.ClientError as e: @@ -209,7 +315,7 @@ class BaseOpenAILLMService(LLMService): function_name = "" arguments = "" tool_call_id = "" - + print(f"_____openai.py * _process_context: tool_call_id {tool_call_id}") await self.start_ttfb_metrics() @@ -248,7 +354,9 @@ class BaseOpenAILLMService(LLMService): # yield a frame containing the function name and the arguments. tool_call = chunk.choices[0].delta.tool_calls[0] - print(f"___________________________openai.py * tool_call_id: {tool_call_id} {tool_call.function.arguments}") + print( + f"___________________________openai.py * tool_call_id: {tool_call_id} {tool_call.function.arguments}" + ) if tool_call.index != func_idx: functions_list.append(function_name) arguments_list.append(arguments) @@ -540,7 +648,7 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): self._pending_image_frame_message = frame await self._push_aggregation() # else: - # print(f"___<2>__openai.py * OpenAIAssistantContextAggregator process_frame frame : {frame}") + # print(f"___<2>__openai.py * OpenAIAssistantContextAggregator process_frame frame : {frame}") async def _push_aggregation(self): if not (