wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add messages ahead of time

Browse Source
This commit is contained in:
James Hush
2024-11-22 14:48:29 +08:00
parent 0c5105e334
commit d803737843
2 changed files with 156 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
examples/foundational/bigly.py
View File

@@ -359,7 +359,9 @@ async def main():
"description": """"" "description": """""
Used to find information from the knowledge base. Use this tool in the following scenarios: Used to find information from the knowledge base. Use this tool in the following scenarios:
- When the user asks questions about the company. - When the user asks questions about the company.
- If user is not showing interest and you need to convince them by using information about the company - If you need to convince the user to purchase solar panels.
IMPORTANT: ALWAYS call this tool after EVERY question the user asks.
""", """,
"strict": True, # type: ignore[typeddict-unknown-key] "strict": True, # type: ignore[typeddict-unknown-key]
"parameters": { "parameters": {
@@ -367,7 +369,7 @@ async def main():
"properties": { "properties": {
"question": { "question": {
"type": "string", "type": "string",
"description": "User questions or concerns about the Company", "description": "The question the user asked.",
}, },
"reason": { "reason": {
"type": "string", "type": "string",
@@ -479,38 +481,38 @@ async def main():
), ),
) )
@transport.event_handler("on_call_state_updated") # @transport.event_handler("on_call_state_updated")
async def on_call_state_updated(transport, state: str) -> None: # async def on_call_state_updated(transport, state: str) -> None:
logger.info("Call State Updated, state: {state}", state=state) # logger.info("Call State Updated, state: {state}", state=state)
async def _dialout_retry_handler() -> None: # async def _dialout_retry_handler() -> None:
try: # try:
for i in range(3): # for i in range(3):
logger.info("Attempting a Dial-Out, Attempt: {attempt}", attempt=i + 1) # logger.info("Attempting a Dial-Out, Attempt: {attempt}", attempt=i + 1)
await transport.start_dialout( # await transport.start_dialout(
{"phoneNumber": TO, "video": False} # {"phoneNumber": TO, "video": False}
# {"phoneNumber": TO, "callerId": FROM_DAILY_CALLER_ID, "video": False} # # {"phoneNumber": TO, "callerId": FROM_DAILY_CALLER_ID, "video": False}
) # )
await asyncio.sleep(15) # await asyncio.sleep(15)
current_participant_count = transport.participant_counts() # current_participant_count = transport.participant_counts()
if current_participant_count["present"] >= 2: # if current_participant_count["present"] >= 2:
return # return
raise Exception("Unable to perform a dial-out for Daily-Co ROOM") # raise Exception("Unable to perform a dial-out for Daily-Co ROOM")
except Exception as e: # except Exception as e:
raise e # raise e
async def _dialout_task_exception(task: asyncio.Task) -> None: # async def _dialout_task_exception(task: asyncio.Task) -> None:
if task.exception(): # if task.exception():
await task.queue_frames([EndFrame()]) # await task.queue_frames([EndFrame()])
else: # else:
logger.info("Dial-out completed successfully.") # logger.info("Dial-out completed successfully.")
def _handle_dialout_completion(task: asyncio.Task) -> None: # def _handle_dialout_completion(task: asyncio.Task) -> None:
asyncio.create_task(_dialout_task_exception(task)) # asyncio.create_task(_dialout_task_exception(task))
if state == "joined": # if state == "joined":
task = transport.input().get_event_loop().create_task(_dialout_retry_handler()) # task = transport.input().get_event_loop().create_task(_dialout_retry_handler())
task.add_done_callback(_handle_dialout_completion) # task.add_done_callback(_handle_dialout_completion)
# Event handler for on_first_participant_joined # Event handler for on_first_participant_joined
@transport.event_handler("on_first_participant_joined") @transport.event_handler("on_first_participant_joined")
@@ -519,8 +521,8 @@ async def main():
"First Participant Joined with ID {participant_id}", "First Participant Joined with ID {participant_id}",
participant_id=participant["id"], participant_id=participant["id"],
) )
PARTICIPANT_ID[0] = participant["id"] # PARTICIPANT_ID[0] = participant["id"]
await transport.capture_participant_transcription(participant["id"]) # await transport.capture_participant_transcription(participant["id"])
@transport.event_handler("on_dialout_error") @transport.event_handler("on_dialout_error")
async def on_dialout_error(transport, cdata) -> None: async def on_dialout_error(transport, cdata) -> None:
@@ -528,10 +530,10 @@ async def main():
await task.queue_frames([LLMMessagesFrame(messages)]) await task.queue_frames([LLMMessagesFrame(messages)])
# Event handler for on_dialout_answered # Event handler for on_dialout_answered
@transport.event_handler("on_dialout_answered") # @transport.event_handler("on_dialout_answered")
async def on_dialout_answered(transport, cdata) -> None: # async def on_dialout_answered(transport, cdata) -> None:
logger.info("Dial-Out Answered with data as follow: {data}", data=cdata) # logger.info("Dial-Out Answered with data as follow: {data}", data=cdata)
await task.queue_frames([LLMMessagesFrame(messages)]) # await task.queue_frames([LLMMessagesFrame(messages)])
# Event handler for on_participant_left # Event handler for on_participant_left
@transport.event_handler("on_participant_left") @transport.event_handler("on_participant_left")

126
src/pipecat/services/openai.py
View File

@@ -48,13 +48,7 @@ from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.ai_services import ImageGenService, LLMService, TTSService from pipecat.services.ai_services import ImageGenService, LLMService, TTSService
try: try:
from openai import ( from openai import NOT_GIVEN, AsyncOpenAI, AsyncStream, BadRequestError, DefaultAsyncHttpxClient
NOT_GIVEN,
AsyncOpenAI,
AsyncStream,
BadRequestError,
DefaultAsyncHttpxClient,
)
from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
except ModuleNotFoundError as e: except ModuleNotFoundError as e:
logger.error(f"Exception: {e}") logger.error(f"Exception: {e}")
@@ -165,6 +159,118 @@ class BaseOpenAILLMService(LLMService):
params.update(self._settings["extra"]) params.update(self._settings["extra"])
# test_params {'model': 'gpt-4o', 'stream': True, 'messages': [{...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, {...}, ...], 'tools': [{...}, {...}], 'tool_choice': NOT_GIVEN, 'stream_options': {'include_usage': True}, 'frequency_penalty': NOT_GIVEN, 'presence_penalty': NOT_GIVEN, 'seed': NOT_GIVEN, 'temperature': NOT_GIVEN, 'top_p': NOT_GIVEN, 'max_tokens': NOT_GIVEN, 'max_completion_tokens': NOT_GIVEN}
default_messages = [
{
"role": "system",
"content": "You are a friendly sales person for a solar panel company. Your responses will be converted to audio. Please do not include any special characters in your response other than '!' or '?'. ",
},
{"role": "user", "content": "Hey there."},
{"role": "assistant", "content": "Hi! How can I assist you today?"},
{"role": "user", "content": "I want to learn more about your company."},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_BPxFAYW2i8Ke4iAjS2ODZjwS",
"function": {
"name": "get_knowledge_base",
"arguments": '{"question": "Tell me about your company", "reason": "GENERAL_QUESTION"}',
},
"type": "function",
}
],
},
{
"role": "tool",
"content": '"Sure! Here are some random facts that could be associated with a solar panel company: 1. **History**: The company was founded in 2008 by a group of renewable energy enthusiasts who wanted to make solar power more accessible to homeowners and businesses. 2. **Headquarters**: Their headquarters is a net-zero energy building powered entirely by their own solar panels, showcasing their commitment to sustainability. 3. **Products**: They produce three main types of solar panels: monocrystalline, polycrystalline, and thin-film, catering to different customer needs and budgets. 4. **Innovation**: The company holds patents for advanced solar cell technology that increases efficiency by 20% compared to industry standards. 5. **Global Reach**: They have installed solar systems in over 40 countries and have manufacturing plants on three continents.6. **Community Impact**: For every 100 solar panels sold, they donate a panel to a school or community center in underprivileged areas. 7. **Workforce**: The company employs over 5,000 people, 40% of whom are in research and development roles. 8. **Recognition**: They won the \\u201cGreen Energy Innovator of the Year\\u201d award in 2022 for their work on solar panels made from recycled materials. 9. **Sustainability**: Their panels are designed to last 25+ years and are 95% recyclable at the end of their life cycle. 10. **Customer Perks**: They offer a 25-year warranty and real-time monitoring systems that allow users to track energy production via an app. 11. **Mission Statement**: \\"Empowering the world with clean energy, one panel at a time.\\" 12. **Energy Production**: The combined output of all their installations generates enough electricity to power over 2 million homes annually. 13. **R&D Efforts**: They are actively working on integrating solar panels into everyday items like backpacks and electric vehicles. 14. **Solar Farms**: The company has partnered with governments to develop large-scale solar farms, including one that spans over 10,000 acres. 15. **Future Goals**: By 2030, they aim to make solar power the most affordable energy source worldwide. Would you like these customized for a specific scenario?"',
"tool_call_id": "call_BPxFAYW2i8Ke4iAjS2ODZjwS",
},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
{"role": "assistant", "content": "Sure! Our company was founded in"},
{"role": "user", "content": "Hi. Can you tell me more about the company?"},
]
params['messages'] = default_messages + params['messages']
# print(f"_____openai.py get_chat_completions * params: {params['messages']}")
try: try:
chunks = await self._client.chat.completions.create(**params) chunks = await self._client.chat.completions.create(**params)
except aiohttp.ClientError as e: except aiohttp.ClientError as e:
@@ -248,7 +354,9 @@ class BaseOpenAILLMService(LLMService):
# yield a frame containing the function name and the arguments. # yield a frame containing the function name and the arguments.
tool_call = chunk.choices[0].delta.tool_calls[0] tool_call = chunk.choices[0].delta.tool_calls[0]
print(f"___________________________openai.py * tool_call_id: {tool_call_id} {tool_call.function.arguments}") print(
f"___________________________openai.py * tool_call_id: {tool_call_id} {tool_call.function.arguments}"
)
if tool_call.index != func_idx: if tool_call.index != func_idx:
functions_list.append(function_name) functions_list.append(function_name)
arguments_list.append(arguments) arguments_list.append(arguments)
@@ -540,7 +648,7 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
self._pending_image_frame_message = frame self._pending_image_frame_message = frame
await self._push_aggregation() await self._push_aggregation()
# else: # else:
# print(f"___<2>__openai.py * OpenAIAssistantContextAggregator process_frame frame : {frame}") # print(f"___<2>__openai.py * OpenAIAssistantContextAggregator process_frame frame : {frame}")
async def _push_aggregation(self): async def _push_aggregation(self):
if not ( if not (