From 93c9e219ce01d8446a40e4c262bea0bf5bcab6c0 Mon Sep 17 00:00:00 2001
From: Kwindla Hultman Kramer <kwindla@gmail.com>
Date: Wed, 16 Oct 2024 12:40:20 -0700
Subject: [PATCH] fix for message handling bug on initialization

---
 .../foundational/19-openai-realtime-beta.py   | 16 +++++++++++++++-
 .../services/openai_realtime_beta/context.py  | 19 +++++++++++++++++--
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py
index a361987f3..6a48de56d 100644
--- a/examples/foundational/19-openai-realtime-beta.py
+++ b/examples/foundational/19-openai-realtime-beta.py
@@ -124,7 +124,21 @@ Remember, your responses should be short. Just one or two sentences, usually."""
         # llm.register_function(None, fetch_weather_from_api)
         llm.register_function("get_current_weather", fetch_weather_from_api)
 
-        context = OpenAILLMContext([{"role": "user", "content": "Say hello!"}], tools)
+        context = OpenAILLMContext(
+            [{"role": "user", "content": "Say hello!"}],
+            # [{"role": "user", "content": [{"type": "text", "text": "Say hello!"}]}],
+            #     [
+            #         {
+            #             "role": "user",
+            #             "content": [
+            #                 {"type": "text", "text": "Say"},
+            #                 {"type": "text", "text": "yo what's up!"},
+            #             ],
+            #         }
+            #     ],
+            tools,
+        )
+
         context_aggregator = llm.create_context_aggregator(context)
 
         pipeline = Pipeline(
diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py
index 1c8292085..2b6ff968f 100644
--- a/src/pipecat/services/openai_realtime_beta/context.py
+++ b/src/pipecat/services/openai_realtime_beta/context.py
@@ -45,9 +45,24 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext):
 
     # todo
     #   - finish implementing all frames
-    #   - add message conversion functions to OpenAILLMContext base class
 
     def from_standard_message(self, message):
+        if message.get("role") == "user":
+            content = message.get("content")
+            if isinstance(message.get("content"), list):
+                content = ""
+                for c in message.get("content"):
+                    if c.get("type") == "text":
+                        content += " " + c.get("text")
+                    else:
+                        logger.error(
+                            f"Unhandled content type in context message: {c.get('type')} - {message}"
+                        )
+            return events.ConversationItem(
+                role="user",
+                type="message",
+                content=[events.ItemContent(type="input_text", text=content)],
+            )
         if message.get("role") == "assistant" and message.get("tool_calls"):
             tc = message.get("tool_calls")[0]
             return events.ConversationItem(
@@ -83,7 +98,7 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext):
 
         # If we have just a single "user" item, we can just send it normally
         if len(messages) == 1 and messages[0].get("role") == "user":
-            return messages
+            return [self.from_standard_message(messages[0])]
 
         # Otherwise, let's pack everything into a single "user" message with a bit of
         # explanation for the LLM