From 0783f5c891badc96146d654d1f2bcbdd99f433af Mon Sep 17 00:00:00 2001
From: oobabooga <112222186+oobabooga@users.noreply.github.com>
Date: Thu, 5 Jun 2025 11:42:12 -0700
Subject: [PATCH] Use the latest format

---
 extensions/openai/completions.py |  2 +-
 modules/chat.py                  | 21 +++++++++++++++++----
 modules/llama_cpp_server.py      | 12 ++++++------
 3 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py
index 2374733a..4e4e310f 100644
--- a/extensions/openai/completions.py
+++ b/extensions/openai/completions.py
@@ -150,7 +150,7 @@ def convert_history(history):
             # Process multimodal content
             processed_content, images = process_multimodal_content(content)
             if images:
-                image_refs = "".join(f"[img-{img['image_id']}]" for img in images)
+                image_refs = "".join("<__media__>" for img in images)
                 processed_content = f"{processed_content} {image_refs}"
 
             user_input = processed_content
diff --git a/modules/chat.py b/modules/chat.py
index 275f28f9..9dc8d1fd 100644
--- a/modules/chat.py
+++ b/modules/chat.py
@@ -225,7 +225,7 @@ def generate_chat_prompt(user_input, state, **kwargs):
                 for attachment in metadata[user_key]["attachments"]:
                     if attachment.get("type") == "image":
                         # Add image reference for multimodal models
-                        image_refs += f"[img-{attachment['image_id']}]"
+                        image_refs += "<__media__>"
                     else:
                         # Handle text/PDF attachments as before
                         filename = attachment.get("name", "file")
@@ -260,7 +260,7 @@ def generate_chat_prompt(user_input, state, **kwargs):
 
             for attachment in metadata[user_key]["attachments"]:
                 if attachment.get("type") == "image":
-                    image_refs += f"[img-{attachment['image_id']}]"
+                    image_refs += "<__media__>"
                 else:
                     filename = attachment.get("name", "file")
                     content = attachment.get("content", "")
@@ -517,17 +517,30 @@ def add_message_attachment(history, row_idx, file_path, is_user=True):
             with open(path, 'rb') as f:
                 image_data = base64.b64encode(f.read()).decode('utf-8')
 
+            # Determine MIME type from extension
+            mime_type_map = {
+                '.jpg': 'image/jpeg',
+                '.jpeg': 'image/jpeg',
+                '.png': 'image/png',
+                '.webp': 'image/webp',
+                '.bmp': 'image/bmp',
+                '.gif': 'image/gif'
+            }
+            mime_type = mime_type_map.get(file_extension, 'image/jpeg')
+
+            # Format as data URL
+            data_url = f"data:{mime_type};base64,{image_data}"
+
             # Generate unique image ID
             image_id = len([att for att in history['metadata'][key]["attachments"] if att.get("type") == "image"]) + 1
 
             attachment = {
                 "name": filename,
                 "type": "image",
-                "image_data": image_data,
+                "image_data": data_url,
                 "image_id": image_id,
                 "file_path": str(path)  # For UI preview
             }
-
         elif file_extension == '.pdf':
             # Process PDF file
             content = extract_pdf_text(path)
diff --git a/modules/llama_cpp_server.py b/modules/llama_cpp_server.py
index 8b4ed7a7..ca1b2c47 100644
--- a/modules/llama_cpp_server.py
+++ b/modules/llama_cpp_server.py
@@ -123,15 +123,15 @@ class LlamaServer:
 
         # Add image data if present
         if 'image_attachments' in state:
-            image_data = []
+            medias = []
             for attachment in state['image_attachments']:
-                image_data.append({
-                    "data": attachment['image_data'],
-                    "id": attachment['image_id']
+                medias.append({
+                    "type": "image",
+                    "data": attachment['image_data']
                 })
 
-            if image_data:
-                payload["image_data"] = image_data
+            if medias:
+                payload["medias"] = medias
 
         return payload