From 912751e268b3dae59c1d3b47fa1c638b51479afa Mon Sep 17 00:00:00 2001
From: Dmitry Stepanov <stepdi@gmail.com>
Date: Tue, 11 Jun 2024 19:10:19 +0300
Subject: [PATCH] Ollama vision support (#22734)

**Description:** Ollama vision with messages in OpenAI-style support `{
"image_url": { "url": ... } }`
**Issue:** #22460

Added flexible solution for ChatOllama to support chat messages with
images. Works when you provide either `image_url` as a string or as a
dict with "url" inside (like OpenAI does). So it makes available to use
tuples with `ChatPromptTemplate.from_messages()`

---------

Co-authored-by: Bagatur <baskaryan@gmail.com>
---
 .../langchain_community/chat_models/ollama.py | 28 +++++++++++++------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/libs/community/langchain_community/chat_models/ollama.py b/libs/community/langchain_community/chat_models/ollama.py
index b0fd0944ee..ca936a1c51 100644
--- a/libs/community/langchain_community/chat_models/ollama.py
+++ b/libs/community/langchain_community/chat_models/ollama.py
@@ -137,18 +137,28 @@ class ChatOllama(BaseChatModel, _OllamaCommon):
                     if content_part.get("type") == "text":
                         content += f"\n{content_part['text']}"
                     elif content_part.get("type") == "image_url":
-                        if isinstance(content_part.get("image_url"), str):
-                            image_url_components = content_part["image_url"].split(",")
-                            # Support data:image/jpeg;base64,<image> format
-                            # and base64 strings
-                            if len(image_url_components) > 1:
-                                images.append(image_url_components[1])
-                            else:
-                                images.append(image_url_components[0])
+                        image_url = None
+                        temp_image_url = content_part.get("image_url")
+                        if isinstance(temp_image_url, str):
+                            image_url = content_part["image_url"]
+                        elif (
+                            isinstance(temp_image_url, dict) and "url" in temp_image_url
+                        ):
+                            image_url = temp_image_url
                         else:
                             raise ValueError(
-                                "Only string image_url " "content parts are supported."
+                                "Only string image_url or dict with string 'url' "
+                                "inside content parts are supported."
                             )
+
+                        image_url_components = image_url.split(",")
+                        # Support data:image/jpeg;base64,<image> format
+                        # and base64 strings
+                        if len(image_url_components) > 1:
+                            images.append(image_url_components[1])
+                        else:
+                            images.append(image_url_components[0])
+
                     else:
                         raise ValueError(
                             "Unsupported message content type. "