From 6da2246215c61ad366354398ecf1334ccce28470 Mon Sep 17 00:00:00 2001 From: Takuya Igei <28337009+itok01@users.noreply.github.com> Date: Sat, 23 Dec 2023 06:19:09 +0900 Subject: [PATCH] Add support Vertex AI Gemini uses a public image URL (#14949) ## What Since `langchain_google_genai.ChatGoogleGenerativeAI` supported A public image URL, we add to support it in `langchain.chat_models.ChatVertexAI` as well. ### Example ```py from langchain.chat_models.vertexai import ChatVertexAI from langchain_core.messages import HumanMessage llm = ChatVertexAI(model_name="gemini-pro-vision") image_message = { "type": "image_url", "image_url": { "url": "https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png", }, } text_message = { "type": "text", "text": "What is shown in this image?", } message = HumanMessage(content=[text_message, image_message]) output = llm([message]) print(output.content) ``` ## Refs - https://python.langchain.com/docs/integrations/llms/google_vertex_ai_palm - https://python.langchain.com/docs/integrations/chat/google_generative_ai --- .../llms/google_vertex_ai_palm.ipynb | 29 +++++++++++++++++++ .../chat_models/vertexai.py | 15 ++++++++++ 2 files changed, 44 insertions(+) diff --git a/docs/docs/integrations/llms/google_vertex_ai_palm.ipynb b/docs/docs/integrations/llms/google_vertex_ai_palm.ipynb index 40b64f43c4..f93a254475 100644 --- a/docs/docs/integrations/llms/google_vertex_ai_palm.ipynb +++ b/docs/docs/integrations/llms/google_vertex_ai_palm.ipynb @@ -539,6 +539,35 @@ "print(output2.content)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also use the public image URL:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "image_message = {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png\",\n", + " },\n", + "}\n", + "text_message = {\n", + " \"type\": \"text\",\n", + " \"text\": \"What is shown in this image?\",\n", + "}\n", + "message = HumanMessage(content=[text_message, image_message])\n", + "\n", + "output = llm([message])\n", + "print(output.content)" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/libs/community/langchain_community/chat_models/vertexai.py b/libs/community/langchain_community/chat_models/vertexai.py index 8cc493a1fc..32b126ef44 100644 --- a/libs/community/langchain_community/chat_models/vertexai.py +++ b/libs/community/langchain_community/chat_models/vertexai.py @@ -6,7 +6,9 @@ import logging import re from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast +from urllib.parse import urlparse +import requests from langchain_core.callbacks import ( AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun, @@ -87,6 +89,15 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory: return chat_history +def _is_url(s: str) -> bool: + try: + result = urlparse(s) + return all([result.scheme, result.netloc]) + except Exception as e: + logger.debug(f"Unable to parse URL: {e}") + return False + + def _parse_chat_history_gemini( history: List[BaseMessage], project: Optional[str] ) -> List["Content"]: @@ -118,6 +129,10 @@ def _parse_chat_history_gemini( "data:image/;base64,." ) image = Image.from_bytes(base64.b64decode(encoded)) + elif _is_url(path): + response = requests.get(path) + response.raise_for_status() + image = Image.from_bytes(response.content) else: image = Image.load_from_file(path) else: