Add support Vertex AI Gemini uses a public image URL (#14949)

## What

Since `langchain_google_genai.ChatGoogleGenerativeAI` supported A public
image URL, we add to support it in `langchain.chat_models.ChatVertexAI`
as well.

### Example

```py
from langchain.chat_models.vertexai import ChatVertexAI
from langchain_core.messages import HumanMessage

llm = ChatVertexAI(model_name="gemini-pro-vision")
image_message = {
    "type": "image_url",
    "image_url": {
        "url": "https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png",
    },
}
text_message = {
    "type": "text",
    "text": "What is shown in this image?",
}
message = HumanMessage(content=[text_message, image_message])

output = llm([message])
print(output.content)
```

## Refs

-
https://python.langchain.com/docs/integrations/llms/google_vertex_ai_palm
-
https://python.langchain.com/docs/integrations/chat/google_generative_ai
pull/15078/head
Takuya Igei 7 months ago committed by GitHub
parent affa3e755a
commit 6da2246215
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -539,6 +539,35 @@
"print(output2.content)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can also use the public image URL:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"image_message = {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": \"https://python.langchain.com/assets/images/cell-18-output-1-0c7fb8b94ff032d51bfe1880d8370104.png\",\n",
" },\n",
"}\n",
"text_message = {\n",
" \"type\": \"text\",\n",
" \"text\": \"What is shown in this image?\",\n",
"}\n",
"message = HumanMessage(content=[text_message, image_message])\n",
"\n",
"output = llm([message])\n",
"print(output.content)"
]
},
{
"cell_type": "markdown",
"metadata": {

@ -6,7 +6,9 @@ import logging
import re
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast
from urllib.parse import urlparse
import requests
from langchain_core.callbacks import (
AsyncCallbackManagerForLLMRun,
CallbackManagerForLLMRun,
@ -87,6 +89,15 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory:
return chat_history
def _is_url(s: str) -> bool:
try:
result = urlparse(s)
return all([result.scheme, result.netloc])
except Exception as e:
logger.debug(f"Unable to parse URL: {e}")
return False
def _parse_chat_history_gemini(
history: List[BaseMessage], project: Optional[str]
) -> List["Content"]:
@ -118,6 +129,10 @@ def _parse_chat_history_gemini(
"data:image/<image_type>;base64,<base64_encoded_image>."
)
image = Image.from_bytes(base64.b64decode(encoded))
elif _is_url(path):
response = requests.get(path)
response.raise_for_status()
image = Image.from_bytes(response.content)
else:
image = Image.load_from_file(path)
else:

Loading…
Cancel
Save