core[patch]: extract input variables for path and detail keys in order to format an ImagePromptTemplate (#22613)

- Description: Add support for `path` and `detail` keys in
`ImagePromptTemplate`. Previously, only variables associated with the
`url` key were considered. This PR allows for the inclusion of a local
image path and a detail parameter as input to the format method.
- Issues:
    - fixes #20820 
    - related to #22024 
- Dependencies: None
- Twitter handle: @DeschampsTho5

---------

Co-authored-by: tdeschamps <tdeschamps@kameleoon.com>
Co-authored-by: Eugene Yurtsev <eyurtsev@gmail.com>
Co-authored-by: Eugene Yurtsev <eugene@langchain.dev>
This commit is contained in:
Théo Deschamps 2024-07-03 11:58:42 -07:00 committed by GitHub
parent a4798802ef
commit 39b19cf764
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 70 additions and 8 deletions

View File

@ -473,6 +473,7 @@ class _StringImageMessagePromptTemplate(BaseMessagePromptTemplate):
)
elif isinstance(tmpl, dict) and "image_url" in tmpl:
img_template = cast(_ImageTemplateParam, tmpl)["image_url"]
input_variables = []
if isinstance(img_template, str):
vars = get_template_variables(img_template, "f-string")
if vars:
@ -483,20 +484,19 @@ class _StringImageMessagePromptTemplate(BaseMessagePromptTemplate):
f"\nFrom: {tmpl}"
)
input_variables = [vars[0]]
else:
input_variables = None
img_template = {"url": img_template}
img_template_obj = ImagePromptTemplate(
input_variables=input_variables, template=img_template
)
elif isinstance(img_template, dict):
img_template = dict(img_template)
if "url" in img_template:
input_variables = get_template_variables(
img_template["url"], "f-string"
)
else:
input_variables = None
for key in ["url", "path", "detail"]:
if key in img_template:
input_variables.extend(
get_template_variables(
img_template[key], "f-string"
)
)
img_template_obj = ImagePromptTemplate(
input_variables=input_variables, template=img_template
)

View File

@ -1,3 +1,5 @@
import base64
import tempfile
from pathlib import Path
from typing import Any, List, Union
@ -559,6 +561,7 @@ async def test_chat_tmpl_from_messages_multipart_text_with_template() -> None:
async def test_chat_tmpl_from_messages_multipart_image() -> None:
"""Test multipart image URL formatting."""
base64_image = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAA"
other_base64_image = "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAA"
template = ChatPromptTemplate.from_messages(
@ -641,6 +644,65 @@ async def test_chat_tmpl_from_messages_multipart_image() -> None:
assert messages == expected
async def test_chat_tmpl_from_messages_multipart_formatting_with_path() -> None:
"""Verify that we can pass `path` for an image as a variable."""
in_mem = "base64mem"
in_file_data = "base64file01"
with tempfile.NamedTemporaryFile(delete=True, suffix=".jpg") as temp_file:
temp_file.write(base64.b64decode(in_file_data))
temp_file.flush()
template = ChatPromptTemplate.from_messages(
[
("system", "You are an AI assistant named {name}."),
(
"human",
[
{"type": "text", "text": "What's in this image?"},
{
"type": "image_url",
"image_url": "data:image/jpeg;base64,{in_mem}",
},
{
"type": "image_url",
"image_url": {"path": "{file_path}"},
},
],
),
]
)
expected = [
SystemMessage(content="You are an AI assistant named R2D2."),
HumanMessage(
content=[
{"type": "text", "text": "What's in this image?"},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{in_mem}"},
},
{
"type": "image_url",
"image_url": {"url": f"data:image/jpeg;base64,{in_file_data}"},
},
]
),
]
messages = template.format_messages(
name="R2D2",
in_mem=in_mem,
file_path=temp_file.name,
)
assert messages == expected
messages = await template.aformat_messages(
name="R2D2",
in_mem=in_mem,
file_path=temp_file.name,
)
assert messages == expected
def test_messages_placeholder() -> None:
prompt = MessagesPlaceholder("history")
with pytest.raises(KeyError):