mirror of https://github.com/hwchase17/langchain
Adding compatibility for OllamaFunctions with ImagePromptTemplate (#24499)
- [ ] **PR title**: "experimental: Adding compatibility for OllamaFunctions with ImagePromptTemplate" - [ ] **PR message**: - **Description:** Removes the outdated `_convert_messages_to_ollama_messages` method override in the `OllamaFunctions` class to ensure that ollama multimodal models can be invoked with an image. - **Issue:** #24174 --------- Co-authored-by: Joel Akeret <joel.akeret@ti&m.com> Co-authored-by: Isaac Francisco <78627776+isahers1@users.noreply.github.com> Co-authored-by: isaac hershenson <ihershenson@hmc.edu>pull/24638/head
parent
8f3c052db1
commit
acfce30017
@ -0,0 +1,30 @@
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.pydantic_v1 import BaseModel
|
||||
|
||||
from langchain_experimental.llms.ollama_functions import OllamaFunctions
|
||||
|
||||
|
||||
class Schema(BaseModel):
|
||||
pass
|
||||
|
||||
|
||||
@patch.object(OllamaFunctions, "_create_stream")
|
||||
def test_convert_image_prompt(
|
||||
_create_stream_mock: Any,
|
||||
) -> None:
|
||||
response = {"message": {"content": '{"tool": "Schema", "tool_input": {}}'}}
|
||||
_create_stream_mock.return_value = [json.dumps(response)]
|
||||
|
||||
prompt = ChatPromptTemplate.from_messages(
|
||||
[("human", [{"image_url": "data:image/jpeg;base64,{image_url}"}])]
|
||||
)
|
||||
|
||||
lmm = prompt | OllamaFunctions().with_structured_output(schema=Schema)
|
||||
|
||||
schema_instance = lmm.invoke(dict(image_url=""))
|
||||
|
||||
assert schema_instance is not None
|
Loading…
Reference in New Issue