mirror of https://github.com/hwchase17/langchain
Fix: Nested Dicts Handling of Document Metadata (#9880)
## Description When the `MultiQueryRetriever` is used to get the list of documents relevant according to a query, inside a vector store, and at least one of these contain metadata with nested dictionaries, a `TypeError: unhashable type: 'dict'` exception is thrown. This is caused by the `unique_union` function which, to guarantee the uniqueness of the returned documents, tries, unsuccessfully, to hash the nested dictionaries and use them as a part of key. ```python unique_documents_dict = { (doc.page_content, tuple(sorted(doc.metadata.items()))): doc for doc in documents } ``` ## Issue #9872 (MultiQueryRetriever (get_relevant_documents) raises TypeError: unhashable type: 'dict' with dic metadata) ## Solution A possible solution is to dump the metadata dict to a string and use it as a part of hashed key. ```python unique_documents_dict = { (doc.page_content, json.dumps(doc.metadata, sort_keys=True)): doc for doc in documents } ``` --------- Co-authored-by: Bagatur <baskaryan@gmail.com>pull/10155/head
parent
a52fe9528e
commit
00a7c31ffd
@ -0,0 +1,40 @@
|
||||
from typing import List
|
||||
|
||||
import pytest as pytest
|
||||
|
||||
from langchain.retrievers.multi_query import _unique_documents
|
||||
from langchain.schema import Document
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"documents,expected",
|
||||
[
|
||||
([], []),
|
||||
([Document(page_content="foo")], [Document(page_content="foo")]),
|
||||
([Document(page_content="foo")] * 2, [Document(page_content="foo")]),
|
||||
(
|
||||
[Document(page_content="foo", metadata={"bar": "baz"})] * 2,
|
||||
[Document(page_content="foo", metadata={"bar": "baz"})],
|
||||
),
|
||||
(
|
||||
[Document(page_content="foo", metadata={"bar": [1, 2]})] * 2,
|
||||
[Document(page_content="foo", metadata={"bar": [1, 2]})],
|
||||
),
|
||||
(
|
||||
[Document(page_content="foo", metadata={"bar": {1, 2}})] * 2,
|
||||
[Document(page_content="foo", metadata={"bar": {1, 2}})],
|
||||
),
|
||||
(
|
||||
[
|
||||
Document(page_content="foo", metadata={"bar": [1, 2]}),
|
||||
Document(page_content="foo", metadata={"bar": [2, 1]}),
|
||||
],
|
||||
[
|
||||
Document(page_content="foo", metadata={"bar": [1, 2]}),
|
||||
Document(page_content="foo", metadata={"bar": [2, 1]}),
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test__unique_documents(documents: List[Document], expected: List[Document]) -> None:
|
||||
assert _unique_documents(documents) == expected
|
Loading…
Reference in New Issue