langchain/libs/community/tests/integration_tests/retrievers/docarray/test_backends.py

from typing import Any

import pytest

from langchain_community.retrievers import DocArrayRetriever
from tests.integration_tests.retrievers.docarray.fixtures import (  # noqa: F401
    init_elastic,
    init_hnsw,
    init_in_memory,
    init_qdrant,
    init_weaviate,
)


@pytest.mark.parametrize(
    "backend",
    ["init_hnsw", "init_in_memory", "init_qdrant", "init_elastic", "init_weaviate"],
)
def test_backends(request: Any, backend: Any) -> None:
    index, filter_query, embeddings = request.getfixturevalue(backend)

    # create a retriever
    retriever = DocArrayRetriever(
        index=index,
        embeddings=embeddings,
        search_field="title_embedding",
        content_field="title",
    )

    docs = retriever.get_relevant_documents("my docs")

    assert len(docs) == 1
    assert "My document" in docs[0].page_content
    assert "id" in docs[0].metadata and "year" in docs[0].metadata
    assert "other_emb" not in docs[0].metadata

    # create a retriever with filters
    retriever = DocArrayRetriever(
        index=index,
        embeddings=embeddings,
        search_field="title_embedding",
        content_field="title",
        filters=filter_query,
    )

    docs = retriever.get_relevant_documents("my docs")

    assert len(docs) == 1
    assert "My document" in docs[0].page_content
    assert "id" in docs[0].metadata and "year" in docs[0].metadata
    assert "other_emb" not in docs[0].metadata
    assert docs[0].metadata["year"] <= 90

    # create a retriever with MMR search
    retriever = DocArrayRetriever(
        index=index,
        embeddings=embeddings,
        search_field="title_embedding",
        search_type="mmr",
        content_field="title",
        filters=filter_query,
    )

    docs = retriever.get_relevant_documents("my docs")

    assert len(docs) == 1
    assert "My document" in docs[0].page_content
    assert "id" in docs[0].metadata and "year" in docs[0].metadata
    assert "other_emb" not in docs[0].metadata
    assert docs[0].metadata["year"] <= 90
DocArray as a Retriever (#6031) ## DocArray as a Retriever [DocArray](https://github.com/docarray/docarray) is an open-source tool for managing your multi-modal data. It offers flexibility to store and search through your data using various document index backends. This PR introduces `DocArrayRetriever` - which works with any available backend and serves as a retriever for Langchain apps. Also, I added 2 notebooks: DocArray Backends - intro to all 5 currently supported backends, how to initialize, index, and use them as a retriever DocArray Usage - showcasing what additional search parameters you can pass to create versatile retrievers Example: ```python from docarray.index import InMemoryExactNNIndex from docarray import BaseDoc, DocList from docarray.typing import NdArray from langchain.embeddings.openai import OpenAIEmbeddings from langchain.retrievers import DocArrayRetriever # define document schema class MyDoc(BaseDoc): description: str description_embedding: NdArray[1536] embeddings = OpenAIEmbeddings() # create documents descriptions = ["description 1", "description 2"] desc_embeddings = embeddings.embed_documents(texts=descriptions) docs = DocList[MyDoc]( [ MyDoc(description=desc, description_embedding=embedding) for desc, embedding in zip(descriptions, desc_embeddings) ] ) # initialize document index with data db = InMemoryExactNNIndex[MyDoc](docs) # create a retriever retriever = DocArrayRetriever( index=db, embeddings=embeddings, search_field="description_embedding", content_field="description", ) # find the relevant document doc = retriever.get_relevant_documents("action movies") print(doc) ``` #### Who can review? @dev2049 --------- Signed-off-by: jupyterjazz <saba.sturua@jina.ai> 1 year ago			`from typing import Any`

			`import pytest`

community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463) Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes 10 months ago			`from langchain_community.retrievers import DocArrayRetriever`
DocArray as a Retriever (#6031) ## DocArray as a Retriever [DocArray](https://github.com/docarray/docarray) is an open-source tool for managing your multi-modal data. It offers flexibility to store and search through your data using various document index backends. This PR introduces `DocArrayRetriever` - which works with any available backend and serves as a retriever for Langchain apps. Also, I added 2 notebooks: DocArray Backends - intro to all 5 currently supported backends, how to initialize, index, and use them as a retriever DocArray Usage - showcasing what additional search parameters you can pass to create versatile retrievers Example: ```python from docarray.index import InMemoryExactNNIndex from docarray import BaseDoc, DocList from docarray.typing import NdArray from langchain.embeddings.openai import OpenAIEmbeddings from langchain.retrievers import DocArrayRetriever # define document schema class MyDoc(BaseDoc): description: str description_embedding: NdArray[1536] embeddings = OpenAIEmbeddings() # create documents descriptions = ["description 1", "description 2"] desc_embeddings = embeddings.embed_documents(texts=descriptions) docs = DocList[MyDoc]( [ MyDoc(description=desc, description_embedding=embedding) for desc, embedding in zip(descriptions, desc_embeddings) ] ) # initialize document index with data db = InMemoryExactNNIndex[MyDoc](docs) # create a retriever retriever = DocArrayRetriever( index=db, embeddings=embeddings, search_field="description_embedding", content_field="description", ) # find the relevant document doc = retriever.get_relevant_documents("action movies") print(doc) ``` #### Who can review? @dev2049 --------- Signed-off-by: jupyterjazz <saba.sturua@jina.ai> 1 year ago			`from tests.integration_tests.retrievers.docarray.fixtures import ( # noqa: F401`
			`init_elastic,`
			`init_hnsw,`
			`init_in_memory,`
			`init_qdrant,`
			`init_weaviate,`
			`)`


			`@pytest.mark.parametrize(`
			`"backend",`
			`["init_hnsw", "init_in_memory", "init_qdrant", "init_elastic", "init_weaviate"],`
			`)`
scheduled tests GHA (#8879) Adding scheduled daily GHA that runs marked integration tests. To start just marking some tests in test_openai 1 year ago			`def test_backends(request: Any, backend: Any) -> None:`
DocArray as a Retriever (#6031) ## DocArray as a Retriever [DocArray](https://github.com/docarray/docarray) is an open-source tool for managing your multi-modal data. It offers flexibility to store and search through your data using various document index backends. This PR introduces `DocArrayRetriever` - which works with any available backend and serves as a retriever for Langchain apps. Also, I added 2 notebooks: DocArray Backends - intro to all 5 currently supported backends, how to initialize, index, and use them as a retriever DocArray Usage - showcasing what additional search parameters you can pass to create versatile retrievers Example: ```python from docarray.index import InMemoryExactNNIndex from docarray import BaseDoc, DocList from docarray.typing import NdArray from langchain.embeddings.openai import OpenAIEmbeddings from langchain.retrievers import DocArrayRetriever # define document schema class MyDoc(BaseDoc): description: str description_embedding: NdArray[1536] embeddings = OpenAIEmbeddings() # create documents descriptions = ["description 1", "description 2"] desc_embeddings = embeddings.embed_documents(texts=descriptions) docs = DocList[MyDoc]( [ MyDoc(description=desc, description_embedding=embedding) for desc, embedding in zip(descriptions, desc_embeddings) ] ) # initialize document index with data db = InMemoryExactNNIndex[MyDoc](docs) # create a retriever retriever = DocArrayRetriever( index=db, embeddings=embeddings, search_field="description_embedding", content_field="description", ) # find the relevant document doc = retriever.get_relevant_documents("action movies") print(doc) ``` #### Who can review? @dev2049 --------- Signed-off-by: jupyterjazz <saba.sturua@jina.ai> 1 year ago			`index, filter_query, embeddings = request.getfixturevalue(backend)`

			`# create a retriever`
			`retriever = DocArrayRetriever(`
			`index=index,`
			`embeddings=embeddings,`
			`search_field="title_embedding",`
			`content_field="title",`
			`)`

			`docs = retriever.get_relevant_documents("my docs")`

			`assert len(docs) == 1`
			`assert "My document" in docs[0].page_content`
			`assert "id" in docs[0].metadata and "year" in docs[0].metadata`
			`assert "other_emb" not in docs[0].metadata`

			`# create a retriever with filters`
			`retriever = DocArrayRetriever(`
			`index=index,`
			`embeddings=embeddings,`
			`search_field="title_embedding",`
			`content_field="title",`
			`filters=filter_query,`
			`)`

			`docs = retriever.get_relevant_documents("my docs")`

			`assert len(docs) == 1`
			`assert "My document" in docs[0].page_content`
			`assert "id" in docs[0].metadata and "year" in docs[0].metadata`
			`assert "other_emb" not in docs[0].metadata`
			`assert docs[0].metadata["year"] <= 90`

			`# create a retriever with MMR search`
			`retriever = DocArrayRetriever(`
			`index=index,`
			`embeddings=embeddings,`
			`search_field="title_embedding",`
			`search_type="mmr",`
			`content_field="title",`
			`filters=filter_query,`
			`)`

			`docs = retriever.get_relevant_documents("my docs")`

			`assert len(docs) == 1`
			`assert "My document" in docs[0].page_content`
			`assert "id" in docs[0].metadata and "year" in docs[0].metadata`
			`assert "other_emb" not in docs[0].metadata`
			`assert docs[0].metadata["year"] <= 90`