mirror of
https://github.com/hwchase17/langchain
synced 2024-10-29 17:07:25 +00:00
8effd90be0
- Description: Add two new document transformers that translates documents into different languages and converts documents into q&a format to improve vector search results. Uses OpenAI function calling via the [doctran](https://github.com/psychic-api/doctran/tree/main) library. - Issue: N/A - Dependencies: `doctran = "^0.0.5"` - Tag maintainer: @rlancemartin @eyurtsev @hwchase17 - Twitter handle: @psychicapi or @jfan001 Notes - Adheres to the `DocumentTransformer` abstraction set by @dev2049 in #3182 - refactored `EmbeddingsRedundantFilter` to put it in a file under a new `document_transformers` module - Added basic docs for `DocumentInterrogator`, `DocumentTransformer` as well as the existing `EmbeddingsRedundantFilter` --------- Co-authored-by: Lance Martin <lance@langchain.dev> Co-authored-by: Bagatur <baskaryan@gmail.com>
18 lines
625 B
Python
18 lines
625 B
Python
"""Unit tests for document transformers."""
|
|
from langchain.document_transformers.embeddings_redundant_filter import (
|
|
_filter_similar_embeddings,
|
|
)
|
|
from langchain.math_utils import cosine_similarity
|
|
|
|
|
|
def test__filter_similar_embeddings() -> None:
|
|
threshold = 0.79
|
|
embedded_docs = [[1.0, 2.0], [1.0, 2.0], [2.0, 1.0], [2.0, 0.5], [0.0, 0.0]]
|
|
expected = [1, 3, 4]
|
|
actual = _filter_similar_embeddings(embedded_docs, cosine_similarity, threshold)
|
|
assert expected == actual
|
|
|
|
|
|
def test__filter_similar_embeddings_empty() -> None:
|
|
assert len(_filter_similar_embeddings([], cosine_similarity, 0.0)) == 0
|