mirror of https://github.com/hwchase17/langchain
Elasticsearch self-query retriever (#9248)
Now with ElasticsearchStore VectorStore merged, i've added support for the self-query retriever. I've added a notebook also to demonstrate capability. I've also added unit tests. **Credit** @elastic and @phoey1 on twitter.pull/9254/head
parent
0a04e63811
commit
5e9687a196
@ -0,0 +1,362 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "13afcae7",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Elasticsearch self-querying "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "68e75fb9",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Creating a Elasticsearch vectorstore\n",
|
||||||
|
"First we'll want to create a Elasticsearch VectorStore and seed it with some data. We've created a small demo set of documents that contain summaries of movies.\n",
|
||||||
|
"\n",
|
||||||
|
"NOTE: The self-query retriever requires you to have `lark` installed (`pip install lark`). We also need the `elasticsearch` package."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"id": "63a8af5b",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"#!pip install lark elasticsearch"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "cb4a5787",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from langchain.schema import Document\n",
|
||||||
|
"from langchain.embeddings.openai import OpenAIEmbeddings\n",
|
||||||
|
"from langchain.vectorstores import ElasticsearchStore\n",
|
||||||
|
"import os\n",
|
||||||
|
"import getpass\n",
|
||||||
|
"\n",
|
||||||
|
"os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")\n",
|
||||||
|
"\n",
|
||||||
|
"embeddings = OpenAIEmbeddings()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 9,
|
||||||
|
"id": "bcbe04d9",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"docs = [\n",
|
||||||
|
" Document(\n",
|
||||||
|
" page_content=\"A bunch of scientists bring back dinosaurs and mayhem breaks loose\",\n",
|
||||||
|
" metadata={\"year\": 1993, \"rating\": 7.7, \"genre\": \"science fiction\"},\n",
|
||||||
|
" ),\n",
|
||||||
|
" Document(\n",
|
||||||
|
" page_content=\"Leo DiCaprio gets lost in a dream within a dream within a dream within a ...\",\n",
|
||||||
|
" metadata={\"year\": 2010, \"director\": \"Christopher Nolan\", \"rating\": 8.2},\n",
|
||||||
|
" ),\n",
|
||||||
|
" Document(\n",
|
||||||
|
" page_content=\"A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea\",\n",
|
||||||
|
" metadata={\"year\": 2006, \"director\": \"Satoshi Kon\", \"rating\": 8.6},\n",
|
||||||
|
" ),\n",
|
||||||
|
" Document(\n",
|
||||||
|
" page_content=\"A bunch of normal-sized women are supremely wholesome and some men pine after them\",\n",
|
||||||
|
" metadata={\"year\": 2019, \"director\": \"Greta Gerwig\", \"rating\": 8.3},\n",
|
||||||
|
" ),\n",
|
||||||
|
" Document(\n",
|
||||||
|
" page_content=\"Toys come alive and have a blast doing so\",\n",
|
||||||
|
" metadata={\"year\": 1995, \"genre\": \"animated\"},\n",
|
||||||
|
" ),\n",
|
||||||
|
" Document(\n",
|
||||||
|
" page_content=\"Three men walk into the Zone, three men walk out of the Zone\",\n",
|
||||||
|
" metadata={\n",
|
||||||
|
" \"year\": 1979,\n",
|
||||||
|
" \"rating\": 9.9,\n",
|
||||||
|
" \"director\": \"Andrei Tarkovsky\",\n",
|
||||||
|
" \"genre\": \"science fiction\",\n",
|
||||||
|
" \"rating\": 9.9,\n",
|
||||||
|
" },\n",
|
||||||
|
" ),\n",
|
||||||
|
"]\n",
|
||||||
|
"vectorstore = ElasticsearchStore.from_documents(\n",
|
||||||
|
" docs, embeddings, index_name=\"elasticsearch-self-query-demo\", es_url=\"http://localhost:9200\"\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "5ecaab6d",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Creating our self-querying retriever\n",
|
||||||
|
"Now we can instantiate our retriever. To do this we'll need to provide some information upfront about the metadata fields that our documents support and a short description of the document contents."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"id": "86e34dbf",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from langchain.llms import OpenAI\n",
|
||||||
|
"from langchain.retrievers.self_query.base import SelfQueryRetriever\n",
|
||||||
|
"from langchain.chains.query_constructor.base import AttributeInfo\n",
|
||||||
|
"\n",
|
||||||
|
"metadata_field_info = [\n",
|
||||||
|
" AttributeInfo(\n",
|
||||||
|
" name=\"genre\",\n",
|
||||||
|
" description=\"The genre of the movie\",\n",
|
||||||
|
" type=\"string or list[string]\",\n",
|
||||||
|
" ),\n",
|
||||||
|
" AttributeInfo(\n",
|
||||||
|
" name=\"year\",\n",
|
||||||
|
" description=\"The year the movie was released\",\n",
|
||||||
|
" type=\"integer\",\n",
|
||||||
|
" ),\n",
|
||||||
|
" AttributeInfo(\n",
|
||||||
|
" name=\"director\",\n",
|
||||||
|
" description=\"The name of the movie director\",\n",
|
||||||
|
" type=\"string\",\n",
|
||||||
|
" ),\n",
|
||||||
|
" AttributeInfo(\n",
|
||||||
|
" name=\"rating\", description=\"A 1-10 rating for the movie\", type=\"float\"\n",
|
||||||
|
" ),\n",
|
||||||
|
"]\n",
|
||||||
|
"document_content_description = \"Brief summary of a movie\"\n",
|
||||||
|
"llm = OpenAI(temperature=0)\n",
|
||||||
|
"retriever = SelfQueryRetriever.from_llm(\n",
|
||||||
|
" llm, vectorstore, document_content_description, metadata_field_info, verbose=True\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "ea9df8d4",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Testing it out\n",
|
||||||
|
"And now we can try actually using our retriever!"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 10,
|
||||||
|
"id": "38a126e9",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"query='dinosaur' filter=None limit=None\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'year': 1993, 'rating': 7.7, 'genre': 'science fiction'}),\n",
|
||||||
|
" Document(page_content='Toys come alive and have a blast doing so', metadata={'year': 1995, 'genre': 'animated'}),\n",
|
||||||
|
" Document(page_content='Three men walk into the Zone, three men walk out of the Zone', metadata={'year': 1979, 'rating': 9.9, 'director': 'Andrei Tarkovsky', 'genre': 'science fiction'}),\n",
|
||||||
|
" Document(page_content='A psychologist / detective gets lost in a series of dreams within dreams within dreams and Inception reused the idea', metadata={'year': 2006, 'director': 'Satoshi Kon', 'rating': 8.6})]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 10,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# This example only specifies a relevant query\n",
|
||||||
|
"retriever.get_relevant_documents(\"What are some movies about dinosaurs\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 11,
|
||||||
|
"id": "b19d4da0",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"query='women' filter=Comparison(comparator=<Comparator.EQ: 'eq'>, attribute='director', value='Greta Gerwig') limit=None\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[Document(page_content='A bunch of normal-sized women are supremely wholesome and some men pine after them', metadata={'year': 2019, 'director': 'Greta Gerwig', 'rating': 8.3})]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 11,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# This example specifies a query and a filter\n",
|
||||||
|
"retriever.get_relevant_documents(\"Has Greta Gerwig directed any movies about women\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "39bd1de1-b9fe-4a98-89da-58d8a7a6ae51",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Filter k\n",
|
||||||
|
"\n",
|
||||||
|
"We can also use the self query retriever to specify `k`: the number of documents to fetch.\n",
|
||||||
|
"\n",
|
||||||
|
"We can do this by passing `enable_limit=True` to the constructor."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 12,
|
||||||
|
"id": "bff36b88-b506-4877-9c63-e5a1a8d78e64",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"retriever = SelfQueryRetriever.from_llm(\n",
|
||||||
|
" llm,\n",
|
||||||
|
" vectorstore,\n",
|
||||||
|
" document_content_description,\n",
|
||||||
|
" metadata_field_info,\n",
|
||||||
|
" enable_limit=True,\n",
|
||||||
|
" verbose=True,\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 13,
|
||||||
|
"id": "2758d229-4f97-499c-819f-888acaf8ee10",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"query='dinosaur' filter=None limit=2\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[Document(page_content='A bunch of scientists bring back dinosaurs and mayhem breaks loose', metadata={'year': 1993, 'rating': 7.7, 'genre': 'science fiction'}),\n",
|
||||||
|
" Document(page_content='Toys come alive and have a blast doing so', metadata={'year': 1995, 'genre': 'animated'})]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 13,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# This example only specifies a relevant query\n",
|
||||||
|
"retriever.get_relevant_documents(\"what are two movies about dinosaurs\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "61a10294",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Complex queries in Action!\n",
|
||||||
|
"We've tried out some simple queries, but what about more complex ones? Let's try out a few more complex queries that utilize the full power of Elasticsearch."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 18,
|
||||||
|
"id": "e460da93",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"query='animated toys' filter=Operation(operator=<Operator.AND: 'and'>, arguments=[Operation(operator=<Operator.OR: 'or'>, arguments=[Comparison(comparator=<Comparator.EQ: 'eq'>, attribute='genre', value='animated'), Comparison(comparator=<Comparator.EQ: 'eq'>, attribute='genre', value='comedy')]), Comparison(comparator=<Comparator.GTE: 'gte'>, attribute='year', value=1990)]) limit=None\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"[Document(page_content='Toys come alive and have a blast doing so', metadata={'year': 1995, 'genre': 'animated'})]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 18,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"retriever.get_relevant_documents(\"what animated or comedy movies have been released in the last 30 years about animated toys?\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 8,
|
||||||
|
"id": "0851fc42",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"ObjectApiResponse({'acknowledged': True})"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 8,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"vectorstore.client.indices.delete(index=\"elasticsearch-self-query-demo\")"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
@ -0,0 +1,93 @@
|
|||||||
|
from typing import Dict, Tuple, Union
|
||||||
|
|
||||||
|
from langchain.chains.query_constructor.ir import (
|
||||||
|
Comparator,
|
||||||
|
Comparison,
|
||||||
|
Operation,
|
||||||
|
Operator,
|
||||||
|
StructuredQuery,
|
||||||
|
Visitor,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ElasticsearchTranslator(Visitor):
|
||||||
|
"""Translate the internal query language elements to valid filters."""
|
||||||
|
|
||||||
|
allowed_comparators = [
|
||||||
|
Comparator.EQ,
|
||||||
|
Comparator.GT,
|
||||||
|
Comparator.GTE,
|
||||||
|
Comparator.LT,
|
||||||
|
Comparator.LTE,
|
||||||
|
Comparator.CONTAIN,
|
||||||
|
Comparator.LIKE,
|
||||||
|
]
|
||||||
|
"""Subset of allowed logical comparators."""
|
||||||
|
|
||||||
|
allowed_operators = [Operator.AND, Operator.OR, Operator.NOT]
|
||||||
|
"""Subset of allowed logical operators."""
|
||||||
|
|
||||||
|
def _format_func(self, func: Union[Operator, Comparator]) -> str:
|
||||||
|
self._validate_func(func)
|
||||||
|
map_dict = {
|
||||||
|
Operator.OR: "should",
|
||||||
|
Operator.NOT: "must_not",
|
||||||
|
Operator.AND: "must",
|
||||||
|
Comparator.EQ: "term",
|
||||||
|
Comparator.GT: "gt",
|
||||||
|
Comparator.GTE: "gte",
|
||||||
|
Comparator.LT: "lt",
|
||||||
|
Comparator.LTE: "lte",
|
||||||
|
Comparator.CONTAIN: "match",
|
||||||
|
Comparator.LIKE: "fuzzy",
|
||||||
|
}
|
||||||
|
return map_dict[func]
|
||||||
|
|
||||||
|
def visit_operation(self, operation: Operation) -> Dict:
|
||||||
|
args = [arg.accept(self) for arg in operation.arguments]
|
||||||
|
|
||||||
|
return {"bool": {self._format_func(operation.operator): args}}
|
||||||
|
|
||||||
|
def visit_comparison(self, comparison: Comparison) -> Dict:
|
||||||
|
# ElasticsearchStore filters require to target
|
||||||
|
# the metadata object field
|
||||||
|
field = f"metadata.{comparison.attribute}"
|
||||||
|
|
||||||
|
is_range_comparator = comparison.comparator in [
|
||||||
|
Comparator.GT,
|
||||||
|
Comparator.GTE,
|
||||||
|
Comparator.LT,
|
||||||
|
Comparator.LTE,
|
||||||
|
]
|
||||||
|
|
||||||
|
if is_range_comparator:
|
||||||
|
return {
|
||||||
|
"range": {
|
||||||
|
field: {self._format_func(comparison.comparator): comparison.value}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if comparison.comparator == Comparator.LIKE:
|
||||||
|
return {
|
||||||
|
self._format_func(comparison.comparator): {
|
||||||
|
field: {"value": comparison.value, "fuzziness": "AUTO"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if comparison.comparator == Comparator.CONTAIN:
|
||||||
|
return {self._format_func(comparison.comparator): {field: comparison.value}}
|
||||||
|
|
||||||
|
# we assume that if the value is a string,
|
||||||
|
# we want to use the keyword field
|
||||||
|
field = f"{field}.keyword" if isinstance(comparison.value, str) else field
|
||||||
|
|
||||||
|
return {self._format_func(comparison.comparator): {field: comparison.value}}
|
||||||
|
|
||||||
|
def visit_structured_query(
|
||||||
|
self, structured_query: StructuredQuery
|
||||||
|
) -> Tuple[str, dict]:
|
||||||
|
if structured_query.filter is None:
|
||||||
|
kwargs = {}
|
||||||
|
else:
|
||||||
|
kwargs = {"filter": [structured_query.filter.accept(self)]}
|
||||||
|
return structured_query.query, kwargs
|
@ -0,0 +1,220 @@
|
|||||||
|
from typing import Dict, Tuple
|
||||||
|
|
||||||
|
from langchain.chains.query_constructor.ir import (
|
||||||
|
Comparator,
|
||||||
|
Comparison,
|
||||||
|
Operation,
|
||||||
|
Operator,
|
||||||
|
StructuredQuery,
|
||||||
|
)
|
||||||
|
from langchain.retrievers.self_query.elasticsearch import ElasticsearchTranslator
|
||||||
|
|
||||||
|
DEFAULT_TRANSLATOR = ElasticsearchTranslator()
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.EQ, attribute="foo", value="1")
|
||||||
|
expected = {"term": {"metadata.foo.keyword": "1"}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison_range_gt() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.GT, attribute="foo", value=1)
|
||||||
|
expected = {"range": {"metadata.foo": {"gt": 1}}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison_range_gte() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.GTE, attribute="foo", value=1)
|
||||||
|
expected = {"range": {"metadata.foo": {"gte": 1}}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison_range_lt() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.LT, attribute="foo", value=1)
|
||||||
|
expected = {"range": {"metadata.foo": {"lt": 1}}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison_range_lte() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.LTE, attribute="foo", value=1)
|
||||||
|
expected = {"range": {"metadata.foo": {"lte": 1}}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison_range_match() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.CONTAIN, attribute="foo", value="1")
|
||||||
|
expected = {"match": {"metadata.foo": "1"}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_comparison_range_like() -> None:
|
||||||
|
comp = Comparison(comparator=Comparator.LIKE, attribute="foo", value="bar")
|
||||||
|
expected = {"fuzzy": {"metadata.foo": {"value": "bar", "fuzziness": "AUTO"}}}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_comparison(comp)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_operation() -> None:
|
||||||
|
op = Operation(
|
||||||
|
operator=Operator.AND,
|
||||||
|
arguments=[
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="foo", value=2),
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
expected = {
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{"term": {"metadata.foo": 2}},
|
||||||
|
{"term": {"metadata.bar.keyword": "baz"}},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_operation(op)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_operation_or() -> None:
|
||||||
|
op = Operation(
|
||||||
|
operator=Operator.OR,
|
||||||
|
arguments=[
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="foo", value=2),
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
expected = {
|
||||||
|
"bool": {
|
||||||
|
"should": [
|
||||||
|
{"term": {"metadata.foo": 2}},
|
||||||
|
{"term": {"metadata.bar.keyword": "baz"}},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_operation(op)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_operation_not() -> None:
|
||||||
|
op = Operation(
|
||||||
|
operator=Operator.NOT,
|
||||||
|
arguments=[
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="foo", value=2),
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
expected = {
|
||||||
|
"bool": {
|
||||||
|
"must_not": [
|
||||||
|
{"term": {"metadata.foo": 2}},
|
||||||
|
{"term": {"metadata.bar.keyword": "baz"}},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_operation(op)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_structured_query() -> None:
|
||||||
|
query = "What is the capital of France?"
|
||||||
|
|
||||||
|
structured_query = StructuredQuery(query=query, filter=None, limit=None)
|
||||||
|
expected: Tuple[str, Dict] = (query, {})
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_structured_query_filter() -> None:
|
||||||
|
query = "What is the capital of France?"
|
||||||
|
comp = Comparison(comparator=Comparator.EQ, attribute="foo", value="1")
|
||||||
|
structured_query = StructuredQuery(query=query, filter=comp, limit=None)
|
||||||
|
expected = (
|
||||||
|
query,
|
||||||
|
{"filter": [{"term": {"metadata.foo.keyword": "1"}}]},
|
||||||
|
)
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_structured_query_filter_and() -> None:
|
||||||
|
query = "What is the capital of France?"
|
||||||
|
op = Operation(
|
||||||
|
operator=Operator.AND,
|
||||||
|
arguments=[
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="foo", value=2),
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="bar", value="baz"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
structured_query = StructuredQuery(query=query, filter=op, limit=None)
|
||||||
|
expected = (
|
||||||
|
query,
|
||||||
|
{
|
||||||
|
"filter": [
|
||||||
|
{
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{"term": {"metadata.foo": 2}},
|
||||||
|
{"term": {"metadata.bar.keyword": "baz"}},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
|
||||||
|
assert expected == actual
|
||||||
|
|
||||||
|
|
||||||
|
def test_visit_structured_query_complex() -> None:
|
||||||
|
query = "What is the capital of France?"
|
||||||
|
op = Operation(
|
||||||
|
operator=Operator.AND,
|
||||||
|
arguments=[
|
||||||
|
Comparison(comparator=Comparator.EQ, attribute="foo", value=2),
|
||||||
|
Operation(
|
||||||
|
operator=Operator.OR,
|
||||||
|
arguments=[
|
||||||
|
Comparison(comparator=Comparator.LT, attribute="bar", value=1),
|
||||||
|
Comparison(comparator=Comparator.LIKE, attribute="bar", value="10"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
structured_query = StructuredQuery(query=query, filter=op, limit=None)
|
||||||
|
expected = (
|
||||||
|
query,
|
||||||
|
{
|
||||||
|
"filter": [
|
||||||
|
{
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{"term": {"metadata.foo": 2}},
|
||||||
|
{
|
||||||
|
"bool": {
|
||||||
|
"should": [
|
||||||
|
{"range": {"metadata.bar": {"lt": 1}}},
|
||||||
|
{
|
||||||
|
"fuzzy": {
|
||||||
|
"metadata.bar": {
|
||||||
|
"value": "10",
|
||||||
|
"fuzziness": "AUTO",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
)
|
||||||
|
actual = DEFAULT_TRANSLATOR.visit_structured_query(structured_query)
|
||||||
|
assert expected == actual
|
Loading…
Reference in New Issue