From b0859c9b185fe897f3c8e2699835a669b2a2ba61 Mon Sep 17 00:00:00 2001 From: Zander Chase <130414180+vowelparrot@users.noreply.github.com> Date: Fri, 30 Jun 2023 14:44:03 -0700 Subject: [PATCH] Add New Retriever Interface with Callbacks (#5962) Handle the new retriever events in a way that (I think) is entirely backwards compatible? Needs more testing for some of the chain changes and all. This creates an entire new run type, however. We could also just treat this as an event within a chain run presumably (same with memory) Adds a subclass initializer that upgrades old retriever implementations to the new schema, along with tests to ensure they work. First commit doesn't upgrade any of our retriever implementations (to show that we can pass the tests along with additional ones testing the upgrade logic). Second commit upgrades the known universe of retrievers in langchain. - [X] Add callback handling methods for retriever start/end/error (open to renaming to 'retrieval' if you want that) - [X] Update BaseRetriever schema to support callbacks - [X] Tests for upgrading old "v1" retrievers for backwards compatibility - [X] Update existing retriever implementations to implement the new interface - [X] Update calls within chains to .{a]get_relevant_documents to pass the child callback manager - [X] Update the notebooks/docs to reflect the new interface - [X] Test notebooks thoroughly Not handled: - Memory pass throughs: retrieval memory doesn't have a parent callback manager passed through the method --------- Co-authored-by: Nuno Campos Co-authored-by: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> --- .../modules/chains/additional/flare.ipynb | 15 +- .../integrations/azuresearch.ipynb | 1 - .../retrievers/get_started.mdx | 32 +- .../retrievers/how_to/custom_retriever.mdx | 162 ++++++ langchain/callbacks/aim_callback.py | 7 + langchain/callbacks/base.py | 79 ++- langchain/callbacks/manager.py | 154 +++++ langchain/callbacks/tracers/base.py | 72 ++- langchain/callbacks/tracers/langchain.py | 18 + langchain/callbacks/utils.py | 2 + .../chains/conversational_retrieval/base.py | 78 ++- langchain/chains/qa_with_sources/base.py | 48 +- langchain/chains/qa_with_sources/loading.py | 8 +- langchain/chains/qa_with_sources/retrieval.py | 20 +- langchain/chains/qa_with_sources/vector_db.py | 12 +- .../chains/question_answering/__init__.py | 6 +- langchain/chains/retrieval_qa/base.py | 73 ++- .../client/tracing_datasets.ipynb | 2 +- langchain/retrievers/__init__.py | 2 +- langchain/retrievers/arxiv.py | 22 +- .../retrievers/azure_cognitive_search.py | 23 +- .../retrievers/chatgpt_plugin_retriever.py | 22 +- .../retrievers/contextual_compression.py | 37 +- langchain/retrievers/databerry.py | 22 +- langchain/retrievers/docarray.py | 18 +- .../retrievers/document_compressors/base.py | 50 +- .../document_compressors/chain_extract.py | 17 +- .../document_compressors/chain_filter.py | 15 +- .../document_compressors/cohere_rerank.py | 13 +- .../document_compressors/embeddings_filter.py | 11 +- langchain/retrievers/elastic_search_bm25.py | 21 +- langchain/retrievers/kendra.py | 18 +- langchain/retrievers/knn.py | 20 +- langchain/retrievers/llama_index.py | 38 +- langchain/retrievers/merger_retriever.py | 43 +- langchain/retrievers/metal.py | 20 +- langchain/retrievers/milvus.py | 24 +- langchain/retrievers/multi_query.py | 40 +- .../retrievers/pinecone_hybrid_search.py | 21 +- langchain/retrievers/pubmed.py | 35 ++ langchain/retrievers/pupmed.py | 21 +- langchain/retrievers/remote_retriever.py | 22 +- langchain/retrievers/self_query/base.py | 26 +- langchain/retrievers/svm.py | 20 +- langchain/retrievers/tfidf.py | 21 +- .../retrievers/time_weighted_retriever.py | 21 +- langchain/retrievers/vespa_retriever.py | 21 +- .../retrievers/weaviate_hybrid_search.py | 23 +- langchain/retrievers/wikipedia.py | 22 +- langchain/retrievers/zep.py | 24 +- langchain/retrievers/zilliz.py | 24 +- langchain/schema.py | 145 ++++- langchain/vectorstores/azuresearch.py | 18 +- langchain/vectorstores/base.py | 21 +- langchain/vectorstores/redis.py | 21 +- langchain/vectorstores/singlestoredb.py | 32 +- poetry.lock | 544 +++++++++++++++++- .../callbacks/fake_callback_handler.py | 44 +- .../callbacks/test_callback_manager.py | 16 + tests/unit_tests/retrievers/test_base.py | 220 +++++++ 60 files changed, 2398 insertions(+), 229 deletions(-) create mode 100644 docs/snippets/modules/data_connection/retrievers/how_to/custom_retriever.mdx create mode 100644 langchain/retrievers/pubmed.py create mode 100644 tests/unit_tests/retrievers/test_base.py diff --git a/docs/extras/modules/chains/additional/flare.ipynb b/docs/extras/modules/chains/additional/flare.ipynb index a681d232c3..b8fdebd206 100644 --- a/docs/extras/modules/chains/additional/flare.ipynb +++ b/docs/extras/modules/chains/additional/flare.ipynb @@ -71,11 +71,13 @@ "import numpy as np\n", "\n", "from langchain.schema import BaseRetriever\n", + "from langchain.callbacks.manager import AsyncCallbackManagerForRetrieverRun, CallbackManagerForRetrieverRun\n", "from langchain.utilities import GoogleSerperAPIWrapper\n", "from langchain.embeddings import OpenAIEmbeddings\n", "from langchain.chat_models import ChatOpenAI\n", "from langchain.llms import OpenAI\n", - "from langchain.schema import Document" + "from langchain.schema import Document\n", + "from typing import Any" ] }, { @@ -97,11 +99,16 @@ " def __init__(self, search):\n", " self.search = search\n", "\n", - " def get_relevant_documents(self, query: str):\n", + " def _get_relevant_documents(self, query: str, *, run_manager: CallbackManagerForRetrieverRun, **kwargs: Any) -> List[Document]:\n", " return [Document(page_content=self.search.run(query))]\n", "\n", - " async def aget_relevant_documents(self, query: str):\n", - " raise NotImplemented\n", + " async def _aget_relevant_documents(self,\n", + " query: str,\n", + " *,\n", + " run_manager: AsyncCallbackManagerForRetrieverRun,\n", + " **kwargs: Any,\n", + " ) -> List[Document]:\n", + " raise NotImplementedError()\n", "\n", "\n", "retriever = SerperSearchRetriever(GoogleSerperAPIWrapper())" diff --git a/docs/extras/modules/data_connection/vectorstores/integrations/azuresearch.ipynb b/docs/extras/modules/data_connection/vectorstores/integrations/azuresearch.ipynb index c36f525fd2..c910d2e590 100644 --- a/docs/extras/modules/data_connection/vectorstores/integrations/azuresearch.ipynb +++ b/docs/extras/modules/data_connection/vectorstores/integrations/azuresearch.ipynb @@ -43,7 +43,6 @@ "import openai\n", "from dotenv import load_dotenv\n", "from langchain.embeddings.openai import OpenAIEmbeddings\n", - "from langchain.schema import BaseRetriever\n", "from langchain.vectorstores.azuresearch import AzureSearch" ] }, diff --git a/docs/snippets/modules/data_connection/retrievers/get_started.mdx b/docs/snippets/modules/data_connection/retrievers/get_started.mdx index 124ee73c2e..0a5b793d2b 100644 --- a/docs/snippets/modules/data_connection/retrievers/get_started.mdx +++ b/docs/snippets/modules/data_connection/retrievers/get_started.mdx @@ -1,24 +1,40 @@ -The `BaseRetriever` class in LangChain is as follows: +The public API of the `BaseRetriever` class in LangChain is as follows: ```python from abc import ABC, abstractmethod -from typing import List +from typing import Any, List from langchain.schema import Document +from langchain.callbacks.manager import Callbacks class BaseRetriever(ABC): - @abstractmethod - def get_relevant_documents(self, query: str) -> List[Document]: - """Get texts relevant for a query. - + ... + def get_relevant_documents( + self, query: str, *, callbacks: Callbacks = None, **kwargs: Any + ) -> List[Document]: + """Retrieve documents relevant to a query. Args: - query: string to find relevant texts for + query: string to find relevant documents for + callbacks: Callback manager or list of callbacks + Returns: + List of relevant documents + """ + ... + async def aget_relevant_documents( + self, query: str, *, callbacks: Callbacks = None, **kwargs: Any + ) -> List[Document]: + """Asynchronously get documents relevant to a query. + Args: + query: string to find relevant documents for + callbacks: Callback manager or list of callbacks Returns: List of relevant documents """ + ... ``` -It's that simple! The `get_relevant_documents` method can be implemented however you see fit. +It's that simple! You can call `get_relevant_documents` or the async `get_relevant_documents` methods to retrieve documents relevant to a query, where "relevance" is defined by +the specific retriever object you are calling. Of course, we also help construct what we think useful Retrievers are. The main type of Retriever that we focus on is a Vectorstore retriever. We will focus on that for the rest of this guide. diff --git a/docs/snippets/modules/data_connection/retrievers/how_to/custom_retriever.mdx b/docs/snippets/modules/data_connection/retrievers/how_to/custom_retriever.mdx new file mode 100644 index 0000000000..911784d96c --- /dev/null +++ b/docs/snippets/modules/data_connection/retrievers/how_to/custom_retriever.mdx @@ -0,0 +1,162 @@ +# Implement a Custom Retriever + +In this walkthrough, you will implement a simple custom retriever in LangChain using a simple dot product distance lookup. + +All retrievers inherit from the `BaseRetriever` class and override the following abstract methods: + +```python +from abc import ABC, abstractmethod +from typing import Any, List +from langchain.schema import Document +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) + +class BaseRetriever(ABC): + @abstractmethod + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun, **kwargs: Any + ) -> List[Document]: + """Get documents relevant to a query. + Args: + query: string to find relevant documents for + run_manager: The callbacks handler to use + Returns: + List of relevant documents + """ + + @abstractmethod + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + """Asynchronously get documents relevant to a query. + Args: + query: string to find relevant documents for + run_manager: The callbacks handler to use + Returns: + List of relevant documents + """ +``` + + +The `_get_relevant_documents` and async `_get_relevant_documents` methods can be implemented however you see fit. The `run_manager` is useful if your retriever calls other traceable LangChain primitives like LLMs, chains, or tools. + + +Below, implement an example that fetches the most similar documents from a list of documents using a numpy array of embeddings. + + +```python +from typing import Any, List, Optional + +import numpy as np + +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) +from langchain.embeddings import OpenAIEmbeddings +from langchain.embeddings.base import Embeddings +from langchain.schema import BaseRetriever, Document + + +class NumpyRetriever(BaseRetriever): + """Retrieves documents from a numpy array.""" + + def __init__( + self, + texts: List[str], + vectors: np.ndarray, + embeddings: Optional[Embeddings] = None, + num_to_return: int = 1, + ) -> None: + super().__init__() + self.embeddings = embeddings or OpenAIEmbeddings() + self.texts = texts + self.vectors = vectors + self.num_to_return = num_to_return + + @classmethod + def from_texts( + cls, + texts: List[str], + embeddings: Optional[Embeddings] = None, + **kwargs: Any, + ) -> "NumpyRetriever": + embeddings = embeddings or OpenAIEmbeddings() + vectors = np.array(embeddings.embed_documents(texts)) + return cls(texts, vectors, embeddings) + + def _get_relevant_documents_from_query_vector( + self, vector_query: np.ndarray + ) -> List[Document]: + dot_product = np.dot(self.vectors, vector_query) + # Get the indices of the min 5 documents + indices = np.argpartition( + dot_product, -min(self.num_to_return, len(self.vectors)) + )[-self.num_to_return :] + # Sort indices by distance + indices = indices[np.argsort(dot_product[indices])] + return [ + Document( + page_content=self.texts[idx], + metadata={"index": idx}, + ) + for idx in indices + ] + + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun, **kwargs: Any + ) -> List[Document]: + """Get documents relevant to a query. + Args: + query: string to find relevant documents for + run_manager: The callbacks handler to use + Returns: + List of relevant documents + """ + vector_query = np.array(self.embeddings.embed_query(query)) + return self._get_relevant_documents_from_query_vector(vector_query) + + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + """Asynchronously get documents relevant to a query. + Args: + query: string to find relevant documents for + run_manager: The callbacks handler to use + Returns: + List of relevant documents + """ + query_emb = await self.embeddings.aembed_query(query) + return self._get_relevant_documents_from_query_vector(np.array(query_emb)) +``` + +The retriever can be instantiated through the class method `from_texts`. It embeds the texts and stores them in a numpy array. To look up documents, it embeds the query and finds the most similar documents using a simple dot product distance. +Once the retriever is implemented, you can use it like any other retriever in LangChain. + + +```python +retriever = NumpyRetriever.from_texts(texts= ["hello world", "goodbye world"]) +``` + +You can then use the retriever to get relevant documents. + +```python +retriever.get_relevant_documents("Hi there!") + +# [Document(page_content='hello world', metadata={'index': 0})] +``` + +```python +retriever.get_relevant_documents("Bye!") +# [Document(page_content='goodbye world', metadata={'index': 1})] +``` diff --git a/langchain/callbacks/aim_callback.py b/langchain/callbacks/aim_callback.py index 2cb3925601..9941f92989 100644 --- a/langchain/callbacks/aim_callback.py +++ b/langchain/callbacks/aim_callback.py @@ -30,6 +30,7 @@ class BaseMetadataCallbackHandler: ignore_llm_ (bool): Whether to ignore llm callbacks. ignore_chain_ (bool): Whether to ignore chain callbacks. ignore_agent_ (bool): Whether to ignore agent callbacks. + ignore_retriever_ (bool): Whether to ignore retriever callbacks. always_verbose_ (bool): Whether to always be verbose. chain_starts (int): The number of times the chain start method has been called. chain_ends (int): The number of times the chain end method has been called. @@ -52,6 +53,7 @@ class BaseMetadataCallbackHandler: self.ignore_llm_ = False self.ignore_chain_ = False self.ignore_agent_ = False + self.ignore_retriever_ = False self.always_verbose_ = False self.chain_starts = 0 @@ -86,6 +88,11 @@ class BaseMetadataCallbackHandler: """Whether to ignore agent callbacks.""" return self.ignore_agent_ + @property + def ignore_retriever(self) -> bool: + """Whether to ignore retriever callbacks.""" + return self.ignore_retriever_ + def get_custom_callback_meta(self) -> Dict[str, Any]: return { "step": self.step, diff --git a/langchain/callbacks/base.py b/langchain/callbacks/base.py index 2f4ed83dc9..6092970d01 100644 --- a/langchain/callbacks/base.py +++ b/langchain/callbacks/base.py @@ -1,15 +1,34 @@ """Base callback handler that can be used to handle callbacks in langchain.""" from __future__ import annotations -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Sequence, Union from uuid import UUID -from langchain.schema import ( - AgentAction, - AgentFinish, - BaseMessage, - LLMResult, -) +from langchain.schema import AgentAction, AgentFinish, BaseMessage, Document, LLMResult + + +class RetrieverManagerMixin: + """Mixin for Retriever callbacks.""" + + def on_retriever_error( + self, + error: Union[Exception, KeyboardInterrupt], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + """Run when Retriever errors.""" + + def on_retriever_end( + self, + documents: Sequence[Document], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + """Run when Retriever ends running.""" class LLMManagerMixin: @@ -144,6 +163,16 @@ class CallbackManagerMixin: f"{self.__class__.__name__} does not implement `on_chat_model_start`" ) + def on_retriever_start( + self, + query: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + """Run when Retriever starts running.""" + def on_chain_start( self, serialized: Dict[str, Any], @@ -187,6 +216,7 @@ class BaseCallbackHandler( LLMManagerMixin, ChainManagerMixin, ToolManagerMixin, + RetrieverManagerMixin, CallbackManagerMixin, RunManagerMixin, ): @@ -211,6 +241,11 @@ class BaseCallbackHandler( """Whether to ignore agent callbacks.""" return False + @property + def ignore_retriever(self) -> bool: + """Whether to ignore retriever callbacks.""" + return False + @property def ignore_chat_model(self) -> bool: """Whether to ignore chat model callbacks.""" @@ -371,6 +406,36 @@ class AsyncCallbackHandler(BaseCallbackHandler): ) -> None: """Run on agent end.""" + async def on_retriever_start( + self, + query: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + """Run on retriever start.""" + + async def on_retriever_end( + self, + documents: Sequence[Document], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + """Run on retriever end.""" + + async def on_retriever_error( + self, + error: Union[Exception, KeyboardInterrupt], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + """Run on retriever error.""" + class BaseCallbackManager(CallbackManagerMixin): """Base callback manager that can be used to handle callbacks from LangChain.""" diff --git a/langchain/callbacks/manager.py b/langchain/callbacks/manager.py index 18173b77ef..568d11a9a8 100644 --- a/langchain/callbacks/manager.py +++ b/langchain/callbacks/manager.py @@ -14,6 +14,7 @@ from typing import ( Generator, List, Optional, + Sequence, Type, TypeVar, Union, @@ -27,6 +28,7 @@ from langchain.callbacks.base import ( BaseCallbackManager, ChainManagerMixin, LLMManagerMixin, + RetrieverManagerMixin, RunManagerMixin, ToolManagerMixin, ) @@ -40,6 +42,7 @@ from langchain.schema import ( AgentAction, AgentFinish, BaseMessage, + Document, LLMResult, get_buffer_string, ) @@ -899,6 +902,97 @@ class AsyncCallbackManagerForToolRun(AsyncRunManager, ToolManagerMixin): ) +class CallbackManagerForRetrieverRun(RunManager, RetrieverManagerMixin): + """Callback manager for retriever run.""" + + def get_child(self, tag: Optional[str] = None) -> CallbackManager: + """Get a child callback manager.""" + manager = CallbackManager([], parent_run_id=self.run_id) + manager.set_handlers(self.inheritable_handlers) + manager.add_tags(self.inheritable_tags) + if tag is not None: + manager.add_tags([tag], False) + return manager + + def on_retriever_end( + self, + documents: Sequence[Document], + **kwargs: Any, + ) -> None: + """Run when retriever ends running.""" + _handle_event( + self.handlers, + "on_retriever_end", + "ignore_retriever", + documents, + run_id=self.run_id, + parent_run_id=self.parent_run_id, + **kwargs, + ) + + def on_retriever_error( + self, + error: Union[Exception, KeyboardInterrupt], + **kwargs: Any, + ) -> None: + """Run when retriever errors.""" + _handle_event( + self.handlers, + "on_retriever_error", + "ignore_retriever", + error, + run_id=self.run_id, + parent_run_id=self.parent_run_id, + **kwargs, + ) + + +class AsyncCallbackManagerForRetrieverRun( + AsyncRunManager, + RetrieverManagerMixin, +): + """Async callback manager for retriever run.""" + + def get_child(self, tag: Optional[str] = None) -> AsyncCallbackManager: + """Get a child callback manager.""" + manager = AsyncCallbackManager([], parent_run_id=self.run_id) + manager.set_handlers(self.inheritable_handlers) + manager.add_tags(self.inheritable_tags) + if tag is not None: + manager.add_tags([tag], False) + return manager + + async def on_retriever_end( + self, documents: Sequence[Document], **kwargs: Any + ) -> None: + """Run when retriever ends running.""" + await _ahandle_event( + self.handlers, + "on_retriever_end", + "ignore_retriever", + documents, + run_id=self.run_id, + parent_run_id=self.parent_run_id, + **kwargs, + ) + + async def on_retriever_error( + self, + error: Union[Exception, KeyboardInterrupt], + **kwargs: Any, + ) -> None: + """Run when retriever errors.""" + await _ahandle_event( + self.handlers, + "on_retriever_error", + "ignore_retriever", + error, + run_id=self.run_id, + parent_run_id=self.parent_run_id, + **kwargs, + ) + + class CallbackManager(BaseCallbackManager): """Callback manager that can be used to handle callbacks from langchain.""" @@ -1077,6 +1171,36 @@ class CallbackManager(BaseCallbackManager): inheritable_tags=self.inheritable_tags, ) + def on_retriever_start( + self, + query: str, + run_id: Optional[UUID] = None, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> CallbackManagerForRetrieverRun: + """Run when retriever starts running.""" + if run_id is None: + run_id = uuid4() + + _handle_event( + self.handlers, + "on_retriever_start", + "ignore_retriever", + query, + run_id=run_id, + parent_run_id=self.parent_run_id, + **kwargs, + ) + + return CallbackManagerForRetrieverRun( + run_id=run_id, + handlers=self.handlers, + inheritable_handlers=self.inheritable_handlers, + parent_run_id=self.parent_run_id, + tags=self.tags, + inheritable_tags=self.inheritable_tags, + ) + @classmethod def configure( cls, @@ -1313,6 +1437,36 @@ class AsyncCallbackManager(BaseCallbackManager): inheritable_tags=self.inheritable_tags, ) + async def on_retriever_start( + self, + query: str, + run_id: Optional[UUID] = None, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> AsyncCallbackManagerForRetrieverRun: + """Run when retriever starts running.""" + if run_id is None: + run_id = uuid4() + + await _ahandle_event( + self.handlers, + "on_retriever_start", + "ignore_retriever", + query, + run_id=run_id, + parent_run_id=self.parent_run_id, + **kwargs, + ) + + return AsyncCallbackManagerForRetrieverRun( + run_id=run_id, + handlers=self.handlers, + inheritable_handlers=self.inheritable_handlers, + parent_run_id=self.parent_run_id, + tags=self.tags, + inheritable_tags=self.inheritable_tags, + ) + @classmethod def configure( cls, diff --git a/langchain/callbacks/tracers/base.py b/langchain/callbacks/tracers/base.py index 6cfad4d431..edee1e2799 100644 --- a/langchain/callbacks/tracers/base.py +++ b/langchain/callbacks/tracers/base.py @@ -4,12 +4,12 @@ from __future__ import annotations import logging from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Sequence, Union from uuid import UUID from langchain.callbacks.base import BaseCallbackHandler from langchain.callbacks.tracers.schemas import Run, RunTypeEnum -from langchain.schema import LLMResult +from langchain.schema import Document, LLMResult logger = logging.getLogger(__name__) @@ -265,6 +265,65 @@ class BaseTracer(BaseCallbackHandler, ABC): self._end_trace(tool_run) self._on_tool_error(tool_run) + def on_retriever_start( + self, + query: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + """Run when Retriever starts running.""" + parent_run_id_ = str(parent_run_id) if parent_run_id else None + execution_order = self._get_execution_order(parent_run_id_) + retrieval_run = Run( + id=run_id, + name="Retriever", + parent_run_id=parent_run_id, + inputs={"query": query}, + extra=kwargs, + start_time=datetime.utcnow(), + execution_order=execution_order, + child_execution_order=execution_order, + child_runs=[], + run_type=RunTypeEnum.retriever, + ) + self._start_trace(retrieval_run) + self._on_retriever_start(retrieval_run) + + def on_retriever_error( + self, + error: Union[Exception, KeyboardInterrupt], + *, + run_id: UUID, + **kwargs: Any, + ) -> None: + """Run when Retriever errors.""" + if not run_id: + raise TracerException("No run_id provided for on_retriever_error callback.") + retrieval_run = self.run_map.get(str(run_id)) + if retrieval_run is None or retrieval_run.run_type != RunTypeEnum.retriever: + raise TracerException("No retriever Run found to be traced") + + retrieval_run.error = repr(error) + retrieval_run.end_time = datetime.utcnow() + self._end_trace(retrieval_run) + self._on_retriever_error(retrieval_run) + + def on_retriever_end( + self, documents: Sequence[Document], *, run_id: UUID, **kwargs: Any + ) -> None: + """Run when Retriever ends running.""" + if not run_id: + raise TracerException("No run_id provided for on_retriever_end callback.") + retrieval_run = self.run_map.get(str(run_id)) + if retrieval_run is None or retrieval_run.run_type != RunTypeEnum.retriever: + raise TracerException("No retriever Run found to be traced") + retrieval_run.outputs = {"documents": documents} + retrieval_run.end_time = datetime.utcnow() + self._end_trace(retrieval_run) + self._on_retriever_end(retrieval_run) + def __deepcopy__(self, memo: dict) -> BaseTracer: """Deepcopy the tracer.""" return self @@ -302,3 +361,12 @@ class BaseTracer(BaseCallbackHandler, ABC): def _on_chat_model_start(self, run: Run) -> None: """Process the Chat Model Run upon start.""" + + def _on_retriever_start(self, run: Run) -> None: + """Process the Retriever Run upon start.""" + + def _on_retriever_end(self, run: Run) -> None: + """Process the Retriever Run.""" + + def _on_retriever_error(self, run: Run) -> None: + """Process the Retriever Run upon error.""" diff --git a/langchain/callbacks/tracers/langchain.py b/langchain/callbacks/tracers/langchain.py index 394857b570..da8e20512c 100644 --- a/langchain/callbacks/tracers/langchain.py +++ b/langchain/callbacks/tracers/langchain.py @@ -180,6 +180,24 @@ class LangChainTracer(BaseTracer): self.executor.submit(self._update_run_single, run.copy(deep=True)) ) + def _on_retriever_start(self, run: Run) -> None: + """Process the Retriever Run upon start.""" + self._futures.add( + self.executor.submit(self._persist_run_single, run.copy(deep=True)) + ) + + def _on_retriever_end(self, run: Run) -> None: + """Process the Retriever Run.""" + self._futures.add( + self.executor.submit(self._update_run_single, run.copy(deep=True)) + ) + + def _on_retriever_error(self, run: Run) -> None: + """Process the Retriever Run upon error.""" + self._futures.add( + self.executor.submit(self._update_run_single, run.copy(deep=True)) + ) + def wait_for_futures(self) -> None: """Wait for the given futures to complete.""" futures = list(self._futures) diff --git a/langchain/callbacks/utils.py b/langchain/callbacks/utils.py index d84d3f0a79..b83bace601 100644 --- a/langchain/callbacks/utils.py +++ b/langchain/callbacks/utils.py @@ -119,6 +119,7 @@ class BaseMetadataCallbackHandler: ignore_llm_ (bool): Whether to ignore llm callbacks. ignore_chain_ (bool): Whether to ignore chain callbacks. ignore_agent_ (bool): Whether to ignore agent callbacks. + ignore_retriever_ (bool): Whether to ignore retriever callbacks. always_verbose_ (bool): Whether to always be verbose. chain_starts (int): The number of times the chain start method has been called. chain_ends (int): The number of times the chain end method has been called. @@ -149,6 +150,7 @@ class BaseMetadataCallbackHandler: self.ignore_llm_ = False self.ignore_chain_ = False self.ignore_agent_ = False + self.ignore_retriever_ = False self.always_verbose_ = False self.chain_starts = 0 diff --git a/langchain/chains/conversational_retrieval/base.py b/langchain/chains/conversational_retrieval/base.py index 396f209269..4cd6fefc66 100644 --- a/langchain/chains/conversational_retrieval/base.py +++ b/langchain/chains/conversational_retrieval/base.py @@ -1,6 +1,7 @@ """Chain for chatting with a vector database.""" from __future__ import annotations +import inspect import warnings from abc import abstractmethod from pathlib import Path @@ -87,7 +88,13 @@ class BaseConversationalRetrievalChain(Chain): return _output_keys @abstractmethod - def _get_docs(self, question: str, inputs: Dict[str, Any]) -> List[Document]: + def _get_docs( + self, + question: str, + inputs: Dict[str, Any], + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: """Get docs.""" def _call( @@ -107,7 +114,13 @@ class BaseConversationalRetrievalChain(Chain): ) else: new_question = question - docs = self._get_docs(new_question, inputs) + accepts_run_manager = ( + "run_manager" in inspect.signature(self._get_docs).parameters + ) + if accepts_run_manager: + docs = self._get_docs(new_question, inputs, run_manager=_run_manager) + else: + docs = self._get_docs(new_question, inputs) # type: ignore[call-arg] new_inputs = inputs.copy() new_inputs["question"] = new_question new_inputs["chat_history"] = chat_history_str @@ -122,7 +135,13 @@ class BaseConversationalRetrievalChain(Chain): return output @abstractmethod - async def _aget_docs(self, question: str, inputs: Dict[str, Any]) -> List[Document]: + async def _aget_docs( + self, + question: str, + inputs: Dict[str, Any], + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: """Get docs.""" async def _acall( @@ -141,7 +160,14 @@ class BaseConversationalRetrievalChain(Chain): ) else: new_question = question - docs = await self._aget_docs(new_question, inputs) + accepts_run_manager = ( + "run_manager" in inspect.signature(self._aget_docs).parameters + ) + if accepts_run_manager: + docs = await self._aget_docs(new_question, inputs, run_manager=_run_manager) + else: + docs = await self._aget_docs(new_question, inputs) # type: ignore[call-arg] + new_inputs = inputs.copy() new_inputs["question"] = new_question new_inputs["chat_history"] = chat_history_str @@ -187,12 +213,30 @@ class ConversationalRetrievalChain(BaseConversationalRetrievalChain): return docs[:num_docs] - def _get_docs(self, question: str, inputs: Dict[str, Any]) -> List[Document]: - docs = self.retriever.get_relevant_documents(question) + def _get_docs( + self, + question: str, + inputs: Dict[str, Any], + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" + docs = self.retriever.get_relevant_documents( + question, callbacks=run_manager.get_child() + ) return self._reduce_tokens_below_limit(docs) - async def _aget_docs(self, question: str, inputs: Dict[str, Any]) -> List[Document]: - docs = await self.retriever.aget_relevant_documents(question) + async def _aget_docs( + self, + question: str, + inputs: Dict[str, Any], + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" + docs = await self.retriever.aget_relevant_documents( + question, callbacks=run_manager.get_child() + ) return self._reduce_tokens_below_limit(docs) @classmethod @@ -253,14 +297,28 @@ class ChatVectorDBChain(BaseConversationalRetrievalChain): ) return values - def _get_docs(self, question: str, inputs: Dict[str, Any]) -> List[Document]: + def _get_docs( + self, + question: str, + inputs: Dict[str, Any], + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" vectordbkwargs = inputs.get("vectordbkwargs", {}) full_kwargs = {**self.search_kwargs, **vectordbkwargs} return self.vectorstore.similarity_search( question, k=self.top_k_docs_for_context, **full_kwargs ) - async def _aget_docs(self, question: str, inputs: Dict[str, Any]) -> List[Document]: + async def _aget_docs( + self, + question: str, + inputs: Dict[str, Any], + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" raise NotImplementedError("ChatVectorDBChain does not support async") @classmethod diff --git a/langchain/chains/qa_with_sources/base.py b/langchain/chains/qa_with_sources/base.py index 96048a0a23..fe2911b9c4 100644 --- a/langchain/chains/qa_with_sources/base.py +++ b/langchain/chains/qa_with_sources/base.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import re from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional @@ -115,7 +116,12 @@ class BaseQAWithSourcesChain(Chain, ABC): return values @abstractmethod - def _get_docs(self, inputs: Dict[str, Any]) -> List[Document]: + def _get_docs( + self, + inputs: Dict[str, Any], + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: """Get docs to run questioning over.""" def _call( @@ -124,7 +130,14 @@ class BaseQAWithSourcesChain(Chain, ABC): run_manager: Optional[CallbackManagerForChainRun] = None, ) -> Dict[str, str]: _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager() - docs = self._get_docs(inputs) + accepts_run_manager = ( + "run_manager" in inspect.signature(self._get_docs).parameters + ) + if accepts_run_manager: + docs = self._get_docs(inputs, run_manager=_run_manager) + else: + docs = self._get_docs(inputs) # type: ignore[call-arg] + answer = self.combine_documents_chain.run( input_documents=docs, callbacks=_run_manager.get_child(), **inputs ) @@ -141,7 +154,12 @@ class BaseQAWithSourcesChain(Chain, ABC): return result @abstractmethod - async def _aget_docs(self, inputs: Dict[str, Any]) -> List[Document]: + async def _aget_docs( + self, + inputs: Dict[str, Any], + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: """Get docs to run questioning over.""" async def _acall( @@ -150,7 +168,13 @@ class BaseQAWithSourcesChain(Chain, ABC): run_manager: Optional[AsyncCallbackManagerForChainRun] = None, ) -> Dict[str, Any]: _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager() - docs = await self._aget_docs(inputs) + accepts_run_manager = ( + "run_manager" in inspect.signature(self._aget_docs).parameters + ) + if accepts_run_manager: + docs = await self._aget_docs(inputs, run_manager=_run_manager) + else: + docs = await self._aget_docs(inputs) # type: ignore[call-arg] answer = await self.combine_documents_chain.arun( input_documents=docs, callbacks=_run_manager.get_child(), **inputs ) @@ -180,10 +204,22 @@ class QAWithSourcesChain(BaseQAWithSourcesChain): """ return [self.input_docs_key, self.question_key] - def _get_docs(self, inputs: Dict[str, Any]) -> List[Document]: + def _get_docs( + self, + inputs: Dict[str, Any], + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: + """Get docs to run questioning over.""" return inputs.pop(self.input_docs_key) - async def _aget_docs(self, inputs: Dict[str, Any]) -> List[Document]: + async def _aget_docs( + self, + inputs: Dict[str, Any], + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: + """Get docs to run questioning over.""" return inputs.pop(self.input_docs_key) @property diff --git a/langchain/chains/qa_with_sources/loading.py b/langchain/chains/qa_with_sources/loading.py index 9eef4dc1a5..97b76474df 100644 --- a/langchain/chains/qa_with_sources/loading.py +++ b/langchain/chains/qa_with_sources/loading.py @@ -1,4 +1,6 @@ """Load question answering with sources chains.""" +from __future__ import annotations + from typing import Any, Mapping, Optional, Protocol from langchain.base_language import BaseLanguageModel @@ -13,7 +15,9 @@ from langchain.chains.qa_with_sources import ( refine_prompts, stuff_prompt, ) -from langchain.chains.question_answering import map_rerank_prompt +from langchain.chains.question_answering.map_rerank_prompt import ( + PROMPT as MAP_RERANK_PROMPT, +) from langchain.prompts.base import BasePromptTemplate @@ -28,7 +32,7 @@ class LoadingCallable(Protocol): def _load_map_rerank_chain( llm: BaseLanguageModel, - prompt: BasePromptTemplate = map_rerank_prompt.PROMPT, + prompt: BasePromptTemplate = MAP_RERANK_PROMPT, verbose: bool = False, document_variable_name: str = "context", rank_key: str = "score", diff --git a/langchain/chains/qa_with_sources/retrieval.py b/langchain/chains/qa_with_sources/retrieval.py index 1253da9450..beb36322c9 100644 --- a/langchain/chains/qa_with_sources/retrieval.py +++ b/langchain/chains/qa_with_sources/retrieval.py @@ -4,6 +4,10 @@ from typing import Any, Dict, List from pydantic import Field +from langchain.callbacks.manager import ( + AsyncCallbackManagerForChainRun, + CallbackManagerForChainRun, +) from langchain.chains.combine_documents.stuff import StuffDocumentsChain from langchain.chains.qa_with_sources.base import BaseQAWithSourcesChain from langchain.docstore.document import Document @@ -40,12 +44,20 @@ class RetrievalQAWithSourcesChain(BaseQAWithSourcesChain): return docs[:num_docs] - def _get_docs(self, inputs: Dict[str, Any]) -> List[Document]: + def _get_docs( + self, inputs: Dict[str, Any], *, run_manager: CallbackManagerForChainRun + ) -> List[Document]: question = inputs[self.question_key] - docs = self.retriever.get_relevant_documents(question) + docs = self.retriever.get_relevant_documents( + question, callbacks=run_manager.get_child() + ) return self._reduce_tokens_below_limit(docs) - async def _aget_docs(self, inputs: Dict[str, Any]) -> List[Document]: + async def _aget_docs( + self, inputs: Dict[str, Any], *, run_manager: AsyncCallbackManagerForChainRun + ) -> List[Document]: question = inputs[self.question_key] - docs = await self.retriever.aget_relevant_documents(question) + docs = await self.retriever.aget_relevant_documents( + question, callbacks=run_manager.get_child() + ) return self._reduce_tokens_below_limit(docs) diff --git a/langchain/chains/qa_with_sources/vector_db.py b/langchain/chains/qa_with_sources/vector_db.py index 4743961119..5d1689a9b1 100644 --- a/langchain/chains/qa_with_sources/vector_db.py +++ b/langchain/chains/qa_with_sources/vector_db.py @@ -5,6 +5,10 @@ from typing import Any, Dict, List from pydantic import Field, root_validator +from langchain.callbacks.manager import ( + AsyncCallbackManagerForChainRun, + CallbackManagerForChainRun, +) from langchain.chains.combine_documents.stuff import StuffDocumentsChain from langchain.chains.qa_with_sources.base import BaseQAWithSourcesChain from langchain.docstore.document import Document @@ -45,14 +49,18 @@ class VectorDBQAWithSourcesChain(BaseQAWithSourcesChain): return docs[:num_docs] - def _get_docs(self, inputs: Dict[str, Any]) -> List[Document]: + def _get_docs( + self, inputs: Dict[str, Any], *, run_manager: CallbackManagerForChainRun + ) -> List[Document]: question = inputs[self.question_key] docs = self.vectorstore.similarity_search( question, k=self.k, **self.search_kwargs ) return self._reduce_tokens_below_limit(docs) - async def _aget_docs(self, inputs: Dict[str, Any]) -> List[Document]: + async def _aget_docs( + self, inputs: Dict[str, Any], *, run_manager: AsyncCallbackManagerForChainRun + ) -> List[Document]: raise NotImplementedError("VectorDBQAWithSourcesChain does not support async") @root_validator() diff --git a/langchain/chains/question_answering/__init__.py b/langchain/chains/question_answering/__init__.py index 107415cdc2..b2cb1d52d9 100644 --- a/langchain/chains/question_answering/__init__.py +++ b/langchain/chains/question_answering/__init__.py @@ -12,10 +12,12 @@ from langchain.chains.combine_documents.stuff import StuffDocumentsChain from langchain.chains.llm import LLMChain from langchain.chains.question_answering import ( map_reduce_prompt, - map_rerank_prompt, refine_prompts, stuff_prompt, ) +from langchain.chains.question_answering.map_rerank_prompt import ( + PROMPT as MAP_RERANK_PROMPT, +) from langchain.prompts.base import BasePromptTemplate @@ -30,7 +32,7 @@ class LoadingCallable(Protocol): def _load_map_rerank_chain( llm: BaseLanguageModel, - prompt: BasePromptTemplate = map_rerank_prompt.PROMPT, + prompt: BasePromptTemplate = MAP_RERANK_PROMPT, verbose: bool = False, document_variable_name: str = "context", rank_key: str = "score", diff --git a/langchain/chains/retrieval_qa/base.py b/langchain/chains/retrieval_qa/base.py index 40ec2c8dfe..3ff10aa3d4 100644 --- a/langchain/chains/retrieval_qa/base.py +++ b/langchain/chains/retrieval_qa/base.py @@ -1,6 +1,7 @@ """Chain for question-answering against a vector database.""" from __future__ import annotations +import inspect import warnings from abc import abstractmethod from typing import Any, Dict, List, Optional @@ -94,7 +95,12 @@ class BaseRetrievalQA(Chain): return cls(combine_documents_chain=combine_documents_chain, **kwargs) @abstractmethod - def _get_docs(self, question: str) -> List[Document]: + def _get_docs( + self, + question: str, + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: """Get documents to do question answering over.""" def _call( @@ -115,8 +121,13 @@ class BaseRetrievalQA(Chain): """ _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager() question = inputs[self.input_key] - - docs = self._get_docs(question) + accepts_run_manager = ( + "run_manager" in inspect.signature(self._get_docs).parameters + ) + if accepts_run_manager: + docs = self._get_docs(question, run_manager=_run_manager) + else: + docs = self._get_docs(question) # type: ignore[call-arg] answer = self.combine_documents_chain.run( input_documents=docs, question=question, callbacks=_run_manager.get_child() ) @@ -127,7 +138,12 @@ class BaseRetrievalQA(Chain): return {self.output_key: answer} @abstractmethod - async def _aget_docs(self, question: str) -> List[Document]: + async def _aget_docs( + self, + question: str, + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: """Get documents to do question answering over.""" async def _acall( @@ -148,8 +164,13 @@ class BaseRetrievalQA(Chain): """ _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager() question = inputs[self.input_key] - - docs = await self._aget_docs(question) + accepts_run_manager = ( + "run_manager" in inspect.signature(self._aget_docs).parameters + ) + if accepts_run_manager: + docs = await self._aget_docs(question, run_manager=_run_manager) + else: + docs = await self._aget_docs(question) # type: ignore[call-arg] answer = await self.combine_documents_chain.arun( input_documents=docs, question=question, callbacks=_run_manager.get_child() ) @@ -177,11 +198,27 @@ class RetrievalQA(BaseRetrievalQA): retriever: BaseRetriever = Field(exclude=True) - def _get_docs(self, question: str) -> List[Document]: - return self.retriever.get_relevant_documents(question) + def _get_docs( + self, + question: str, + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" + return self.retriever.get_relevant_documents( + question, callbacks=run_manager.get_child() + ) - async def _aget_docs(self, question: str) -> List[Document]: - return await self.retriever.aget_relevant_documents(question) + async def _aget_docs( + self, + question: str, + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" + return await self.retriever.aget_relevant_documents( + question, callbacks=run_manager.get_child() + ) @property def _chain_type(self) -> str: @@ -218,7 +255,13 @@ class VectorDBQA(BaseRetrievalQA): raise ValueError(f"search_type of {search_type} not allowed.") return values - def _get_docs(self, question: str) -> List[Document]: + def _get_docs( + self, + question: str, + *, + run_manager: CallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" if self.search_type == "similarity": docs = self.vectorstore.similarity_search( question, k=self.k, **self.search_kwargs @@ -231,7 +274,13 @@ class VectorDBQA(BaseRetrievalQA): raise ValueError(f"search_type of {self.search_type} not allowed.") return docs - async def _aget_docs(self, question: str) -> List[Document]: + async def _aget_docs( + self, + question: str, + *, + run_manager: AsyncCallbackManagerForChainRun, + ) -> List[Document]: + """Get docs.""" raise NotImplementedError("VectorDBQA does not support async") @property diff --git a/langchain/experimental/client/tracing_datasets.ipynb b/langchain/experimental/client/tracing_datasets.ipynb index d7fb757942..e72161b212 100644 --- a/langchain/experimental/client/tracing_datasets.ipynb +++ b/langchain/experimental/client/tracing_datasets.ipynb @@ -697,7 +697,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.2" } }, "nbformat": 4, diff --git a/langchain/retrievers/__init__.py b/langchain/retrievers/__init__.py index 5fc3d0b8c0..137a8ae19a 100644 --- a/langchain/retrievers/__init__.py +++ b/langchain/retrievers/__init__.py @@ -16,7 +16,7 @@ from langchain.retrievers.metal import MetalRetriever from langchain.retrievers.milvus import MilvusRetriever from langchain.retrievers.multi_query import MultiQueryRetriever from langchain.retrievers.pinecone_hybrid_search import PineconeHybridSearchRetriever -from langchain.retrievers.pupmed import PubMedRetriever +from langchain.retrievers.pubmed import PubMedRetriever from langchain.retrievers.remote_retriever import RemoteLangChainRetriever from langchain.retrievers.self_query.base import SelfQueryRetriever from langchain.retrievers.svm import SVMRetriever diff --git a/langchain/retrievers/arxiv.py b/langchain/retrievers/arxiv.py index 6ff279126f..46223397b7 100644 --- a/langchain/retrievers/arxiv.py +++ b/langchain/retrievers/arxiv.py @@ -1,5 +1,9 @@ -from typing import List +from typing import Any, List +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document from langchain.utilities.arxiv import ArxivAPIWrapper @@ -11,8 +15,20 @@ class ArxivRetriever(BaseRetriever, ArxivAPIWrapper): It uses all ArxivAPIWrapper arguments without any change. """ - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: return self.load(query=query) - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/azure_cognitive_search.py b/langchain/retrievers/azure_cognitive_search.py index f1f0cbaffd..d79053c7e9 100644 --- a/langchain/retrievers/azure_cognitive_search.py +++ b/langchain/retrievers/azure_cognitive_search.py @@ -1,13 +1,18 @@ """Retriever wrapper for Azure Cognitive Search.""" + from __future__ import annotations import json -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import aiohttp import requests from pydantic import BaseModel, Extra, root_validator +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document from langchain.utils import get_from_dict_or_env @@ -81,7 +86,13 @@ class AzureCognitiveSearchRetriever(BaseRetriever, BaseModel): return response_json["value"] - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: search_results = self._search(query) return [ @@ -89,7 +100,13 @@ class AzureCognitiveSearchRetriever(BaseRetriever, BaseModel): for result in search_results ] - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: search_results = await self._asearch(query) return [ diff --git a/langchain/retrievers/chatgpt_plugin_retriever.py b/langchain/retrievers/chatgpt_plugin_retriever.py index e0f3f13c65..5da8d408d9 100644 --- a/langchain/retrievers/chatgpt_plugin_retriever.py +++ b/langchain/retrievers/chatgpt_plugin_retriever.py @@ -1,11 +1,15 @@ from __future__ import annotations -from typing import List, Optional +from typing import Any, List, Optional import aiohttp import requests from pydantic import BaseModel +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -21,7 +25,13 @@ class ChatGPTPluginRetriever(BaseRetriever, BaseModel): arbitrary_types_allowed = True - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: url, json, headers = self._create_request(query) response = requests.post(url, json=json, headers=headers) results = response.json()["results"][0]["results"] @@ -34,7 +44,13 @@ class ChatGPTPluginRetriever(BaseRetriever, BaseModel): docs.append(Document(page_content=content, metadata=metadata)) return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: url, json, headers = self._create_request(query) if not self.aiosession: diff --git a/langchain/retrievers/contextual_compression.py b/langchain/retrievers/contextual_compression.py index 8850991d2a..634706d280 100644 --- a/langchain/retrievers/contextual_compression.py +++ b/langchain/retrievers/contextual_compression.py @@ -1,8 +1,13 @@ """Retriever that wraps a base retriever and filters the results.""" -from typing import List + +from typing import Any, List from pydantic import BaseModel, Extra +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.retrievers.document_compressors.base import ( BaseDocumentCompressor, ) @@ -24,7 +29,13 @@ class ContextualCompressionRetriever(BaseRetriever, BaseModel): extra = Extra.forbid arbitrary_types_allowed = True - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Get documents relevant for a query. Args: @@ -33,14 +44,24 @@ class ContextualCompressionRetriever(BaseRetriever, BaseModel): Returns: Sequence of relevant documents """ - docs = self.base_retriever.get_relevant_documents(query) + docs = self.base_retriever.get_relevant_documents( + query, callbacks=run_manager.get_child(), **kwargs + ) if docs: - compressed_docs = self.base_compressor.compress_documents(docs, query) + compressed_docs = self.base_compressor.compress_documents( + docs, query, callbacks=run_manager.get_child() + ) return list(compressed_docs) else: return [] - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Get documents relevant for a query. Args: @@ -49,10 +70,12 @@ class ContextualCompressionRetriever(BaseRetriever, BaseModel): Returns: List of relevant documents """ - docs = await self.base_retriever.aget_relevant_documents(query) + docs = await self.base_retriever.aget_relevant_documents( + query, callbacks=run_manager.get_child(), **kwargs + ) if docs: compressed_docs = await self.base_compressor.acompress_documents( - docs, query + docs, query, callbacks=run_manager.get_child() ) return list(compressed_docs) else: diff --git a/langchain/retrievers/databerry.py b/langchain/retrievers/databerry.py index bfc6abaa7f..e124d503b4 100644 --- a/langchain/retrievers/databerry.py +++ b/langchain/retrievers/databerry.py @@ -1,8 +1,12 @@ -from typing import List, Optional +from typing import Any, List, Optional import aiohttp import requests +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -23,7 +27,13 @@ class DataberryRetriever(BaseRetriever): self.api_key = api_key self.top_k = top_k - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: response = requests.post( self.datastore_url, json={ @@ -48,7 +58,13 @@ class DataberryRetriever(BaseRetriever): for r in data["results"] ] - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: async with aiohttp.ClientSession() as session: async with session.request( "POST", diff --git a/langchain/retrievers/docarray.py b/langchain/retrievers/docarray.py index 5c99f8a2f3..37cf155593 100644 --- a/langchain/retrievers/docarray.py +++ b/langchain/retrievers/docarray.py @@ -4,6 +4,10 @@ from typing import Any, Dict, List, Optional, Union import numpy as np from pydantic import BaseModel +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever, Document from langchain.vectorstores.utils import maximal_marginal_relevance @@ -49,7 +53,12 @@ class DocArrayRetriever(BaseRetriever, BaseModel): arbitrary_types_allowed = True - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Get documents relevant for a query. Args: @@ -201,5 +210,10 @@ class DocArrayRetriever(BaseRetriever, BaseModel): return lc_doc - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/document_compressors/base.py b/langchain/retrievers/document_compressors/base.py index 6c697f55d5..1e0587c59c 100644 --- a/langchain/retrievers/document_compressors/base.py +++ b/langchain/retrievers/document_compressors/base.py @@ -1,9 +1,11 @@ """Interface for retrieved document compressors.""" from abc import ABC, abstractmethod -from typing import List, Sequence, Union +from inspect import signature +from typing import List, Optional, Sequence, Union from pydantic import BaseModel +from langchain.callbacks.manager import Callbacks from langchain.schema import BaseDocumentTransformer, Document @@ -12,13 +14,19 @@ class BaseDocumentCompressor(BaseModel, ABC): @abstractmethod def compress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Compress retrieved documents given the query context.""" @abstractmethod async def acompress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Compress retrieved documents given the query context.""" @@ -35,12 +43,26 @@ class DocumentCompressorPipeline(BaseDocumentCompressor): arbitrary_types_allowed = True def compress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Transform a list of documents.""" for _transformer in self.transformers: if isinstance(_transformer, BaseDocumentCompressor): - documents = _transformer.compress_documents(documents, query) + accepts_callbacks = ( + signature(_transformer.compress_documents).parameters.get( + "callbacks" + ) + is not None + ) + if accepts_callbacks: + documents = _transformer.compress_documents( + documents, query, callbacks=callbacks + ) + else: + documents = _transformer.compress_documents(documents, query) elif isinstance(_transformer, BaseDocumentTransformer): documents = _transformer.transform_documents(documents) else: @@ -48,12 +70,26 @@ class DocumentCompressorPipeline(BaseDocumentCompressor): return documents async def acompress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Compress retrieved documents given the query context.""" for _transformer in self.transformers: if isinstance(_transformer, BaseDocumentCompressor): - documents = await _transformer.acompress_documents(documents, query) + accepts_callbacks = ( + signature(_transformer.acompress_documents).parameters.get( + "callbacks" + ) + is not None + ) + if accepts_callbacks: + documents = await _transformer.acompress_documents( + documents, query, callbacks=callbacks + ) + else: + documents = await _transformer.acompress_documents(documents, query) elif isinstance(_transformer, BaseDocumentTransformer): documents = await _transformer.atransform_documents(documents) else: diff --git a/langchain/retrievers/document_compressors/chain_extract.py b/langchain/retrievers/document_compressors/chain_extract.py index 9b7947c413..94ff43842f 100644 --- a/langchain/retrievers/document_compressors/chain_extract.py +++ b/langchain/retrievers/document_compressors/chain_extract.py @@ -6,6 +6,7 @@ from typing import Any, Callable, Dict, Optional, Sequence from langchain import LLMChain, PromptTemplate from langchain.base_language import BaseLanguageModel +from langchain.callbacks.manager import Callbacks from langchain.retrievers.document_compressors.base import BaseDocumentCompressor from langchain.retrievers.document_compressors.chain_extract_prompt import ( prompt_template, @@ -48,25 +49,33 @@ class LLMChainExtractor(BaseDocumentCompressor): """Callable for constructing the chain input from the query and a Document.""" def compress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Compress page content of raw documents.""" compressed_docs = [] for doc in documents: _input = self.get_input(query, doc) - output = self.llm_chain.predict_and_parse(**_input) + output = self.llm_chain.predict_and_parse(**_input, callbacks=callbacks) if len(output) == 0: continue compressed_docs.append(Document(page_content=output, metadata=doc.metadata)) return compressed_docs async def acompress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Compress page content of raw documents asynchronously.""" outputs = await asyncio.gather( *[ - self.llm_chain.apredict_and_parse(**self.get_input(query, doc)) + self.llm_chain.apredict_and_parse( + **self.get_input(query, doc), callbacks=callbacks + ) for doc in documents ] ) diff --git a/langchain/retrievers/document_compressors/chain_filter.py b/langchain/retrievers/document_compressors/chain_filter.py index 245e005108..ad44158fc8 100644 --- a/langchain/retrievers/document_compressors/chain_filter.py +++ b/langchain/retrievers/document_compressors/chain_filter.py @@ -3,6 +3,7 @@ from typing import Any, Callable, Dict, Optional, Sequence from langchain import BasePromptTemplate, LLMChain, PromptTemplate from langchain.base_language import BaseLanguageModel +from langchain.callbacks.manager import Callbacks from langchain.output_parsers.boolean import BooleanOutputParser from langchain.retrievers.document_compressors.base import BaseDocumentCompressor from langchain.retrievers.document_compressors.chain_filter_prompt import ( @@ -35,19 +36,27 @@ class LLMChainFilter(BaseDocumentCompressor): """Callable for constructing the chain input from the query and a Document.""" def compress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Filter down documents based on their relevance to the query.""" filtered_docs = [] for doc in documents: _input = self.get_input(query, doc) - include_doc = self.llm_chain.predict_and_parse(**_input) + include_doc = self.llm_chain.predict_and_parse( + **_input, callbacks=callbacks + ) if include_doc: filtered_docs.append(doc) return filtered_docs async def acompress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Filter down documents.""" raise NotImplementedError diff --git a/langchain/retrievers/document_compressors/cohere_rerank.py b/langchain/retrievers/document_compressors/cohere_rerank.py index 63060914c7..dd63b84242 100644 --- a/langchain/retrievers/document_compressors/cohere_rerank.py +++ b/langchain/retrievers/document_compressors/cohere_rerank.py @@ -1,9 +1,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Sequence +from typing import TYPE_CHECKING, Dict, Optional, Sequence from pydantic import Extra, root_validator +from langchain.callbacks.manager import Callbacks from langchain.retrievers.document_compressors.base import BaseDocumentCompressor from langchain.schema import Document from langchain.utils import get_from_dict_or_env @@ -48,7 +49,10 @@ class CohereRerank(BaseDocumentCompressor): return values def compress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: if len(documents) == 0: # to avoid empty api call return [] @@ -65,6 +69,9 @@ class CohereRerank(BaseDocumentCompressor): return final_results async def acompress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: raise NotImplementedError diff --git a/langchain/retrievers/document_compressors/embeddings_filter.py b/langchain/retrievers/document_compressors/embeddings_filter.py index 543380189d..a11dfc4980 100644 --- a/langchain/retrievers/document_compressors/embeddings_filter.py +++ b/langchain/retrievers/document_compressors/embeddings_filter.py @@ -4,6 +4,7 @@ from typing import Callable, Dict, Optional, Sequence import numpy as np from pydantic import root_validator +from langchain.callbacks.manager import Callbacks from langchain.document_transformers import ( _get_embeddings_from_stateful_docs, get_stateful_documents, @@ -44,7 +45,10 @@ class EmbeddingsFilter(BaseDocumentCompressor): return values def compress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Filter documents based on similarity of their embeddings to the query.""" stateful_documents = get_stateful_documents(documents) @@ -64,7 +68,10 @@ class EmbeddingsFilter(BaseDocumentCompressor): return [stateful_documents[i] for i in included_idxs] async def acompress_documents( - self, documents: Sequence[Document], query: str + self, + documents: Sequence[Document], + query: str, + callbacks: Optional[Callbacks] = None, ) -> Sequence[Document]: """Filter down documents.""" raise NotImplementedError diff --git a/langchain/retrievers/elastic_search_bm25.py b/langchain/retrievers/elastic_search_bm25.py index 3fe61aa589..bdcfcaa116 100644 --- a/langchain/retrievers/elastic_search_bm25.py +++ b/langchain/retrievers/elastic_search_bm25.py @@ -1,9 +1,14 @@ """Wrapper around Elasticsearch vector database.""" + from __future__ import annotations import uuid from typing import Any, Iterable, List +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.schema import BaseRetriever @@ -111,7 +116,13 @@ class ElasticSearchBM25Retriever(BaseRetriever): self.client.indices.refresh(index=self.index_name) return ids - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: query_dict = {"query": {"match": {"content": query}}} res = self.client.search(index=self.index_name, body=query_dict) @@ -120,5 +131,11 @@ class ElasticSearchBM25Retriever(BaseRetriever): docs.append(Document(page_content=r["_source"]["content"])) return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/kendra.py b/langchain/retrievers/kendra.py index c86b812244..b748c11ad3 100644 --- a/langchain/retrievers/kendra.py +++ b/langchain/retrievers/kendra.py @@ -3,6 +3,10 @@ from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, Extra +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.schema import BaseRetriever @@ -257,7 +261,12 @@ class AmazonKendraRetriever(BaseRetriever): docs = r_result.get_top_k_docs(top_k) return docs - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Run search on Kendra index and get top k documents Example: @@ -269,5 +278,10 @@ class AmazonKendraRetriever(BaseRetriever): docs = self._kendra_query(query, self.top_k, self.attribute_filter) return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError("Async version is not implemented for Kendra yet.") diff --git a/langchain/retrievers/knn.py b/langchain/retrievers/knn.py index a7ab478d34..924f991601 100644 --- a/langchain/retrievers/knn.py +++ b/langchain/retrievers/knn.py @@ -10,6 +10,10 @@ from typing import Any, List, Optional import numpy as np from pydantic import BaseModel +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever, Document @@ -51,7 +55,13 @@ class KNNRetriever(BaseRetriever, BaseModel): index = create_index(texts, embeddings) return cls(embeddings=embeddings, index=index, texts=texts, **kwargs) - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: query_embeds = np.array(self.embeddings.embed_query(query)) # calc L2 norm index_embeds = self.index / np.sqrt((self.index**2).sum(1, keepdims=True)) @@ -73,5 +83,11 @@ class KNNRetriever(BaseRetriever, BaseModel): ] return top_k_results - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/llama_index.py b/langchain/retrievers/llama_index.py index 9e650c53c9..f9cf3b36ad 100644 --- a/langchain/retrievers/llama_index.py +++ b/langchain/retrievers/llama_index.py @@ -1,7 +1,11 @@ -from typing import Any, Dict, List, cast +from typing import Any, Dict, List, Optional, cast from pydantic import BaseModel, Field +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -11,7 +15,13 @@ class LlamaIndexRetriever(BaseRetriever, BaseModel): index: Any query_kwargs: Dict = Field(default_factory=dict) - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Get documents relevant for a query.""" try: from llama_index.indices.base import BaseGPTIndex @@ -33,7 +43,13 @@ class LlamaIndexRetriever(BaseRetriever, BaseModel): ) return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: Optional[AsyncCallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError("LlamaIndexRetriever does not support async") @@ -43,7 +59,13 @@ class LlamaIndexGraphRetriever(BaseRetriever, BaseModel): graph: Any query_configs: List[Dict] = Field(default_factory=list) - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Get documents relevant for a query.""" try: from llama_index.composability.graph import ( @@ -73,5 +95,11 @@ class LlamaIndexGraphRetriever(BaseRetriever, BaseModel): ) return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError("LlamaIndexGraphRetriever does not support async") diff --git a/langchain/retrievers/merger_retriever.py b/langchain/retrievers/merger_retriever.py index a9dccdc4ba..ec8dce44f1 100644 --- a/langchain/retrievers/merger_retriever.py +++ b/langchain/retrievers/merger_retriever.py @@ -1,5 +1,9 @@ -from typing import List +from typing import Any, List +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -24,7 +28,12 @@ class MergerRetriever(BaseRetriever): self.retrievers = retrievers - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """ Get the relevant documents for a given query. @@ -36,11 +45,16 @@ class MergerRetriever(BaseRetriever): """ # Merge the results of the retrievers. - merged_documents = self.merge_documents(query) + merged_documents = self.merge_documents(query, run_manager) return merged_documents - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """ Asynchronously get the relevant documents for a given query. @@ -52,11 +66,13 @@ class MergerRetriever(BaseRetriever): """ # Merge the results of the retrievers. - merged_documents = await self.amerge_documents(query) + merged_documents = await self.amerge_documents(query, run_manager) return merged_documents - def merge_documents(self, query: str) -> List[Document]: + def merge_documents( + self, query: str, run_manager: CallbackManagerForRetrieverRun + ) -> List[Document]: """ Merge the results of the retrievers. @@ -69,7 +85,10 @@ class MergerRetriever(BaseRetriever): # Get the results of all retrievers. retriever_docs = [ - retriever.get_relevant_documents(query) for retriever in self.retrievers + retriever.get_relevant_documents( + query, callbacks=run_manager.get_child("retriever_{}".format(i + 1)) + ) + for i, retriever in enumerate(self.retrievers) ] # Merge the results of the retrievers. @@ -82,7 +101,9 @@ class MergerRetriever(BaseRetriever): return merged_documents - async def amerge_documents(self, query: str) -> List[Document]: + async def amerge_documents( + self, query: str, run_manager: AsyncCallbackManagerForRetrieverRun + ) -> List[Document]: """ Asynchronously merge the results of the retrievers. @@ -95,8 +116,10 @@ class MergerRetriever(BaseRetriever): # Get the results of all retrievers. retriever_docs = [ - await retriever.aget_relevant_documents(query) - for retriever in self.retrievers + await retriever.aget_relevant_documents( + query, callbacks=run_manager.get_child("retriever_{}".format(i + 1)) + ) + for i, retriever in enumerate(self.retrievers) ] # Merge the results of the retrievers. diff --git a/langchain/retrievers/metal.py b/langchain/retrievers/metal.py index 561f8dab34..ec6824eb79 100644 --- a/langchain/retrievers/metal.py +++ b/langchain/retrievers/metal.py @@ -1,5 +1,9 @@ from typing import Any, List, Optional +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -17,7 +21,13 @@ class MetalRetriever(BaseRetriever): self.client: Metal = client self.params = params or {} - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: results = self.client.search({"text": query}, **self.params) final_results = [] for r in results["data"]: @@ -25,5 +35,11 @@ class MetalRetriever(BaseRetriever): final_results.append(Document(page_content=r["text"], metadata=metadata)) return final_results - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/milvus.py b/langchain/retrievers/milvus.py index 2260aa37bb..3255cde4ba 100644 --- a/langchain/retrievers/milvus.py +++ b/langchain/retrievers/milvus.py @@ -2,6 +2,10 @@ import warnings from typing import Any, Dict, List, Optional +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever, Document from langchain.vectorstores.milvus import Milvus @@ -39,10 +43,24 @@ class MilvusRetriever(BaseRetriever): """ self.store.add_texts(texts, metadatas) - def get_relevant_documents(self, query: str) -> List[Document]: - return self.retriever.get_relevant_documents(query) + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + return self.retriever.get_relevant_documents( + query, run_manager=run_manager.get_child(), **kwargs + ) - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/multi_query.py b/langchain/retrievers/multi_query.py index 925a7d0a9a..bf330284cc 100644 --- a/langchain/retrievers/multi_query.py +++ b/langchain/retrievers/multi_query.py @@ -1,8 +1,12 @@ import logging -from typing import List +from typing import Any, List from pydantic import BaseModel, Field +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.chains.llm import LLMChain from langchain.llms.base import BaseLLM from langchain.output_parsers.pydantic import PydanticOutputParser @@ -91,7 +95,12 @@ class MultiQueryRetriever(BaseRetriever): parser_key=parser_key, ) - def get_relevant_documents(self, question: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Get relevated documents given a user query. Args: @@ -100,15 +109,22 @@ class MultiQueryRetriever(BaseRetriever): Returns: Unique union of relevant documents from all generated queries """ - queries = self.generate_queries(question) - documents = self.retrieve_documents(queries) + queries = self.generate_queries(query, run_manager) + documents = self.retrieve_documents(queries, run_manager) unique_documents = self.unique_union(documents) return unique_documents - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError - def generate_queries(self, question: str) -> List[str]: + def generate_queries( + self, question: str, run_manager: CallbackManagerForRetrieverRun + ) -> List[str]: """Generate queries based upon user input. Args: @@ -117,13 +133,17 @@ class MultiQueryRetriever(BaseRetriever): Returns: List of LLM generated queries that are similar to the user input """ - response = self.llm_chain({"question": question}) + response = self.llm_chain( + {"question": question}, callbacks=run_manager.get_child() + ) lines = getattr(response["text"], self.parser_key, []) if self.verbose: logger.info(f"Generated queries: {lines}") return lines - def retrieve_documents(self, queries: List[str]) -> List[Document]: + def retrieve_documents( + self, queries: List[str], run_manager: CallbackManagerForRetrieverRun + ) -> List[Document]: """Run all LLM generated queries. Args: @@ -134,7 +154,9 @@ class MultiQueryRetriever(BaseRetriever): """ documents = [] for query in queries: - docs = self.retriever.get_relevant_documents(query) + docs = self.retriever.get_relevant_documents( + query, callbacks=run_manager.get_child() + ) documents.extend(docs) return documents diff --git a/langchain/retrievers/pinecone_hybrid_search.py b/langchain/retrievers/pinecone_hybrid_search.py index c6cd0c0ad5..481856d65f 100644 --- a/langchain/retrievers/pinecone_hybrid_search.py +++ b/langchain/retrievers/pinecone_hybrid_search.py @@ -1,9 +1,14 @@ """Taken from: https://docs.pinecone.io/docs/hybrid-search""" + import hashlib from typing import Any, Dict, List, Optional from pydantic import BaseModel, Extra, root_validator +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever, Document @@ -137,7 +142,13 @@ class PineconeHybridSearchRetriever(BaseRetriever, BaseModel): ) return values - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: from pinecone_text.hybrid import hybrid_convex_scale sparse_vec = self.sparse_encoder.encode_queries(query) @@ -162,5 +173,11 @@ class PineconeHybridSearchRetriever(BaseRetriever, BaseModel): # return search results as json return final_result - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/pubmed.py b/langchain/retrievers/pubmed.py new file mode 100644 index 0000000000..edb05063fb --- /dev/null +++ b/langchain/retrievers/pubmed.py @@ -0,0 +1,35 @@ +"""A retriever that uses PubMed API to retrieve documents.""" +from typing import Any, List + +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) +from langchain.schema import BaseRetriever, Document +from langchain.utilities.pupmed import PubMedAPIWrapper + + +class PubMedRetriever(BaseRetriever, PubMedAPIWrapper): + """ + It is effectively a wrapper for PubMedAPIWrapper. + It wraps load() to get_relevant_documents(). + It uses all PubMedAPIWrapper arguments without any change. + """ + + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + return self.load_docs(query=query) + + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + raise NotImplementedError diff --git a/langchain/retrievers/pupmed.py b/langchain/retrievers/pupmed.py index c898afdae3..3104dac6f4 100644 --- a/langchain/retrievers/pupmed.py +++ b/langchain/retrievers/pupmed.py @@ -1,18 +1,5 @@ -from typing import List +from langchain.retrievers.pubmed import PubMedRetriever -from langchain.schema import BaseRetriever, Document -from langchain.utilities.pupmed import PubMedAPIWrapper - - -class PubMedRetriever(BaseRetriever, PubMedAPIWrapper): - """ - It is effectively a wrapper for PubMedAPIWrapper. - It wraps load() to get_relevant_documents(). - It uses all PubMedAPIWrapper arguments without any change. - """ - - def get_relevant_documents(self, query: str) -> List[Document]: - return self.load_docs(query=query) - - async def aget_relevant_documents(self, query: str) -> List[Document]: - raise NotImplementedError +__all__ = [ + "PubMedRetriever", +] diff --git a/langchain/retrievers/remote_retriever.py b/langchain/retrievers/remote_retriever.py index 53b2e7dd79..f6d8758d02 100644 --- a/langchain/retrievers/remote_retriever.py +++ b/langchain/retrievers/remote_retriever.py @@ -1,9 +1,13 @@ -from typing import List, Optional +from typing import Any, List, Optional import aiohttp import requests from pydantic import BaseModel +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -15,7 +19,13 @@ class RemoteLangChainRetriever(BaseRetriever, BaseModel): page_content_key: str = "page_content" metadata_key: str = "metadata" - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: response = requests.post( self.url, json={self.input_key: query}, headers=self.headers ) @@ -27,7 +37,13 @@ class RemoteLangChainRetriever(BaseRetriever, BaseModel): for r in result[self.response_key] ] - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: async with aiohttp.ClientSession() as session: async with session.request( "POST", self.url, headers=self.headers, json={self.input_key: query} diff --git a/langchain/retrievers/self_query/base.py b/langchain/retrievers/self_query/base.py index f4d99816b3..859d4d1d92 100644 --- a/langchain/retrievers/self_query/base.py +++ b/langchain/retrievers/self_query/base.py @@ -1,11 +1,15 @@ """Retriever that generates and executes structured queries over its own data source.""" + from typing import Any, Dict, List, Optional, Type, cast from pydantic import BaseModel, Field, root_validator from langchain import LLMChain from langchain.base_language import BaseLanguageModel -from langchain.callbacks.manager import Callbacks +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.chains.query_constructor.base import load_query_constructor_chain from langchain.chains.query_constructor.ir import StructuredQuery, Visitor from langchain.chains.query_constructor.schema import AttributeInfo @@ -79,8 +83,12 @@ class SelfQueryRetriever(BaseRetriever, BaseModel): ) return values - def get_relevant_documents( - self, query: str, callbacks: Callbacks = None + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, ) -> List[Document]: """Get documents relevant for a query. @@ -93,7 +101,9 @@ class SelfQueryRetriever(BaseRetriever, BaseModel): inputs = self.llm_chain.prep_inputs({"query": query}) structured_query = cast( StructuredQuery, - self.llm_chain.predict_and_parse(callbacks=callbacks, **inputs), + self.llm_chain.predict_and_parse( + callbacks=run_manager.get_child(), **inputs + ), ) if self.verbose: print(structured_query) @@ -110,7 +120,13 @@ class SelfQueryRetriever(BaseRetriever, BaseModel): docs = self.vectorstore.search(new_query, self.search_type, **search_kwargs) return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: Optional[AsyncCallbackManagerForRetrieverRun], + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError @classmethod diff --git a/langchain/retrievers/svm.py b/langchain/retrievers/svm.py index 15fe8e7fa8..2e96781220 100644 --- a/langchain/retrievers/svm.py +++ b/langchain/retrievers/svm.py @@ -10,6 +10,10 @@ from typing import Any, List, Optional import numpy as np from pydantic import BaseModel +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever, Document @@ -50,7 +54,13 @@ class SVMRetriever(BaseRetriever, BaseModel): index = create_index(texts, embeddings) return cls(embeddings=embeddings, index=index, texts=texts, **kwargs) - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: from sklearn import svm query_embeds = np.array(self.embeddings.embed_query(query)) @@ -87,5 +97,11 @@ class SVMRetriever(BaseRetriever, BaseModel): top_k_results.append(Document(page_content=self.texts[row - 1])) return top_k_results - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/tfidf.py b/langchain/retrievers/tfidf.py index db14f61e27..71b1afb473 100644 --- a/langchain/retrievers/tfidf.py +++ b/langchain/retrievers/tfidf.py @@ -2,12 +2,17 @@ Largely based on https://github.com/asvskartheek/Text-Retrieval/blob/master/TF-IDF%20Search%20Engine%20(SKLEARN).ipynb""" + from __future__ import annotations from typing import Any, Dict, Iterable, List, Optional from pydantic import BaseModel +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document @@ -58,7 +63,13 @@ class TFIDFRetriever(BaseRetriever, BaseModel): texts=texts, tfidf_params=tfidf_params, metadatas=metadatas, **kwargs ) - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: from sklearn.metrics.pairwise import cosine_similarity query_vec = self.vectorizer.transform( @@ -70,5 +81,11 @@ class TFIDFRetriever(BaseRetriever, BaseModel): return_docs = [self.docs[i] for i in results.argsort()[-self.k :][::-1]] return return_docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/time_weighted_retriever.py b/langchain/retrievers/time_weighted_retriever.py index b46325f4ca..0f0681ad5b 100644 --- a/langchain/retrievers/time_weighted_retriever.py +++ b/langchain/retrievers/time_weighted_retriever.py @@ -1,10 +1,15 @@ """Retriever that combines embedding similarity with recency in retrieving values.""" + import datetime from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple from pydantic import BaseModel, Field +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document from langchain.vectorstores.base import VectorStore @@ -80,7 +85,13 @@ class TimeWeightedVectorStoreRetriever(BaseRetriever, BaseModel): results[buffer_idx] = (doc, relevance) return results - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Return documents that are relevant to the query.""" current_time = datetime.datetime.now() docs_and_scores = { @@ -103,7 +114,13 @@ class TimeWeightedVectorStoreRetriever(BaseRetriever, BaseModel): result.append(buffered_doc) return result - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: """Return documents that are relevant to the query.""" raise NotImplementedError diff --git a/langchain/retrievers/vespa_retriever.py b/langchain/retrievers/vespa_retriever.py index 7f0d0045eb..b95ce59c42 100644 --- a/langchain/retrievers/vespa_retriever.py +++ b/langchain/retrievers/vespa_retriever.py @@ -1,9 +1,14 @@ """Wrapper for retrieving documents from Vespa.""" + from __future__ import annotations import json from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Sequence, Union +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document if TYPE_CHECKING: @@ -59,12 +64,24 @@ class VespaRetriever(BaseRetriever): docs.append(Document(page_content=page_content, metadata=metadata)) return docs - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: body = self._query_body.copy() body["query"] = query return self._query(body) - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError def get_relevant_documents_with_filter( diff --git a/langchain/retrievers/weaviate_hybrid_search.py b/langchain/retrievers/weaviate_hybrid_search.py index 42157a6e51..b34edd9440 100644 --- a/langchain/retrievers/weaviate_hybrid_search.py +++ b/langchain/retrievers/weaviate_hybrid_search.py @@ -1,4 +1,5 @@ """Wrapper around weaviate vector database.""" + from __future__ import annotations from typing import Any, Dict, List, Optional @@ -6,6 +7,10 @@ from uuid import uuid4 from pydantic import Extra +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.schema import BaseRetriever @@ -82,8 +87,13 @@ class WeaviateHybridSearchRetriever(BaseRetriever): ids.append(_id) return ids - def get_relevant_documents( - self, query: str, where_filter: Optional[Dict[str, object]] = None + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + where_filter: Optional[Dict[str, object]] = None, + **kwargs: Any, ) -> List[Document]: """Look up similar documents in Weaviate.""" query_obj = self._client.query.get(self._index_name, self._query_attrs) @@ -101,7 +111,12 @@ class WeaviateHybridSearchRetriever(BaseRetriever): docs.append(Document(page_content=text, metadata=res)) return docs - async def aget_relevant_documents( - self, query: str, where_filter: Optional[Dict[str, object]] = None + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + where_filter: Optional[Dict[str, object]] = None, + **kwargs: Any, ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/wikipedia.py b/langchain/retrievers/wikipedia.py index ab857f1b93..013728396a 100644 --- a/langchain/retrievers/wikipedia.py +++ b/langchain/retrievers/wikipedia.py @@ -1,5 +1,9 @@ -from typing import List +from typing import Any, List +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document from langchain.utilities.wikipedia import WikipediaAPIWrapper @@ -11,8 +15,20 @@ class WikipediaRetriever(BaseRetriever, WikipediaAPIWrapper): It uses all WikipediaAPIWrapper arguments without any change. """ - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: return self.load(query=query) - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/retrievers/zep.py b/langchain/retrievers/zep.py index fff01da0d6..d6d43bf1b8 100644 --- a/langchain/retrievers/zep.py +++ b/langchain/retrievers/zep.py @@ -1,7 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.schema import BaseRetriever, Document if TYPE_CHECKING: @@ -54,8 +58,13 @@ class ZepRetriever(BaseRetriever): if r.message ] - def get_relevant_documents( - self, query: str, metadata: Optional[Dict] = None + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + metadata: Optional[Dict] = None, + **kwargs: Any, ) -> List[Document]: from zep_python import MemorySearchPayload @@ -69,8 +78,13 @@ class ZepRetriever(BaseRetriever): return self._search_result_to_doc(results) - async def aget_relevant_documents( - self, query: str, metadata: Optional[Dict] = None + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + metadata: Optional[Dict] = None, + **kwargs: Any, ) -> List[Document]: from zep_python import MemorySearchPayload diff --git a/langchain/retrievers/zilliz.py b/langchain/retrievers/zilliz.py index 52f567e8ef..810f6ae4a0 100644 --- a/langchain/retrievers/zilliz.py +++ b/langchain/retrievers/zilliz.py @@ -2,6 +2,10 @@ import warnings from typing import Any, Dict, List, Optional +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever, Document from langchain.vectorstores.zilliz import Zilliz @@ -39,10 +43,24 @@ class ZillizRetriever(BaseRetriever): """ self.store.add_texts(texts, metadatas) - def get_relevant_documents(self, query: str) -> List[Document]: - return self.retriever.get_relevant_documents(query) + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + return self.retriever.get_relevant_documents( + query, run_manager=run_manager.get_child(), **kwargs + ) - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError diff --git a/langchain/schema.py b/langchain/schema.py index 07106699b5..34b091bae5 100644 --- a/langchain/schema.py +++ b/langchain/schema.py @@ -1,9 +1,12 @@ """Common schema objects.""" from __future__ import annotations +import warnings from abc import ABC, abstractmethod from dataclasses import dataclass +from inspect import signature from typing import ( + TYPE_CHECKING, Any, Dict, Generic, @@ -20,6 +23,13 @@ from pydantic import BaseModel, Field, root_validator from langchain.load.serializable import Serializable +if TYPE_CHECKING: + from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, + Callbacks, + ) + RUN_KEY = "__run" @@ -360,29 +370,150 @@ class Document(Serializable): class BaseRetriever(ABC): - """Base interface for retrievers.""" + """Base interface for a retriever.""" + + _new_arg_supported: bool = False + _expects_other_args: bool = False + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + # Version upgrade for old retrievers that implemented the public + # methods directly. + if cls.get_relevant_documents != BaseRetriever.get_relevant_documents: + warnings.warn( + "Retrievers must implement abstract `_get_relevant_documents` method" + " instead of `get_relevant_documents`", + DeprecationWarning, + ) + swap = cls.get_relevant_documents + cls.get_relevant_documents = ( # type: ignore[assignment] + BaseRetriever.get_relevant_documents + ) + cls._get_relevant_documents = swap # type: ignore[assignment] + if ( + hasattr(cls, "aget_relevant_documents") + and cls.aget_relevant_documents != BaseRetriever.aget_relevant_documents + ): + warnings.warn( + "Retrievers must implement abstract `_aget_relevant_documents` method" + " instead of `aget_relevant_documents`", + DeprecationWarning, + ) + aswap = cls.aget_relevant_documents + cls.aget_relevant_documents = ( # type: ignore[assignment] + BaseRetriever.aget_relevant_documents + ) + cls._aget_relevant_documents = aswap # type: ignore[assignment] + parameters = signature(cls._get_relevant_documents).parameters + cls._new_arg_supported = parameters.get("run_manager") is not None + # If a V1 retriever broke the interface and expects additional arguments + cls._expects_other_args = (not cls._new_arg_supported) and len(parameters) > 2 @abstractmethod - def get_relevant_documents(self, query: str) -> List[Document]: - """Get documents relevant for a query. - + def _get_relevant_documents( + self, query: str, *, run_manager: CallbackManagerForRetrieverRun, **kwargs: Any + ) -> List[Document]: + """Get documents relevant to a query. Args: query: string to find relevant documents for - + run_manager: The callbacks handler to use Returns: List of relevant documents """ @abstractmethod - async def aget_relevant_documents(self, query: str) -> List[Document]: - """Get documents relevant for a query. + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + """Asynchronously get documents relevant to a query. + Args: + query: string to find relevant documents for + run_manager: The callbacks handler to use + Returns: + List of relevant documents + """ + def get_relevant_documents( + self, query: str, *, callbacks: Callbacks = None, **kwargs: Any + ) -> List[Document]: + """Retrieve documents relevant to a query. Args: query: string to find relevant documents for + callbacks: Callback manager or list of callbacks + Returns: + List of relevant documents + """ + from langchain.callbacks.manager import CallbackManager + callback_manager = CallbackManager.configure( + callbacks, None, verbose=kwargs.get("verbose", False) + ) + run_manager = callback_manager.on_retriever_start( + query, + **kwargs, + ) + try: + if self._new_arg_supported: + result = self._get_relevant_documents( + query, run_manager=run_manager, **kwargs + ) + elif self._expects_other_args: + result = self._get_relevant_documents(query, **kwargs) + else: + result = self._get_relevant_documents(query) # type: ignore[call-arg] + except Exception as e: + run_manager.on_retriever_error(e) + raise e + else: + run_manager.on_retriever_end( + result, + **kwargs, + ) + return result + + async def aget_relevant_documents( + self, query: str, *, callbacks: Callbacks = None, **kwargs: Any + ) -> List[Document]: + """Asynchronously get documents relevant to a query. + Args: + query: string to find relevant documents for + callbacks: Callback manager or list of callbacks Returns: List of relevant documents """ + from langchain.callbacks.manager import AsyncCallbackManager + + callback_manager = AsyncCallbackManager.configure( + callbacks, None, verbose=kwargs.get("verbose", False) + ) + run_manager = await callback_manager.on_retriever_start( + query, + **kwargs, + ) + try: + if self._new_arg_supported: + result = await self._aget_relevant_documents( + query, run_manager=run_manager, **kwargs + ) + elif self._expects_other_args: + result = await self._aget_relevant_documents(query, **kwargs) + else: + result = await self._aget_relevant_documents( + query, # type: ignore[call-arg] + ) + except Exception as e: + await run_manager.on_retriever_error(e) + raise e + else: + await run_manager.on_retriever_end( + result, + **kwargs, + ) + return result # For backwards compatibility diff --git a/langchain/vectorstores/azuresearch.py b/langchain/vectorstores/azuresearch.py index bb3bf3cef8..e97c65ed68 100644 --- a/langchain/vectorstores/azuresearch.py +++ b/langchain/vectorstores/azuresearch.py @@ -20,6 +20,10 @@ from typing import ( import numpy as np from pydantic import BaseModel, root_validator +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever @@ -490,7 +494,12 @@ class AzureSearchVectorStoreRetriever(BaseRetriever, BaseModel): raise ValueError(f"search_type of {search_type} not allowed.") return values - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + run_manager: CallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: if self.search_type == "similarity": docs = self.vectorstore.vector_search(query, k=self.k) elif self.search_type == "hybrid": @@ -501,7 +510,12 @@ class AzureSearchVectorStoreRetriever(BaseRetriever, BaseModel): raise ValueError(f"search_type of {self.search_type} not allowed.") return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError( "AzureSearchVectorStoreRetriever does not support async" ) diff --git a/langchain/vectorstores/base.py b/langchain/vectorstores/base.py index 614eccb4ec..4bbe7c8d84 100644 --- a/langchain/vectorstores/base.py +++ b/langchain/vectorstores/base.py @@ -1,4 +1,5 @@ """Interface for vector stores.""" + from __future__ import annotations import asyncio @@ -20,6 +21,10 @@ from typing import ( from pydantic import BaseModel, Field, root_validator +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings from langchain.schema import BaseRetriever @@ -402,7 +407,13 @@ class VectorStoreRetriever(BaseRetriever, BaseModel): ) return values - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: Optional[CallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: if self.search_type == "similarity": docs = self.vectorstore.similarity_search(query, **self.search_kwargs) elif self.search_type == "similarity_score_threshold": @@ -420,7 +431,13 @@ class VectorStoreRetriever(BaseRetriever, BaseModel): raise ValueError(f"search_type of {self.search_type} not allowed.") return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: Optional[AsyncCallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: if self.search_type == "similarity": docs = await self.vectorstore.asimilarity_search( query, **self.search_kwargs diff --git a/langchain/vectorstores/redis.py b/langchain/vectorstores/redis.py index f6bb3d8555..dbed8cb969 100644 --- a/langchain/vectorstores/redis.py +++ b/langchain/vectorstores/redis.py @@ -1,4 +1,5 @@ """Wrapper around Redis vector database.""" + from __future__ import annotations import json @@ -21,6 +22,10 @@ from typing import ( import numpy as np from pydantic import BaseModel, root_validator +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings from langchain.utils import get_from_dict_or_env @@ -614,7 +619,13 @@ class RedisVectorStoreRetriever(VectorStoreRetriever, BaseModel): raise ValueError(f"search_type of {search_type} not allowed.") return values - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: Optional[CallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: if self.search_type == "similarity": docs = self.vectorstore.similarity_search(query, k=self.k) elif self.search_type == "similarity_limit": @@ -625,7 +636,13 @@ class RedisVectorStoreRetriever(VectorStoreRetriever, BaseModel): raise ValueError(f"search_type of {self.search_type} not allowed.") return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: Optional[AsyncCallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError("RedisVectorStoreRetriever does not support async") def add_documents(self, documents: List[Document], **kwargs: Any) -> List[str]: diff --git a/langchain/vectorstores/singlestoredb.py b/langchain/vectorstores/singlestoredb.py index 2e2f7a98b1..b16b6cedb7 100644 --- a/langchain/vectorstores/singlestoredb.py +++ b/langchain/vectorstores/singlestoredb.py @@ -1,21 +1,17 @@ """Wrapper around SingleStore DB.""" + from __future__ import annotations import enum import json -from typing import ( - Any, - ClassVar, - Collection, - Iterable, - List, - Optional, - Tuple, - Type, -) +from typing import Any, ClassVar, Collection, Iterable, List, Optional, Tuple, Type from sqlalchemy.pool import QueuePool +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain.docstore.document import Document from langchain.embeddings.base import Embeddings from langchain.vectorstores.base import VectorStore, VectorStoreRetriever @@ -454,14 +450,26 @@ class SingleStoreDBRetriever(VectorStoreRetriever): k: int = 4 allowed_search_types: ClassVar[Collection[str]] = ("similarity",) - def get_relevant_documents(self, query: str) -> List[Document]: + def _get_relevant_documents( + self, + query: str, + *, + run_manager: Optional[CallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: if self.search_type == "similarity": docs = self.vectorstore.similarity_search(query, k=self.k) else: raise ValueError(f"search_type of {self.search_type} not allowed.") return docs - async def aget_relevant_documents(self, query: str) -> List[Document]: + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: Optional[AsyncCallbackManagerForRetrieverRun] = None, + **kwargs: Any, + ) -> List[Document]: raise NotImplementedError( "SingleStoreDBVectorStoreRetriever does not support async" ) diff --git a/poetry.lock b/poetry.lock index ac3d0e3074..df7a7d91d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "absl-py" version = "1.4.0" description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -15,6 +16,7 @@ files = [ name = "accelerate" version = "0.20.3" description = "Accelerate" +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -43,6 +45,7 @@ testing = ["datasets", "deepspeed", "evaluate", "parameterized", "pytest", "pyte name = "aioboto3" version = "11.2.0" description = "Async boto3 wrapper" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -61,6 +64,7 @@ s3cse = ["cryptography (>=2.3.1)"] name = "aiobotocore" version = "2.5.0" description = "Async client for aws services using botocore and aiohttp" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -83,6 +87,7 @@ boto3 = ["boto3 (>=1.26.76,<1.26.77)"] name = "aiodns" version = "3.0.0" description = "Simple DNS resolver for asyncio" +category = "main" optional = true python-versions = "*" files = [ @@ -97,6 +102,7 @@ pycares = ">=4.0.0" name = "aiofiles" version = "23.1.0" description = "File support for asyncio." +category = "main" optional = true python-versions = ">=3.7,<4.0" files = [ @@ -108,6 +114,7 @@ files = [ name = "aiohttp" version = "3.8.4" description = "Async http client/server framework (asyncio)" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -216,6 +223,7 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aiohttp-retry" version = "2.8.3" description = "Simple retry client for aiohttp" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -230,6 +238,7 @@ aiohttp = "*" name = "aioitertools" version = "0.11.0" description = "itertools and builtins for AsyncIO and mixed iterables" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -244,6 +253,7 @@ typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""} name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -258,6 +268,7 @@ frozenlist = ">=1.1.0" name = "aiostream" version = "0.4.5" description = "Generator-based operators for asynchronous iteration" +category = "main" optional = true python-versions = "*" files = [ @@ -269,6 +280,7 @@ files = [ name = "alabaster" version = "0.7.13" description = "A configurable sidebar-enabled Sphinx theme" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -280,6 +292,7 @@ files = [ name = "aleph-alpha-client" version = "2.17.0" description = "python client to interact with Aleph Alpha api endpoints" +category = "main" optional = true python-versions = "*" files = [ @@ -307,6 +320,7 @@ types = ["mypy", "types-Pillow", "types-requests"] name = "altair" version = "4.2.2" description = "Altair: A declarative statistical visualization library for Python." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -329,6 +343,7 @@ dev = ["black", "docutils", "flake8", "ipython", "m2r", "mistune (<2.0.0)", "pyt name = "anthropic" version = "0.2.10" description = "Library for accessing the anthropic API" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -349,6 +364,7 @@ dev = ["black (>=22.3.0)", "pytest"] name = "anyio" version = "3.7.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -370,6 +386,7 @@ trio = ["trio (<0.22)"] name = "appdirs" version = "1.4.4" description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = true python-versions = "*" files = [ @@ -381,6 +398,7 @@ files = [ name = "appnope" version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" +category = "dev" optional = false python-versions = "*" files = [ @@ -392,6 +410,7 @@ files = [ name = "argon2-cffi" version = "21.3.0" description = "The secure Argon2 password hashing algorithm." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -411,6 +430,7 @@ tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pytest"] name = "argon2-cffi-bindings" version = "21.2.0" description = "Low-level CFFI bindings for Argon2" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -448,6 +468,7 @@ tests = ["pytest"] name = "arrow" version = "1.2.3" description = "Better dates & times for Python" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -462,6 +483,7 @@ python-dateutil = ">=2.7.0" name = "arxiv" version = "1.4.7" description = "Python wrapper for the arXiv API: http://arxiv.org/help/api/" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -476,6 +498,7 @@ feedparser = "*" name = "asgiref" version = "3.7.2" description = "ASGI specs, helper code, and adapters" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -493,6 +516,7 @@ tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] name = "asttokens" version = "2.2.1" description = "Annotate AST trees with source code positions" +category = "dev" optional = false python-versions = "*" files = [ @@ -510,6 +534,7 @@ test = ["astroid", "pytest"] name = "astunparse" version = "1.6.3" description = "An AST unparser for Python" +category = "main" optional = true python-versions = "*" files = [ @@ -525,6 +550,7 @@ wheel = ">=0.23.0,<1.0" name = "async-timeout" version = "4.0.2" description = "Timeout context manager for asyncio programs" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -536,6 +562,7 @@ files = [ name = "atlassian-python-api" version = "3.39.0" description = "Python Atlassian REST API Wrapper" +category = "main" optional = true python-versions = "*" files = [ @@ -556,6 +583,7 @@ kerberos = ["requests-kerberos"] name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -574,6 +602,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "authlib" version = "1.2.0" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." +category = "main" optional = false python-versions = "*" files = [ @@ -588,6 +617,7 @@ cryptography = ">=3.2" name = "autodoc-pydantic" version = "1.8.0" description = "Seamlessly integrate pydantic models in your Sphinx documentation." +category = "dev" optional = false python-versions = ">=3.6,<4.0.0" files = [ @@ -608,6 +638,7 @@ test = ["coverage (>=5,<6)", "pytest (>=6,<7)"] name = "awadb" version = "0.3.3" description = "The AI Native database for embedding vectors" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -632,6 +663,7 @@ test = ["pytest (>=6.0)"] name = "azure-ai-formrecognizer" version = "3.2.1" description = "Microsoft Azure Form Recognizer Client Library for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -649,6 +681,7 @@ typing-extensions = ">=4.0.1" name = "azure-ai-vision" version = "0.11.1b1" description = "Microsoft Azure AI Vision SDK for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -660,6 +693,7 @@ files = [ name = "azure-cognitiveservices-speech" version = "1.29.0" description = "Microsoft Cognitive Services Speech SDK for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -675,6 +709,7 @@ files = [ name = "azure-common" version = "1.1.28" description = "Microsoft Azure Client Library for Python (Common)" +category = "main" optional = true python-versions = "*" files = [ @@ -686,6 +721,7 @@ files = [ name = "azure-core" version = "1.27.1" description = "Microsoft Azure Core Library for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -705,6 +741,7 @@ aio = ["aiohttp (>=3.0)"] name = "azure-cosmos" version = "4.4.0" description = "Microsoft Azure Cosmos Client Library for Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -719,6 +756,7 @@ azure-core = ">=1.23.0,<2.0.0" name = "azure-identity" version = "1.13.0" description = "Microsoft Azure Identity Library for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -737,6 +775,7 @@ six = ">=1.12.0" name = "azure-search-documents" version = "11.4.0a20230509004" description = "Microsoft Azure Cognitive Search Client Library for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -758,6 +797,7 @@ reference = "azure-sdk-dev" name = "babel" version = "2.12.1" description = "Internationalization utilities" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -772,6 +812,7 @@ pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} name = "backcall" version = "0.2.0" description = "Specifications for callback functions passed in to an API" +category = "dev" optional = false python-versions = "*" files = [ @@ -783,6 +824,7 @@ files = [ name = "backoff" version = "2.2.1" description = "Function decoration for backoff and retry" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -794,6 +836,7 @@ files = [ name = "backports-zoneinfo" version = "0.2.1" description = "Backport of the standard library zoneinfo module" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -822,6 +865,7 @@ tzdata = ["tzdata"] name = "beautifulsoup4" version = "4.12.2" description = "Screen-scraping library" +category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -840,6 +884,7 @@ lxml = ["lxml"] name = "bentoml" version = "1.0.22" description = "BentoML: The Unified Model Serving Framework" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -916,6 +961,7 @@ triton = ["tritonclient[all] (>=2.29.0)"] name = "bibtexparser" version = "1.4.0" description = "Bibtex parser for python 3" +category = "main" optional = true python-versions = "*" files = [ @@ -929,6 +975,7 @@ pyparsing = ">=2.0.3" name = "black" version = "23.3.0" description = "The uncompromising code formatter." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -978,6 +1025,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "bleach" version = "6.0.0" description = "An easy safelist-based HTML-sanitizing tool." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -996,6 +1044,7 @@ css = ["tinycss2 (>=1.1.0,<1.2)"] name = "blinker" version = "1.6.2" description = "Fast, simple object-to-object and broadcast signaling" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1007,6 +1056,7 @@ files = [ name = "blis" version = "0.7.9" description = "The Blis BLAS-like linear algebra library, as a self-contained C-extension." +category = "main" optional = true python-versions = "*" files = [ @@ -1047,6 +1097,7 @@ numpy = ">=1.15.0" name = "blurhash" version = "1.1.4" description = "Pure-Python implementation of the blurhash algorithm." +category = "dev" optional = false python-versions = "*" files = [ @@ -1061,6 +1112,7 @@ test = ["Pillow", "numpy", "pytest"] name = "boto3" version = "1.26.76" description = "The AWS SDK for Python" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -1080,6 +1132,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.76" description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -1099,6 +1152,7 @@ crt = ["awscrt (==0.16.9)"] name = "brotli" version = "1.0.9" description = "Python bindings for the Brotli compression library" +category = "main" optional = true python-versions = "*" files = [ @@ -1190,6 +1244,7 @@ files = [ name = "brotlicffi" version = "1.0.9.2" description = "Python CFFI bindings to the Brotli library" +category = "main" optional = true python-versions = "*" files = [ @@ -1232,6 +1287,7 @@ cffi = ">=1.0.0" name = "build" version = "0.10.0" description = "A simple, correct Python build frontend" +category = "main" optional = true python-versions = ">= 3.7" files = [ @@ -1255,6 +1311,7 @@ virtualenv = ["virtualenv (>=20.0.35)"] name = "cachetools" version = "5.3.1" description = "Extensible memoizing collections and decorators" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1266,6 +1323,7 @@ files = [ name = "cassandra-driver" version = "3.28.0" description = "DataStax Driver for Apache Cassandra" +category = "main" optional = false python-versions = "*" files = [ @@ -1317,6 +1375,7 @@ graph = ["gremlinpython (==3.4.6)"] name = "cassio" version = "0.0.6" description = "A framework-agnostic Python library to seamlessly integrate Apache Cassandra with ML/LLM/genAI workloads." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1332,6 +1391,7 @@ numpy = ">=1.0" name = "catalogue" version = "2.0.8" description = "Super lightweight function registries for your library" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1343,6 +1403,7 @@ files = [ name = "cattrs" version = "23.1.2" description = "Composable complex class support for attrs and dataclasses." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1368,6 +1429,7 @@ ujson = ["ujson (>=5.4.0,<6.0.0)"] name = "certifi" version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1379,6 +1441,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = "*" files = [ @@ -1455,6 +1518,7 @@ pycparser = "*" name = "chardet" version = "5.1.0" description = "Universal encoding detector for Python 3" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1466,6 +1530,7 @@ files = [ name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -1550,6 +1615,7 @@ files = [ name = "chromadb" version = "0.3.26" description = "Chroma." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1580,6 +1646,7 @@ uvicorn = {version = ">=0.18.3", extras = ["standard"]} name = "circus" version = "0.18.0" description = "Circus is a program that will let you run and watch multiple processes and sockets." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1599,6 +1666,7 @@ test = ["coverage", "flake8 (==2.1.0)", "gevent", "mock", "nose2", "pyyaml", "to name = "clarifai" version = "9.1.0" description = "Clarifai Python Utilities" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1613,6 +1681,7 @@ clarifai-grpc = ">=9.1.0" name = "clarifai-grpc" version = "9.1.1" description = "Clarifai gRPC API Client" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1630,6 +1699,7 @@ requests = ">=2.25.1" name = "click" version = "8.1.3" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1644,6 +1714,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "click-option-group" version = "0.5.6" description = "Option groups missing in Click" +category = "main" optional = true python-versions = ">=3.6,<4" files = [ @@ -1663,6 +1734,7 @@ tests-cov = ["coverage", "coveralls", "pytest", "pytest-cov"] name = "clickhouse-connect" version = "0.5.25" description = "ClickHouse core driver, SqlAlchemy, and Superset libraries" +category = "main" optional = false python-versions = "~=3.7" files = [ @@ -1752,6 +1824,7 @@ superset = ["apache-superset (>=1.4.1)"] name = "cloudpickle" version = "2.2.1" description = "Extended pickling support for Python objects" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1763,6 +1836,7 @@ files = [ name = "cohere" version = "3.10.0" description = "A Python library for the Cohere API" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1777,6 +1851,7 @@ urllib3 = ">=1.26,<2.0" name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -1788,6 +1863,7 @@ files = [ name = "colored" version = "1.4.4" description = "Simple library for color and formatting to terminal" +category = "dev" optional = false python-versions = "*" files = [ @@ -1798,6 +1874,7 @@ files = [ name = "coloredlogs" version = "15.0.1" description = "Colored terminal output for Python's logging module" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1815,6 +1892,7 @@ cron = ["capturer (>=2.4)"] name = "comm" version = "0.1.3" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1834,6 +1912,7 @@ typing = ["mypy (>=0.990)"] name = "confection" version = "0.0.4" description = "The sweetest config system for Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1849,6 +1928,7 @@ srsly = ">=2.4.0,<3.0.0" name = "contextlib2" version = "21.6.0" description = "Backports and enhancements for the contextlib module" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1860,6 +1940,7 @@ files = [ name = "coverage" version = "7.2.7" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1935,6 +2016,7 @@ toml = ["tomli"] name = "cryptography" version = "41.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1976,6 +2058,7 @@ test-randomorder = ["pytest-randomly"] name = "cymem" version = "2.0.7" description = "Manage calls to calloc/free through Cython" +category = "main" optional = true python-versions = "*" files = [ @@ -2013,6 +2096,7 @@ files = [ name = "dataclasses-json" version = "0.5.8" description = "Easily serialize dataclasses to and from JSON" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2032,6 +2116,7 @@ dev = ["flake8", "hypothesis", "ipython", "mypy (>=0.710)", "portray", "pytest ( name = "datasets" version = "2.13.0" description = "HuggingFace community-driven open-source library of datasets" +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -2074,6 +2159,7 @@ vision = ["Pillow (>=6.2.1)"] name = "debugpy" version = "1.6.7" description = "An implementation of the Debug Adapter Protocol for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2101,6 +2187,7 @@ files = [ name = "decorator" version = "5.1.1" description = "Decorators for Humans" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2112,6 +2199,7 @@ files = [ name = "deeplake" version = "3.6.4" description = "Activeloop Deep Lake" +category = "main" optional = false python-versions = "*" files = [ @@ -2149,6 +2237,7 @@ visualizer = ["IPython", "flask"] name = "deepmerge" version = "1.1.0" description = "a toolset to deeply merge python dictionaries." +category = "main" optional = true python-versions = "*" files = [ @@ -2160,6 +2249,7 @@ files = [ name = "defusedxml" version = "0.7.1" description = "XML bomb protection for Python stdlib modules" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2171,6 +2261,7 @@ files = [ name = "deprecated" version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2188,6 +2279,7 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] name = "dill" version = "0.3.6" description = "serialize all of python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2202,6 +2294,7 @@ graph = ["objgraph (>=1.7.2)"] name = "dnspython" version = "2.3.0" description = "DNS toolkit" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -2222,6 +2315,7 @@ wmi = ["wmi (>=1.5.1,<2.0.0)"] name = "docarray" version = "0.32.1" description = "The data structure for multimodal data" +category = "main" optional = true python-versions = ">=3.7,<4.0" files = [ @@ -2260,6 +2354,7 @@ web = ["fastapi (>=0.87.0)"] name = "docker" version = "6.1.3" description = "A Python library for the Docker Engine API." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2281,6 +2376,7 @@ ssh = ["paramiko (>=2.4.3)"] name = "docutils" version = "0.17.1" description = "Docutils -- Python Documentation Utilities" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2292,6 +2388,7 @@ files = [ name = "duckdb" version = "0.8.1" description = "DuckDB embedded database" +category = "dev" optional = false python-versions = "*" files = [ @@ -2353,6 +2450,7 @@ files = [ name = "duckdb-engine" version = "0.7.3" description = "SQLAlchemy driver for duckdb" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2369,6 +2467,7 @@ sqlalchemy = ">=1.3.22" name = "duckduckgo-search" version = "3.8.3" description = "Search for words, documents, images, news, maps and text translation using the DuckDuckGo.com search engine." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2386,6 +2485,7 @@ lxml = ">=4.9.2" name = "ecdsa" version = "0.18.0" description = "ECDSA cryptographic signature library (pure python)" +category = "main" optional = true python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2404,6 +2504,7 @@ gmpy2 = ["gmpy2"] name = "elastic-transport" version = "8.4.0" description = "Transport classes and utilities shared among Python Elastic client libraries" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2422,6 +2523,7 @@ develop = ["aiohttp", "mock", "pytest", "pytest-asyncio", "pytest-cov", "pytest- name = "elasticsearch" version = "8.8.0" description = "Python client for Elasticsearch" +category = "main" optional = false python-versions = ">=3.6, <4" files = [ @@ -2441,6 +2543,7 @@ requests = ["requests (>=2.4.0,<3.0.0)"] name = "entrypoints" version = "0.4" description = "Discover and load entry points from installed packages." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2452,6 +2555,7 @@ files = [ name = "esprima" version = "4.0.1" description = "ECMAScript parsing infrastructure for multipurpose analysis in Python" +category = "main" optional = true python-versions = "*" files = [ @@ -2462,6 +2566,7 @@ files = [ name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2476,6 +2581,7 @@ test = ["pytest (>=6)"] name = "executing" version = "1.2.0" description = "Get the currently executing AST node of a frame, and other information" +category = "dev" optional = false python-versions = "*" files = [ @@ -2490,6 +2596,7 @@ tests = ["asttokens", "littleutils", "pytest", "rich"] name = "faiss-cpu" version = "1.7.4" description = "A library for efficient similarity search and clustering of dense vectors." +category = "main" optional = true python-versions = "*" files = [ @@ -2524,6 +2631,7 @@ files = [ name = "fastapi" version = "0.95.2" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2545,6 +2653,7 @@ test = ["anyio[trio] (>=3.2.1,<4.0.0)", "black (==23.1.0)", "coverage[toml] (>=6 name = "fastjsonschema" version = "2.17.1" description = "Fastest Python implementation of JSON schema" +category = "dev" optional = false python-versions = "*" files = [ @@ -2559,6 +2668,7 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc name = "feedparser" version = "6.0.10" description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2573,6 +2683,7 @@ sgmllib3k = "*" name = "filelock" version = "3.12.2" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2588,6 +2699,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "filetype" version = "1.2.0" description = "Infer file type and MIME type of any file/buffer. No external dependencies." +category = "main" optional = true python-versions = "*" files = [ @@ -2599,6 +2711,7 @@ files = [ name = "flatbuffers" version = "23.5.26" description = "The FlatBuffers serialization format for Python" +category = "main" optional = false python-versions = "*" files = [ @@ -2610,6 +2723,7 @@ files = [ name = "fluent-logger" version = "0.10.0" description = "A Python logging handler for Fluentd event collector" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2624,6 +2738,7 @@ msgpack = ">1.0" name = "fqdn" version = "1.5.1" description = "Validates fully-qualified domain names against RFC 1123, so that they are acceptable to modern bowsers" +category = "dev" optional = false python-versions = ">=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4, <4" files = [ @@ -2635,6 +2750,7 @@ files = [ name = "freezegun" version = "1.2.2" description = "Let your Python tests travel through time" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2649,6 +2765,7 @@ python-dateutil = ">=2.7" name = "frozenlist" version = "1.3.3" description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2732,6 +2849,7 @@ files = [ name = "fs" version = "2.4.16" description = "Python's filesystem abstraction layer" +category = "main" optional = true python-versions = "*" files = [ @@ -2751,6 +2869,7 @@ scandir = ["scandir (>=1.5,<2.0)"] name = "fsspec" version = "2023.6.0" description = "File-system specification" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2790,6 +2909,7 @@ tqdm = ["tqdm"] name = "future" version = "0.18.3" description = "Clean single-source support for Python 3 and 2" +category = "main" optional = true python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2800,6 +2920,7 @@ files = [ name = "gast" version = "0.4.0" description = "Python AST that abstracts the underlying Python version" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -2811,6 +2932,7 @@ files = [ name = "geojson" version = "2.5.0" description = "Python bindings and utilities for GeoJSON" +category = "main" optional = true python-versions = "*" files = [ @@ -2822,6 +2944,7 @@ files = [ name = "geomet" version = "0.2.1.post1" description = "GeoJSON <-> WKT/WKB conversion utilities" +category = "main" optional = false python-versions = ">2.6, !=3.3.*, <4" files = [ @@ -2837,6 +2960,7 @@ six = "*" name = "gitdb" version = "4.0.10" description = "Git Object Database" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2851,6 +2975,7 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2865,6 +2990,7 @@ gitdb = ">=4.0.1,<5" name = "google-api-core" version = "2.11.1" description = "Google API client core library" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2887,6 +3013,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] name = "google-api-python-client" version = "2.70.0" description = "Google API Client Library for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -2895,7 +3022,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" google-auth = ">=1.19.0,<3.0.0dev" google-auth-httplib2 = ">=0.1.0" httplib2 = ">=0.15.0,<1dev" @@ -2905,6 +3032,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.20.0" description = "Google Authentication Library" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2930,6 +3058,7 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] name = "google-auth-httplib2" version = "0.1.0" description = "Google Authentication Library: httplib2 transport" +category = "main" optional = true python-versions = "*" files = [ @@ -2946,6 +3075,7 @@ six = "*" name = "google-auth-oauthlib" version = "0.4.6" description = "Google Authentication Library" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -2964,6 +3094,7 @@ tool = ["click (>=6.0.0)"] name = "google-pasta" version = "0.2.0" description = "pasta is an AST-based Python refactoring library" +category = "main" optional = true python-versions = "*" files = [ @@ -2979,6 +3110,7 @@ six = "*" name = "google-search-results" version = "2.4.2" description = "Scrape and search localized results from Google, Bing, Baidu, Yahoo, Yandex, Ebay, Homedepot, youtube at scale using SerpApi.com" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2992,6 +3124,7 @@ requests = "*" name = "googleapis-common-protos" version = "1.59.1" description = "Common protobufs used in Google APIs" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3009,6 +3142,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "gptcache" version = "0.1.32" description = "GPTCache, a powerful caching library that can be used to speed up and lower the cost of chat applications that rely on the LLM service. GPTCache works as a memcache for AIGC applications, similar to how Redis works for traditional applications." +category = "main" optional = false python-versions = ">=3.8.1" files = [ @@ -3025,6 +3159,7 @@ requests = "*" name = "gql" version = "3.4.1" description = "GraphQL client for Python" +category = "main" optional = true python-versions = "*" files = [ @@ -3051,6 +3186,7 @@ websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] name = "graphlib-backport" version = "1.0.3" description = "Backport of the Python 3.9 graphlib module for Python 3.6+" +category = "dev" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -3062,6 +3198,7 @@ files = [ name = "graphql-core" version = "3.2.3" description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +category = "main" optional = true python-versions = ">=3.6,<4" files = [ @@ -3073,6 +3210,7 @@ files = [ name = "greenlet" version = "2.0.2" description = "Lightweight in-process concurrent programming" +category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ @@ -3146,6 +3284,7 @@ test = ["objgraph", "psutil"] name = "grpcio" version = "1.47.5" description = "HTTP/2-based RPC framework" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3207,6 +3346,7 @@ protobuf = ["grpcio-tools (>=1.47.5)"] name = "grpcio-health-checking" version = "1.47.5" description = "Standard Health Checking Service for gRPC" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3222,6 +3362,7 @@ protobuf = ">=3.12.0" name = "grpcio-reflection" version = "1.47.5" description = "Standard Protobuf Reflection Service for gRPC" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3237,6 +3378,7 @@ protobuf = ">=3.12.0" name = "grpcio-tools" version = "1.47.5" description = "Protobuf code generator for gRPC" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -3297,6 +3439,7 @@ setuptools = "*" name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3308,6 +3451,7 @@ files = [ name = "h2" version = "4.1.0" description = "HTTP/2 State-Machine based protocol implementation" +category = "main" optional = true python-versions = ">=3.6.1" files = [ @@ -3323,6 +3467,7 @@ hyperframe = ">=6.0,<7" name = "h5py" version = "3.8.0" description = "Read and write HDF5 files from Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3360,6 +3505,7 @@ numpy = ">=1.14.5" name = "hnswlib" version = "0.7.0" description = "hnswlib" +category = "main" optional = false python-versions = "*" files = [ @@ -3373,6 +3519,7 @@ numpy = "*" name = "hpack" version = "4.0.0" description = "Pure-Python HPACK header compression" +category = "main" optional = true python-versions = ">=3.6.1" files = [ @@ -3384,6 +3531,7 @@ files = [ name = "html2text" version = "2020.1.16" description = "Turn HTML into equivalent Markdown-structured text." +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -3395,6 +3543,7 @@ files = [ name = "httpcore" version = "0.17.2" description = "A minimal low-level HTTP client." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3406,16 +3555,17 @@ files = [ anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = "==1.*" +sniffio = ">=1.0.0,<2.0.0" [package.extras] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] [[package]] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3430,6 +3580,7 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "httptools" version = "0.5.0" description = "A collection of framework independent HTTP protocol utils." +category = "main" optional = false python-versions = ">=3.5.0" files = [ @@ -3483,6 +3634,7 @@ test = ["Cython (>=0.29.24,<0.30.0)"] name = "httpx" version = "0.24.1" description = "The next generation HTTP client." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3498,18 +3650,19 @@ h2 = {version = ">=3,<5", optional = true, markers = "extra == \"http2\""} httpcore = ">=0.15.0,<0.18.0" idna = "*" sniffio = "*" -socksio = {version = "==1.*", optional = true, markers = "extra == \"socks\""} +socksio = {version = ">=1.0.0,<2.0.0", optional = true, markers = "extra == \"socks\""} [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] [[package]] name = "huggingface-hub" version = "0.15.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -3541,6 +3694,7 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t name = "humanfriendly" version = "10.0" description = "Human friendly output for text interfaces using Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -3555,6 +3709,7 @@ pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_ve name = "humbug" version = "0.3.1" description = "Humbug: Do you build developer tools? Humbug helps you know your users." +category = "main" optional = false python-versions = "*" files = [ @@ -3574,6 +3729,7 @@ profile = ["GPUtil", "psutil", "types-psutil"] name = "hyperframe" version = "6.0.1" description = "HTTP/2 framing layer for Python" +category = "main" optional = true python-versions = ">=3.6.1" files = [ @@ -3585,6 +3741,7 @@ files = [ name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -3596,6 +3753,7 @@ files = [ name = "imagesize" version = "1.4.1" description = "Getting image size from png/jpeg/jpeg2000/gif file" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -3607,6 +3765,7 @@ files = [ name = "importlib-metadata" version = "6.0.1" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3626,6 +3785,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3644,6 +3804,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "inflection" version = "0.5.1" description = "A port of Ruby on Rails inflector to Python" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -3655,6 +3816,7 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3666,6 +3828,7 @@ files = [ name = "ipykernel" version = "6.23.2" description = "IPython Kernel for Jupyter" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3679,7 +3842,7 @@ comm = ">=0.1.1" debugpy = ">=1.6.5" ipython = ">=7.23.1" jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" matplotlib-inline = ">=0.1" nest-asyncio = "*" packaging = "*" @@ -3699,6 +3862,7 @@ test = ["flaky", "ipyparallel", "pre-commit", "pytest (>=7.0)", "pytest-asyncio" name = "ipython" version = "8.12.2" description = "IPython: Productive Interactive Computing" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3738,6 +3902,7 @@ test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pa name = "ipython-genutils" version = "0.2.0" description = "Vestigial utilities from IPython" +category = "dev" optional = false python-versions = "*" files = [ @@ -3749,6 +3914,7 @@ files = [ name = "ipywidgets" version = "8.0.6" description = "Jupyter interactive widgets" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3770,6 +3936,7 @@ test = ["ipykernel", "jsonschema", "pytest (>=3.6.0)", "pytest-cov", "pytz"] name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" optional = true python-versions = "*" files = [ @@ -3784,6 +3951,7 @@ six = "*" name = "isoduration" version = "20.11.0" description = "Operations with ISO 8601 durations" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -3798,6 +3966,7 @@ arrow = ">=0.15.0" name = "jaraco-context" version = "4.3.0" description = "Context managers by jaraco" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -3813,6 +3982,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "jcloud" version = "0.2.12" description = "Simplify deploying and managing Jina projects on Jina Cloud" +category = "main" optional = true python-versions = "*" files = [ @@ -3835,6 +4005,7 @@ test = ["black (==22.3.0)", "jina (>=3.7.0)", "mock", "pytest", "pytest-asyncio" name = "jedi" version = "0.18.2" description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -3854,6 +4025,7 @@ testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] name = "jina" version = "3.14.1" description = "Build multimodal AI services via cloud native technologies · Neural Search · Generative AI · MLOps" +category = "main" optional = true python-versions = "*" files = [ @@ -3969,6 +4141,7 @@ websockets = ["websockets"] name = "jina-hubble-sdk" version = "0.38.0" description = "SDK for Hubble API at Jina AI." +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -3994,6 +4167,7 @@ full = ["aiohttp", "black (==22.3.0)", "docker", "filelock", "flake8 (==4.0.1)", name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4011,6 +4185,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4022,6 +4197,7 @@ files = [ name = "joblib" version = "1.2.0" description = "Lightweight pipelining with Python functions" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4033,6 +4209,7 @@ files = [ name = "jq" version = "1.4.1" description = "jq is a lightweight and flexible JSON processor." +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -4097,6 +4274,7 @@ files = [ name = "jsonlines" version = "3.1.0" description = "Library with helpers for the jsonlines file format" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -4111,6 +4289,7 @@ attrs = ">=19.2.0" name = "jsonpointer" version = "2.4" description = "Identify specific nodes in a JSON document (RFC 6901)" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ @@ -4122,6 +4301,7 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4151,6 +4331,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jupyter" version = "1.0.0" description = "Jupyter metapackage. Install all the Jupyter components in one go." +category = "dev" optional = false python-versions = "*" files = [ @@ -4171,6 +4352,7 @@ qtconsole = "*" name = "jupyter-cache" version = "0.6.1" description = "A defined interface for working with a cache of jupyter notebooks." +category = "dev" optional = false python-versions = "~=3.8" files = [ @@ -4198,6 +4380,7 @@ testing = ["coverage", "ipykernel", "jupytext", "matplotlib", "nbdime", "nbforma name = "jupyter-client" version = "8.2.0" description = "Jupyter protocol implementation and client libraries" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4207,7 +4390,7 @@ files = [ [package.dependencies] importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" tornado = ">=6.2" @@ -4221,6 +4404,7 @@ test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pyt name = "jupyter-console" version = "6.6.3" description = "Jupyter terminal console" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4232,7 +4416,7 @@ files = [ ipykernel = ">=6.14" ipython = "*" jupyter-client = ">=7.0.0" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" prompt-toolkit = ">=3.0.30" pygments = "*" pyzmq = ">=17" @@ -4245,6 +4429,7 @@ test = ["flaky", "pexpect", "pytest"] name = "jupyter-core" version = "5.3.1" description = "Jupyter core package. A base package on which Jupyter projects rely." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4265,6 +4450,7 @@ test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] name = "jupyter-events" version = "0.6.3" description = "Jupyter Event System library" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4289,6 +4475,7 @@ test = ["click", "coverage", "pre-commit", "pytest (>=7.0)", "pytest-asyncio (>= name = "jupyter-server" version = "2.6.0" description = "The backend—i.e. core services, APIs, and REST endpoints—to Jupyter web applications." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4301,7 +4488,7 @@ anyio = ">=3.1.0" argon2-cffi = "*" jinja2 = "*" jupyter-client = ">=7.4.4" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" jupyter-events = ">=0.6.0" jupyter-server-terminals = "*" nbconvert = ">=6.4.4" @@ -4325,6 +4512,7 @@ test = ["ipykernel", "pre-commit", "pytest (>=7.0)", "pytest-console-scripts", " name = "jupyter-server-terminals" version = "0.4.4" description = "A Jupyter Server Extension Providing Terminals." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -4344,6 +4532,7 @@ test = ["coverage", "jupyter-server (>=2.0.0)", "pytest (>=7.0)", "pytest-cov", name = "jupyterlab-pygments" version = "0.2.2" description = "Pygments theme using JupyterLab CSS variables" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4355,6 +4544,7 @@ files = [ name = "jupyterlab-widgets" version = "3.0.7" description = "Jupyter interactive widgets for JupyterLab" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4366,6 +4556,7 @@ files = [ name = "keras" version = "2.11.0" description = "Deep learning for humans." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -4376,6 +4567,7 @@ files = [ name = "lancedb" version = "0.1.8" description = "lancedb" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -4398,6 +4590,7 @@ tests = ["doctest", "pytest", "pytest-mock"] name = "langchainplus-sdk" version = "0.0.17" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +category = "main" optional = false python-versions = ">=3.8.1,<4.0" files = [ @@ -4414,6 +4607,7 @@ tenacity = ">=8.1.0,<9.0.0" name = "langcodes" version = "3.3.0" description = "Tools for labeling human languages with IETF language tags" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -4428,6 +4622,7 @@ data = ["language-data (>=1.1,<2.0)"] name = "langkit" version = "0.0.1" description = "A collection of text metric udfs for whylogs profiling and monitoring in WhyLabs" +category = "main" optional = true python-versions = ">=3.8,<4.0" files = [ @@ -4447,6 +4642,7 @@ all = ["datasets (>=2.12.0,<3.0.0)", "nltk (>=3.8.1,<4.0.0)", "openai (>=0.27.6, name = "lark" version = "1.1.5" description = "a modern parsing library" +category = "main" optional = false python-versions = "*" files = [ @@ -4463,6 +4659,7 @@ regex = ["regex"] name = "libclang" version = "16.0.0" description = "Clang Python Bindings, mirrored from the official LLVM repo: https://github.com/llvm/llvm-project/tree/main/clang/bindings/python, to make the installation process easier." +category = "main" optional = true python-versions = "*" files = [ @@ -4480,6 +4677,7 @@ files = [ name = "linkchecker" version = "10.2.1" description = "check links in web documents or full websites" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4496,6 +4694,7 @@ requests = ">=2.20" name = "livereload" version = "2.6.3" description = "Python LiveReload is an awesome tool for web developers" +category = "dev" optional = false python-versions = "*" files = [ @@ -4511,6 +4710,7 @@ tornado = {version = "*", markers = "python_version > \"2.7\""} name = "loguru" version = "0.7.0" description = "Python logging made (stupidly) simple" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -4529,6 +4729,7 @@ dev = ["Sphinx (==5.3.0)", "colorama (==0.4.5)", "colorama (==0.4.6)", "freezegu name = "lxml" version = "4.9.2" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" files = [ @@ -4621,6 +4822,7 @@ source = ["Cython (>=0.29.7)"] name = "lz4" version = "4.3.2" description = "LZ4 Bindings for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4670,6 +4872,7 @@ tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] name = "manifest-ml" version = "0.0.1" description = "Manifest for Prompt Programming Foundation Models." +category = "main" optional = true python-versions = ">=3.8.0" files = [ @@ -4693,6 +4896,7 @@ dev = ["autopep8 (>=1.6.0)", "black (>=22.3.0)", "docformatter (>=1.4)", "flake8 name = "markdown" version = "3.4.3" description = "Python implementation of John Gruber's Markdown." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -4707,6 +4911,7 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4731,6 +4936,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4790,6 +4996,7 @@ files = [ name = "marshmallow" version = "3.19.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4810,6 +5017,7 @@ tests = ["pytest", "pytz", "simplejson"] name = "marshmallow-enum" version = "1.5.1" description = "Enum field for Marshmallow" +category = "main" optional = false python-versions = "*" files = [ @@ -4824,6 +5032,7 @@ marshmallow = ">=2.0.0" name = "mastodon-py" version = "1.8.1" description = "Python wrapper for the Mastodon API" +category = "dev" optional = false python-versions = "*" files = [ @@ -4849,6 +5058,7 @@ webpush = ["cryptography (>=1.6.0)", "http-ece (>=1.0.5)"] name = "matplotlib-inline" version = "0.1.6" description = "Inline Matplotlib backend for Jupyter" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -4863,6 +5073,7 @@ traitlets = "*" name = "mdit-py-plugins" version = "0.3.5" description = "Collection of plugins for markdown-it-py" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -4882,6 +5093,7 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -4893,6 +5105,7 @@ files = [ name = "mistune" version = "2.0.5" description = "A sane Markdown parser with useful plugins and renderers" +category = "dev" optional = false python-versions = "*" files = [ @@ -4904,6 +5117,7 @@ files = [ name = "mmh3" version = "3.1.0" description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" optional = false python-versions = "*" files = [ @@ -4948,6 +5162,7 @@ files = [ name = "momento" version = "1.6.0" description = "SDK for Momento" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -4964,6 +5179,7 @@ pyjwt = ">=2.4.0,<3.0.0" name = "momento-wire-types" version = "0.64.1" description = "Momento Client Proto Generated Files" +category = "main" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -4979,6 +5195,7 @@ protobuf = ">=3,<5" name = "monotonic" version = "1.6" description = "An implementation of time.monotonic() for Python 2 & < 3.3" +category = "dev" optional = false python-versions = "*" files = [ @@ -4990,6 +5207,7 @@ files = [ name = "more-itertools" version = "9.1.0" description = "More routines for operating on iterables, beyond itertools" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -5001,6 +5219,7 @@ files = [ name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" +category = "main" optional = false python-versions = "*" files = [ @@ -5018,6 +5237,7 @@ tests = ["pytest (>=4.6)"] name = "msal" version = "1.22.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" optional = true python-versions = "*" files = [ @@ -5037,6 +5257,7 @@ broker = ["pymsalruntime (>=0.13.2,<0.14)"] name = "msal-extensions" version = "1.0.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" optional = true python-versions = "*" files = [ @@ -5055,6 +5276,7 @@ portalocker = [ name = "msgpack" version = "1.0.5" description = "MessagePack serializer" +category = "main" optional = true python-versions = "*" files = [ @@ -5127,6 +5349,7 @@ files = [ name = "msrest" version = "0.7.1" description = "AutoRest swagger generator Python client runtime." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5148,6 +5371,7 @@ async = ["aiodns", "aiohttp (>=3.0)"] name = "multidict" version = "6.0.4" description = "multidict implementation" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5231,6 +5455,7 @@ files = [ name = "multiprocess" version = "0.70.14" description = "better multiprocessing and multithreading in python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5257,6 +5482,7 @@ dill = ">=0.3.6" name = "murmurhash" version = "1.0.9" description = "Cython bindings for MurmurHash" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5294,6 +5520,7 @@ files = [ name = "mypy" version = "0.991" description = "Optional static typing for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5344,6 +5571,7 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -5355,6 +5583,7 @@ files = [ name = "mypy-protobuf" version = "3.3.0" description = "Generate mypy stub files from protobuf specs" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5370,6 +5599,7 @@ types-protobuf = ">=3.19.12" name = "myst-nb" version = "0.17.2" description = "A Jupyter Notebook Sphinx reader built on top of the MyST markdown parser." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5398,6 +5628,7 @@ testing = ["beautifulsoup4", "coverage (>=6.4,<8.0)", "ipykernel (>=5.5,<6.0)", name = "myst-parser" version = "0.18.1" description = "An extended commonmark compliant parser, with bridges to docutils & sphinx." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5424,6 +5655,7 @@ testing = ["beautifulsoup4", "coverage[toml]", "pytest (>=6,<7)", "pytest-cov", name = "nbclassic" version = "1.0.0" description = "Jupyter Notebook as a Jupyter Server extension." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5459,6 +5691,7 @@ test = ["coverage", "nbval", "pytest", "pytest-cov", "pytest-jupyter", "pytest-p name = "nbclient" version = "0.7.4" description = "A client library for executing notebooks. Formerly nbconvert's ExecutePreprocessor." +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -5468,7 +5701,7 @@ files = [ [package.dependencies] jupyter-client = ">=6.1.12" -jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" +jupyter-core = ">=4.12,<5.0.0 || >=5.1.0" nbformat = ">=5.1" traitlets = ">=5.3" @@ -5481,6 +5714,7 @@ test = ["flaky", "ipykernel", "ipython", "ipywidgets", "nbconvert (>=7.0.0)", "p name = "nbconvert" version = "7.5.0" description = "Converting Jupyter Notebooks" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5519,6 +5753,7 @@ webpdf = ["pyppeteer (>=1,<1.1)"] name = "nbformat" version = "5.9.0" description = "The Jupyter Notebook format" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -5540,6 +5775,7 @@ test = ["pep440", "pre-commit", "pytest", "testpath"] name = "nbsphinx" version = "0.8.12" description = "Jupyter Notebook Tools for Sphinx" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -5559,6 +5795,7 @@ traitlets = ">=5" name = "nebula3-python" version = "3.4.0" description = "Python client for NebulaGraph V3.4" +category = "main" optional = true python-versions = "*" files = [ @@ -5576,6 +5813,7 @@ six = ">=1.16.0" name = "neo4j" version = "5.9.0" description = "Neo4j Bolt driver for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -5593,6 +5831,7 @@ pandas = ["numpy (>=1.7.0,<2.0.0)", "pandas (>=1.1.0,<3.0.0)"] name = "nest-asyncio" version = "1.5.6" description = "Patch asyncio to allow nested event loops" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -5604,6 +5843,7 @@ files = [ name = "networkx" version = "2.8.8" description = "Python package for creating and manipulating graphs and networks" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -5622,6 +5862,7 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] name = "nlpcloud" version = "1.0.42" description = "Python client for the NLP Cloud API" +category = "main" optional = true python-versions = "*" files = [ @@ -5636,6 +5877,7 @@ requests = "*" name = "nltk" version = "3.8.1" description = "Natural Language Toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5661,6 +5903,7 @@ twitter = ["twython"] name = "nomic" version = "1.1.14" description = "The offical Nomic python client." +category = "main" optional = true python-versions = "*" files = [ @@ -5688,6 +5931,7 @@ gpt4all = ["peft (==0.3.0.dev0)", "sentencepiece", "torch", "transformers (==4.2 name = "notebook" version = "6.5.4" description = "A web-based notebook environment for interactive computing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5722,6 +5966,7 @@ test = ["coverage", "nbval", "pytest", "pytest-cov", "requests", "requests-unixs name = "notebook-shim" version = "0.2.3" description = "A shim layer for notebook traits and config" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -5739,6 +5984,7 @@ test = ["pytest", "pytest-console-scripts", "pytest-jupyter", "pytest-tornasync" name = "numcodecs" version = "0.11.0" description = "A Python package providing buffer compression and transformation codecs for use" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5771,6 +6017,7 @@ zfpy = ["zfpy (>=1.0.0)"] name = "numexpr" version = "2.8.4" description = "Fast numerical expression evaluator for NumPy" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -5813,6 +6060,7 @@ numpy = ">=1.13.3" name = "numpy" version = "1.24.3" description = "Fundamental package for array computing in Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -5850,6 +6098,7 @@ files = [ name = "nvidia-cublas-cu11" version = "11.10.3.66" description = "CUBLAS native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -5865,6 +6114,7 @@ wheel = "*" name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" description = "NVRTC native runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -5881,6 +6131,7 @@ wheel = "*" name = "nvidia-cuda-runtime-cu11" version = "11.7.99" description = "CUDA Runtime native Libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -5896,6 +6147,7 @@ wheel = "*" name = "nvidia-cudnn-cu11" version = "8.5.0.96" description = "cuDNN runtime libraries" +category = "main" optional = false python-versions = ">=3" files = [ @@ -5911,6 +6163,7 @@ wheel = "*" name = "o365" version = "2.0.27" description = "Microsoft Graph and Office 365 API made easy" +category = "main" optional = true python-versions = ">=3.4" files = [ @@ -5931,6 +6184,7 @@ tzlocal = ">=4.0,<5.0" name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -5947,6 +6201,7 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] name = "octoai-sdk" version = "0.1.1" description = "A runtime library for OctoAI." +category = "main" optional = true python-versions = ">=3.8.1,<4.0.0" files = [ @@ -5970,6 +6225,7 @@ uvicorn = ">=0.22.0,<0.23.0" name = "onnxruntime" version = "1.15.1" description = "ONNX Runtime is a runtime accelerator for Machine Learning models" +category = "dev" optional = false python-versions = "*" files = [ @@ -6011,6 +6267,7 @@ sympy = "*" name = "openai" version = "0.27.8" description = "Python client library for the OpenAI API" +category = "main" optional = false python-versions = ">=3.7.1" files = [ @@ -6025,7 +6282,7 @@ tqdm = "*" [package.extras] datalib = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] -dev = ["black (>=21.6b0,<22.0)", "pytest (==6.*)", "pytest-asyncio", "pytest-mock"] +dev = ["black (>=21.6b0,<22.0)", "pytest (>=6.0.0,<7.0.0)", "pytest-asyncio", "pytest-mock"] embeddings = ["matplotlib", "numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "plotly", "scikit-learn (>=1.0.2)", "scipy", "tenacity (>=8.0.1)"] wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)", "wandb"] @@ -6033,6 +6290,7 @@ wandb = ["numpy", "openpyxl (>=3.0.7)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1 name = "openapi-schema-pydantic" version = "1.2.4" description = "OpenAPI (v3) specification schema as pydantic class" +category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -6047,6 +6305,7 @@ pydantic = ">=1.8.2" name = "openllm" version = "0.1.19" description = "OpenLLM: REST/gRPC API server for running any open Large-Language Model - StableLM, Llama, Alpaca, Dolly, Flan-T5, Custom" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -6081,6 +6340,7 @@ starcoder = ["bitsandbytes"] name = "openlm" version = "0.0.5" description = "Drop-in OpenAI-compatible that can call LLMs from other providers" +category = "main" optional = true python-versions = ">=3.8.1,<4.0" files = [ @@ -6095,6 +6355,7 @@ requests = ">=2,<3" name = "opensearch-py" version = "2.2.0" description = "Python client for OpenSearch" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" files = [ @@ -6119,6 +6380,7 @@ kerberos = ["requests-kerberos"] name = "opentelemetry-api" version = "1.17.0" description = "OpenTelemetry Python API" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6135,6 +6397,7 @@ setuptools = ">=16.0" name = "opentelemetry-exporter-otlp" version = "1.17.0" description = "OpenTelemetry Collector Exporters" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6150,6 +6413,7 @@ opentelemetry-exporter-otlp-proto-http = "1.17.0" name = "opentelemetry-exporter-otlp-proto-grpc" version = "1.17.0" description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6172,6 +6436,7 @@ test = ["pytest-grpc"] name = "opentelemetry-exporter-otlp-proto-http" version = "1.17.0" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6194,6 +6459,7 @@ test = ["responses (==0.22.0)"] name = "opentelemetry-exporter-prometheus" version = "1.12.0rc1" description = "Prometheus Metric Exporter for OpenTelemetry" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6210,6 +6476,7 @@ prometheus-client = ">=0.5.0,<1.0.0" name = "opentelemetry-instrumentation" version = "0.38b0" description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6226,6 +6493,7 @@ wrapt = ">=1.0.0,<2.0.0" name = "opentelemetry-instrumentation-aiohttp-client" version = "0.38b0" description = "OpenTelemetry aiohttp client instrumentation" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6248,6 +6516,7 @@ test = ["opentelemetry-instrumentation-aiohttp-client[instruments]"] name = "opentelemetry-instrumentation-asgi" version = "0.38b0" description = "ASGI instrumentation for OpenTelemetry" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6270,6 +6539,7 @@ test = ["opentelemetry-instrumentation-asgi[instruments]", "opentelemetry-test-u name = "opentelemetry-instrumentation-fastapi" version = "0.38b0" description = "OpenTelemetry FastAPI Instrumentation" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6292,6 +6562,7 @@ test = ["httpx (>=0.22,<1.0)", "opentelemetry-instrumentation-fastapi[instrument name = "opentelemetry-instrumentation-grpc" version = "0.38b0" description = "OpenTelemetry gRPC instrumentation" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6314,6 +6585,7 @@ test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (> name = "opentelemetry-proto" version = "1.17.0" description = "OpenTelemetry Python Proto" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6328,6 +6600,7 @@ protobuf = ">=3.19,<5.0" name = "opentelemetry-sdk" version = "1.17.0" description = "OpenTelemetry Python SDK" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6345,6 +6618,7 @@ typing-extensions = ">=3.7.4" name = "opentelemetry-semantic-conventions" version = "0.38b0" description = "OpenTelemetry Semantic Conventions" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6356,6 +6630,7 @@ files = [ name = "opentelemetry-util-http" version = "0.38b0" description = "Web util for OpenTelemetry" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6367,6 +6642,7 @@ files = [ name = "opt-einsum" version = "3.3.0" description = "Optimizing numpys einsum function" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -6385,6 +6661,7 @@ tests = ["pytest", "pytest-cov", "pytest-pep8"] name = "optimum" version = "1.8.8" description = "Optimum Library is an extension of the Hugging Face Transformers library, providing a framework to integrate third-party libraries from Hardware Partners and interface with their specific functionality." +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -6425,6 +6702,7 @@ tests = ["Pillow", "diffusers (>=0.17.0)", "parameterized", "pytest", "pytest-xd name = "orjson" version = "3.9.1" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6480,6 +6758,7 @@ files = [ name = "overrides" version = "7.3.1" description = "A decorator to automatically detect mismatch when overriding a method." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -6491,6 +6770,7 @@ files = [ name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6502,6 +6782,7 @@ files = [ name = "pandas" version = "2.0.2" description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6569,6 +6850,7 @@ xml = ["lxml (>=4.6.3)"] name = "pandocfilters" version = "1.5.0" description = "Utilities for writing pandoc filters in python" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -6580,6 +6862,7 @@ files = [ name = "parso" version = "0.8.3" description = "A Python Parser" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -6595,6 +6878,7 @@ testing = ["docopt", "pytest (<6.0.0)"] name = "pathos" version = "0.3.0" description = "parallel graph management and execution in heterogeneous computing" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6612,6 +6896,7 @@ ppft = ">=1.7.6.6" name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6623,6 +6908,7 @@ files = [ name = "pathy" version = "0.10.1" description = "pathlib.Path subclasses for local and cloud bucket storage" +category = "main" optional = true python-versions = ">= 3.6" files = [ @@ -6645,6 +6931,7 @@ test = ["mock", "pytest", "pytest-coverage", "typer-cli"] name = "pdfminer-six" version = "20221105" description = "PDF parser and analyzer" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6665,6 +6952,7 @@ image = ["Pillow"] name = "pexpect" version = "4.8.0" description = "Pexpect allows easy control of interactive console applications." +category = "main" optional = false python-versions = "*" files = [ @@ -6679,6 +6967,7 @@ ptyprocess = ">=0.5" name = "pgvector" version = "0.1.8" description = "pgvector support for Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -6692,6 +6981,7 @@ numpy = "*" name = "pickleshare" version = "0.7.5" description = "Tiny 'shelve'-like database with concurrency support" +category = "dev" optional = false python-versions = "*" files = [ @@ -6703,6 +6993,7 @@ files = [ name = "pillow" version = "9.5.0" description = "Python Imaging Library (Fork)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6782,6 +7073,7 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pinecone-client" version = "2.2.2" description = "Pinecone client and SDK" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -6807,6 +7099,7 @@ grpc = ["googleapis-common-protos (>=1.53.0)", "grpc-gateway-protoc-gen-openapiv name = "pinecone-text" version = "0.4.2" description = "Text utilities library by Pinecone.io" +category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ @@ -6826,6 +7119,7 @@ wget = ">=3.2,<4.0" name = "pip" version = "23.1.2" description = "The PyPA recommended tool for installing Python packages." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6837,6 +7131,7 @@ files = [ name = "pip-requirements-parser" version = "32.0.1" description = "pip requirements parser - a mostly correct pip requirements parsing library because it uses pip's own code." +category = "main" optional = true python-versions = ">=3.6.0" files = [ @@ -6856,6 +7151,7 @@ testing = ["aboutcode-toolkit (>=6.0.0)", "black", "pytest (>=6,!=7.0.0)", "pyte name = "pip-tools" version = "6.13.0" description = "pip-tools keeps your pinned dependencies fresh." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -6878,6 +7174,7 @@ testing = ["flit-core (>=2,<4)", "poetry-core (>=1.0.0)", "pytest (>=7.2.0)", "p name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -6889,6 +7186,7 @@ files = [ name = "platformdirs" version = "3.6.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6904,6 +7202,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest- name = "playwright" version = "1.35.0" description = "A high-level API to automate web browsers" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -6925,6 +7224,7 @@ typing-extensions = {version = "*", markers = "python_version <= \"3.8\""} name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -6940,6 +7240,7 @@ testing = ["pytest", "pytest-benchmark"] name = "portalocker" version = "2.7.0" description = "Wraps the portalocker recipe for easy usage" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -6959,6 +7260,7 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p name = "posthog" version = "3.0.1" description = "Integrate PostHog into any python application." +category = "dev" optional = false python-versions = "*" files = [ @@ -6982,6 +7284,7 @@ test = ["coverage", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint" name = "pox" version = "0.3.2" description = "utilities for filesystem exploration and automated builds" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -6993,6 +7296,7 @@ files = [ name = "ppft" version = "1.7.6.6" description = "distributed and parallel python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7007,6 +7311,7 @@ dill = ["dill (>=0.3.6)"] name = "preshed" version = "3.0.8" description = "Cython hash table that trusts the keys are pre-hashed" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -7048,6 +7353,7 @@ murmurhash = ">=0.28.0,<1.1.0" name = "prometheus-client" version = "0.17.0" description = "Python client for the Prometheus monitoring system." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -7062,6 +7368,7 @@ twisted = ["twisted"] name = "prompt-toolkit" version = "3.0.38" description = "Library for building powerful interactive command lines in Python" +category = "dev" optional = false python-versions = ">=3.7.0" files = [ @@ -7076,6 +7383,7 @@ wcwidth = "*" name = "promptlayer" version = "0.1.89" description = "PromptLayer is a package to keep track of your GPT models training" +category = "dev" optional = false python-versions = "*" files = [ @@ -7090,6 +7398,7 @@ requests = "*" name = "protobuf" version = "3.19.6" description = "Protocol Buffers" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -7124,6 +7433,7 @@ files = [ name = "psutil" version = "5.9.5" description = "Cross-platform lib for process and system monitoring in Python." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -7150,6 +7460,7 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] name = "psychicapi" version = "0.8.0" description = "Psychic.dev is an open-source data integration platform for LLMs. This is the Python client for Psychic" +category = "main" optional = true python-versions = "*" files = [ @@ -7164,6 +7475,7 @@ requests = "*" name = "psycopg2-binary" version = "2.9.6" description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -7235,6 +7547,7 @@ files = [ name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" +category = "main" optional = false python-versions = "*" files = [ @@ -7246,6 +7559,7 @@ files = [ name = "pulsar-client" version = "3.2.0" description = "Apache Pulsar Python client library" +category = "dev" optional = false python-versions = "*" files = [ @@ -7293,6 +7607,7 @@ functions = ["apache-bookkeeper-client (>=4.16.1)", "grpcio (>=1.8.2)", "prometh name = "pure-eval" version = "0.2.2" description = "Safely evaluate AST nodes without side effects" +category = "dev" optional = false python-versions = "*" files = [ @@ -7307,6 +7622,7 @@ tests = ["pytest"] name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -7318,6 +7634,7 @@ files = [ name = "py-trello" version = "0.19.0" description = "Python wrapper around the Trello API" +category = "main" optional = true python-versions = "*" files = [ @@ -7334,6 +7651,7 @@ requests-oauthlib = ">=0.4.1" name = "py4j" version = "0.10.9.7" description = "Enables Python programs to dynamically access arbitrary Java objects" +category = "main" optional = true python-versions = "*" files = [ @@ -7345,6 +7663,7 @@ files = [ name = "pyaes" version = "1.6.1" description = "Pure-Python Implementation of the AES block-cipher and common modes of operation" +category = "main" optional = true python-versions = "*" files = [ @@ -7355,6 +7674,7 @@ files = [ name = "pyarrow" version = "12.0.1" description = "Python library for Apache Arrow" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7392,6 +7712,7 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -7403,6 +7724,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" +category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -7417,6 +7739,7 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycares" version = "4.3.0" description = "Python interface for c-ares" +category = "main" optional = true python-versions = "*" files = [ @@ -7484,6 +7807,7 @@ idna = ["idna (>=2.1)"] name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -7495,6 +7819,7 @@ files = [ name = "pydantic" version = "1.10.9" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7547,6 +7872,7 @@ email = ["email-validator (>=1.0.3)"] name = "pydata-sphinx-theme" version = "0.8.1" description = "Bootstrap-based Sphinx theme from the PyData community" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -7570,6 +7896,7 @@ test = ["pydata-sphinx-theme[doc]", "pytest"] name = "pydeck" version = "0.8.0" description = "Widget for deck.gl maps" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7589,6 +7916,7 @@ jupyter = ["ipykernel (>=5.1.2)", "ipython (>=5.8.0)", "ipywidgets (>=7,<8)", "t name = "pyee" version = "9.0.4" description = "A port of node.js's EventEmitter to python." +category = "dev" optional = false python-versions = "*" files = [ @@ -7603,6 +7931,7 @@ typing-extensions = "*" name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7617,6 +7946,7 @@ plugins = ["importlib-metadata"] name = "pyjwt" version = "2.7.0" description = "JSON Web Token implementation in Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7637,6 +7967,7 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pylance" version = "0.4.21" description = "python wrapper for lance-rs" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -7658,6 +7989,7 @@ tests = ["duckdb", "polars[pandas,pyarrow]", "pytest"] name = "pymongo" version = "4.3.3" description = "Python driver for MongoDB " +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7752,6 +8084,7 @@ zstd = ["zstandard"] name = "pympler" version = "1.0.1" description = "A development tool to measure, monitor and analyze the memory behavior of Python objects." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -7763,6 +8096,7 @@ files = [ name = "pymupdf" version = "1.22.3" description = "Python bindings for the PDF toolkit and renderer MuPDF" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7802,6 +8136,7 @@ files = [ name = "pynvml" version = "11.5.0" description = "Python Bindings for the NVIDIA Management Library" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -7813,6 +8148,7 @@ files = [ name = "pyowm" version = "3.3.0" description = "A Python wrapper around OpenWeatherMap web APIs" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7832,6 +8168,7 @@ requests = [ name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" optional = true python-versions = ">=3.6.8" files = [ @@ -7846,6 +8183,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pypdf" version = "3.9.1" description = "A pure-python PDF library capable of splitting, merging, cropping, and transforming PDF files" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -7867,6 +8205,7 @@ image = ["Pillow"] name = "pypdfium2" version = "4.15.0" description = "Python bindings to PDFium" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -7888,6 +8227,7 @@ files = [ name = "pyphen" version = "0.14.0" description = "Pure Python module to hyphenate text" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7903,6 +8243,7 @@ test = ["flake8", "isort", "pytest"] name = "pyproject-hooks" version = "1.0.0" description = "Wrappers to call pyproject.toml-based build backend hooks." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7917,6 +8258,7 @@ tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} name = "pyreadline3" version = "3.4.1" description = "A python implementation of GNU readline." +category = "main" optional = false python-versions = "*" files = [ @@ -7928,6 +8270,7 @@ files = [ name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -7964,6 +8307,7 @@ files = [ name = "pysocks" version = "1.7.1" description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -7976,6 +8320,7 @@ files = [ name = "pyspark" version = "3.4.0" description = "Apache Spark Python API" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -7996,6 +8341,7 @@ sql = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] name = "pytesseract" version = "0.3.10" description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -8011,6 +8357,7 @@ Pillow = ">=8.0.0" name = "pytest" version = "7.3.2" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8033,6 +8380,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.20.3" description = "Pytest support for asyncio" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8051,6 +8399,7 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8069,6 +8418,7 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-dotenv" version = "0.5.2" description = "A py.test plugin that parses environment files before running tests" +category = "dev" optional = false python-versions = "*" files = [ @@ -8084,6 +8434,7 @@ python-dotenv = ">=0.9.1" name = "pytest-mock" version = "3.11.1" description = "Thin-wrapper around the mock package for easier use with pytest" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8101,6 +8452,7 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "pytest-socket" version = "0.6.0" description = "Pytest Plugin to disable socket calls during tests" +category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ @@ -8115,6 +8467,7 @@ pytest = ">=3.6.3" name = "pytest-vcr" version = "1.0.2" description = "Plugin for managing VCR.py cassettes" +category = "dev" optional = false python-versions = "*" files = [ @@ -8130,6 +8483,7 @@ vcrpy = "*" name = "pytest-watcher" version = "0.2.6" description = "Continiously runs pytest on changes in *.py files" +category = "dev" optional = false python-versions = ">=3.7.0,<4.0.0" files = [ @@ -8144,6 +8498,7 @@ watchdog = ">=2.0.0" name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -8158,6 +8513,7 @@ six = ">=1.5" name = "python-dotenv" version = "1.0.0" description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -8172,6 +8528,7 @@ cli = ["click (>=5.0)"] name = "python-jose" version = "3.3.0" description = "JOSE implementation in Python" +category = "main" optional = true python-versions = "*" files = [ @@ -8193,6 +8550,7 @@ pycryptodome = ["pyasn1", "pycryptodome (>=3.3.1,<4.0.0)"] name = "python-json-logger" version = "2.0.7" description = "A python library adding a json log formatter" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -8204,6 +8562,7 @@ files = [ name = "python-magic" version = "0.4.27" description = "File type identification using libmagic" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -8215,6 +8574,7 @@ files = [ name = "python-magic-bin" version = "0.4.14" description = "File type identification using libmagic binary package" +category = "dev" optional = false python-versions = "*" files = [ @@ -8227,6 +8587,7 @@ files = [ name = "python-multipart" version = "0.0.6" description = "A streaming multipart parser for Python" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -8241,6 +8602,7 @@ dev = ["atomicwrites (==1.2.1)", "attrs (==19.2.0)", "coverage (==6.5.0)", "hatc name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -8252,6 +8614,7 @@ files = [ name = "pytz-deprecation-shim" version = "0.1.0.post0" description = "Shims to make deprecation of pytz easier" +category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -8267,6 +8630,7 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} name = "pyvespa" version = "0.33.0" description = "Python API for vespa.ai" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -8291,6 +8655,7 @@ ml = ["keras-tuner", "tensorflow", "tensorflow-ranking", "torch (<1.13)", "trans name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "main" optional = false python-versions = "*" files = [ @@ -8314,6 +8679,7 @@ files = [ name = "pywinpty" version = "2.0.10" description = "Pseudo terminal support for Windows from Python." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8329,6 +8695,7 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -8378,6 +8745,7 @@ files = [ name = "pyzmq" version = "25.1.0" description = "Python bindings for 0MQ" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -8467,6 +8835,7 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "qdrant-client" version = "1.1.7" description = "Client library for the Qdrant vector search engine" +category = "main" optional = true python-versions = ">=3.7,<3.12" files = [ @@ -8488,6 +8857,7 @@ urllib3 = ">=1.26.14,<2.0.0" name = "qtconsole" version = "5.4.3" description = "Jupyter Qt console" +category = "dev" optional = false python-versions = ">= 3.7" files = [ @@ -8514,6 +8884,7 @@ test = ["flaky", "pytest", "pytest-qt"] name = "qtpy" version = "2.3.1" description = "Provides an abstraction layer on top of the various Qt bindings (PyQt5/6 and PySide2/6)." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8531,6 +8902,7 @@ test = ["pytest (>=6,!=7.0.0,!=7.0.1)", "pytest-cov (>=3.0.0)", "pytest-qt"] name = "ratelimiter" version = "1.2.0.post0" description = "Simple python rate limiting object" +category = "main" optional = true python-versions = "*" files = [ @@ -8545,6 +8917,7 @@ test = ["pytest (>=3.0)", "pytest-asyncio"] name = "redis" version = "4.5.5" description = "Python client for Redis database and key-value store" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -8563,6 +8936,7 @@ ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)" name = "regex" version = "2023.6.3" description = "Alternative regular expression module, to replace re." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -8660,6 +9034,7 @@ files = [ name = "requests" version = "2.28.2" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7, <4" files = [ @@ -8682,6 +9057,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-oauthlib" version = "1.3.1" description = "OAuthlib authentication support for Requests." +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -8700,6 +9076,7 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] name = "requests-toolbelt" version = "1.0.0" description = "A utility belt for advanced users of python-requests" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -8714,6 +9091,7 @@ requests = ">=2.0.1,<3.0.0" name = "responses" version = "0.22.0" description = "A utility library for mocking out the `requests` Python library." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8734,6 +9112,7 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy name = "retry" version = "0.9.2" description = "Easy to use retry decorator." +category = "main" optional = true python-versions = "*" files = [ @@ -8749,6 +9128,7 @@ py = ">=1.4.26,<2.0.0" name = "rfc3339-validator" version = "0.1.4" description = "A pure python RFC3339 validator" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -8763,6 +9143,7 @@ six = "*" name = "rfc3986-validator" version = "0.1.1" description = "Pure python rfc3986 validator" +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -8774,6 +9155,7 @@ files = [ name = "rich" version = "13.4.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" optional = true python-versions = ">=3.7.0" files = [ @@ -8793,6 +9175,7 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" +category = "main" optional = true python-versions = ">=3.6,<4" files = [ @@ -8807,6 +9190,7 @@ pyasn1 = ">=0.1.3" name = "ruff" version = "0.0.249" description = "An extremely fast Python linter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -8833,6 +9217,7 @@ files = [ name = "s3transfer" version = "0.6.1" description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -8850,6 +9235,7 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "safetensors" version = "0.3.1" description = "Fast and Safe Tensor serialization" +category = "main" optional = false python-versions = "*" files = [ @@ -8910,6 +9296,7 @@ torch = ["torch (>=1.10)"] name = "schema" version = "0.7.5" description = "Simple data validation library" +category = "main" optional = true python-versions = "*" files = [ @@ -8924,6 +9311,7 @@ contextlib2 = ">=0.5.5" name = "scikit-learn" version = "1.2.2" description = "A set of python modules for machine learning and data mining" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -8966,6 +9354,7 @@ tests = ["black (>=22.3.0)", "flake8 (>=3.8.2)", "matplotlib (>=3.1.3)", "mypy ( name = "scipy" version = "1.9.3" description = "Fundamental algorithms for scientific computing in Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -9004,6 +9393,7 @@ test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "sciki name = "semver" version = "3.0.1" description = "Python helper for Semantic Versioning (https://semver.org)" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -9015,6 +9405,7 @@ files = [ name = "send2trash" version = "1.8.2" description = "Send file to trash natively under Mac OS X, Windows and Linux" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -9031,6 +9422,7 @@ win32 = ["pywin32"] name = "sentence-transformers" version = "2.2.2" description = "Multilingual text embeddings" +category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -9053,6 +9445,7 @@ transformers = ">=4.6.0,<5.0.0" name = "sentencepiece" version = "0.1.99" description = "SentencePiece python wrapper" +category = "main" optional = false python-versions = "*" files = [ @@ -9107,6 +9500,7 @@ files = [ name = "setuptools" version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -9123,6 +9517,7 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "sgmllib3k" version = "1.0.0" description = "Py3k port of sgmllib." +category = "main" optional = false python-versions = "*" files = [ @@ -9133,6 +9528,7 @@ files = [ name = "simple-di" version = "0.1.5" description = "simple dependency injection library" +category = "main" optional = true python-versions = ">=3.6.1" files = [ @@ -9147,6 +9543,7 @@ test = ["mypy", "pytest"] name = "singlestoredb" version = "0.7.1" description = "Interface to the SingleStore database and cluster management APIs" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9179,6 +9576,7 @@ sqlalchemy = ["sqlalchemy-singlestoredb"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -9190,6 +9588,7 @@ files = [ name = "smart-open" version = "6.3.0" description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +category = "main" optional = true python-versions = ">=3.6,<4.0" files = [ @@ -9211,6 +9610,7 @@ webhdfs = ["requests"] name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9222,6 +9622,7 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -9233,6 +9634,7 @@ files = [ name = "snowballstemmer" version = "2.2.0" description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +category = "dev" optional = false python-versions = "*" files = [ @@ -9244,6 +9646,7 @@ files = [ name = "socksio" version = "1.0.0" description = "Sans-I/O implementation of SOCKS4, SOCKS4A, and SOCKS5." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9255,6 +9658,7 @@ files = [ name = "soundfile" version = "0.12.1" description = "An audio library based on libsndfile, CFFI and NumPy" +category = "main" optional = true python-versions = "*" files = [ @@ -9278,6 +9682,7 @@ numpy = ["numpy"] name = "soupsieve" version = "2.4.1" description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -9289,6 +9694,7 @@ files = [ name = "spacy" version = "3.5.3" description = "Industrial-strength Natural Language Processing (NLP) in Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9375,6 +9781,7 @@ transformers = ["spacy-transformers (>=1.1.2,<1.3.0)"] name = "spacy-legacy" version = "3.0.12" description = "Legacy registered functions for spaCy backwards compatibility" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9386,6 +9793,7 @@ files = [ name = "spacy-loggers" version = "1.0.4" description = "Logging utilities for SpaCy" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9397,6 +9805,7 @@ files = [ name = "sphinx" version = "4.5.0" description = "Python documentation generator" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -9432,6 +9841,7 @@ test = ["cython", "html5lib", "pytest", "pytest-cov", "typed-ast"] name = "sphinx-autobuild" version = "2021.3.14" description = "Rebuild Sphinx documentation on changes, with live-reload in the browser." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -9451,6 +9861,7 @@ test = ["pytest", "pytest-cov"] name = "sphinx-book-theme" version = "0.3.3" description = "A clean book theme for scientific explanations and documentation with Sphinx" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -9472,6 +9883,7 @@ test = ["beautifulsoup4 (>=4.6.1,<5)", "coverage", "myst-nb (>=0.13.2,<0.14.0)", name = "sphinx-copybutton" version = "0.5.2" description = "Add a copy button to each of your code cells." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -9490,6 +9902,7 @@ rtd = ["ipython", "myst-nb", "sphinx", "sphinx-book-theme", "sphinx-examples"] name = "sphinx-panels" version = "0.6.0" description = "A sphinx extension for creating panels in a grid layout." +category = "dev" optional = false python-versions = "*" files = [ @@ -9511,6 +9924,7 @@ themes = ["myst-parser (>=0.12.9,<0.13.0)", "pydata-sphinx-theme (>=0.4.0,<0.5.0 name = "sphinx-rtd-theme" version = "1.2.2" description = "Read the Docs theme for Sphinx" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -9530,6 +9944,7 @@ dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client", "wheel"] name = "sphinx-typlog-theme" version = "0.8.0" description = "A typlog Sphinx theme" +category = "dev" optional = false python-versions = "*" files = [ @@ -9544,6 +9959,7 @@ dev = ["livereload", "sphinx"] name = "sphinxcontrib-applehelp" version = "1.0.4" description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -9559,6 +9975,7 @@ test = ["pytest"] name = "sphinxcontrib-devhelp" version = "1.0.2" description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -9574,6 +9991,7 @@ test = ["pytest"] name = "sphinxcontrib-htmlhelp" version = "2.0.1" description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -9589,6 +10007,7 @@ test = ["html5lib", "pytest"] name = "sphinxcontrib-jquery" version = "4.1" description = "Extension to include jQuery on newer Sphinx releases" +category = "dev" optional = false python-versions = ">=2.7" files = [ @@ -9603,6 +10022,7 @@ Sphinx = ">=1.8" name = "sphinxcontrib-jsmath" version = "1.0.1" description = "A sphinx extension which renders display math in HTML via JavaScript" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -9617,6 +10037,7 @@ test = ["flake8", "mypy", "pytest"] name = "sphinxcontrib-qthelp" version = "1.0.3" description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -9632,6 +10053,7 @@ test = ["pytest"] name = "sphinxcontrib-serializinghtml" version = "1.1.5" description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -9647,6 +10069,7 @@ test = ["pytest"] name = "sqlalchemy" version = "2.0.16" description = "Database Abstraction Library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -9694,7 +10117,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} typing-extensions = ">=4.2.0" [package.extras] @@ -9725,6 +10148,7 @@ sqlcipher = ["sqlcipher3-binary"] name = "sqlitedict" version = "2.1.0" description = "Persistent dict in Python, backed up by sqlite3 and pickle, multithread-safe." +category = "main" optional = true python-versions = "*" files = [ @@ -9735,6 +10159,7 @@ files = [ name = "sqlparams" version = "5.1.0" description = "Convert between various DB API 2.0 parameter styles." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -9746,6 +10171,7 @@ files = [ name = "srsly" version = "2.4.6" description = "Modern high-performance serialization utilities for Python" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -9786,6 +10212,7 @@ catalogue = ">=2.0.3,<2.1.0" name = "stack-data" version = "0.6.2" description = "Extract data from python stack frames and tracebacks for informative displays" +category = "dev" optional = false python-versions = "*" files = [ @@ -9805,6 +10232,7 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"] name = "starlette" version = "0.27.0" description = "The little ASGI library that shines." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -9823,6 +10251,7 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyam name = "steamship" version = "2.17.10" description = "The fastest way to add language AI to your product." +category = "main" optional = true python-versions = "*" files = [ @@ -9845,6 +10274,7 @@ toml = ">=0.10.2,<0.11.0" name = "streamlit" version = "1.22.0" description = "A faster way to build and share data apps" +category = "main" optional = true python-versions = ">=3.7, !=3.9.7" files = [ @@ -9885,6 +10315,7 @@ snowflake = ["snowflake-snowpark-python"] name = "stringcase" version = "1.2.0" description = "String case converter." +category = "main" optional = true python-versions = "*" files = [ @@ -9895,6 +10326,7 @@ files = [ name = "sympy" version = "1.12" description = "Computer algebra system (CAS) in Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -9909,6 +10341,7 @@ mpmath = ">=0.19" name = "syrupy" version = "4.0.2" description = "Pytest Snapshot Test Utility" +category = "dev" optional = false python-versions = ">=3.8.1,<4" files = [ @@ -9924,6 +10357,7 @@ pytest = ">=7.0.0,<8.0.0" name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -9941,6 +10375,7 @@ widechars = ["wcwidth"] name = "tair" version = "1.3.4" description = "Python client for Tair" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -9955,6 +10390,7 @@ redis = ">=4.4.4" name = "telethon" version = "1.28.5" description = "Full-featured Telegram client library for Python 3" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -9973,6 +10409,7 @@ cryptg = ["cryptg"] name = "tenacity" version = "8.2.2" description = "Retry code until it succeeds" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -9987,6 +10424,7 @@ doc = ["reno", "sphinx", "tornado (>=4.5)"] name = "tensorboard" version = "2.11.2" description = "TensorBoard lets you watch Tensors Flow" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -10012,6 +10450,7 @@ wheel = ">=0.26" name = "tensorboard-data-server" version = "0.6.1" description = "Fast data loading for TensorBoard" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -10024,6 +10463,7 @@ files = [ name = "tensorboard-plugin-wit" version = "1.8.1" description = "What-If Tool TensorBoard plugin." +category = "main" optional = true python-versions = "*" files = [ @@ -10034,6 +10474,7 @@ files = [ name = "tensorflow" version = "2.11.1" description = "TensorFlow is an open source machine learning framework for everyone." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -10078,6 +10519,7 @@ wrapt = ">=1.11.0" name = "tensorflow-estimator" version = "2.11.0" description = "TensorFlow Estimator." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -10088,6 +10530,7 @@ files = [ name = "tensorflow-hub" version = "0.13.0" description = "TensorFlow Hub is a library to foster the publication, discovery, and consumption of reusable parts of machine learning models." +category = "main" optional = true python-versions = "*" files = [ @@ -10106,6 +10549,7 @@ make-nearest-neighbour-index = ["annoy", "apache-beam"] name = "tensorflow-io-gcs-filesystem" version = "0.32.0" description = "TensorFlow IO" +category = "main" optional = true python-versions = ">=3.7, <3.12" files = [ @@ -10136,6 +10580,7 @@ tensorflow-rocm = ["tensorflow-rocm (>=2.12.0,<2.13.0)"] name = "tensorflow-macos" version = "2.11.0" description = "TensorFlow is an open source machine learning framework for everyone." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -10173,6 +10618,7 @@ wrapt = ">=1.11.0" name = "tensorflow-text" version = "2.11.0" description = "TF.Text is a TensorFlow library of text related ops, modules, and subgraphs." +category = "main" optional = true python-versions = "*" files = [ @@ -10199,6 +10645,7 @@ tests = ["absl-py", "pytest", "tensorflow-datasets (>=3.2.0)"] name = "termcolor" version = "2.3.0" description = "ANSI color formatting for output in terminal" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -10213,6 +10660,7 @@ tests = ["pytest", "pytest-cov"] name = "terminado" version = "0.17.1" description = "Tornado websocket backend for the Xterm.js Javascript terminal emulator library." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -10233,6 +10681,7 @@ test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] name = "textstat" version = "0.7.3" description = "Calculate statistical features from text" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -10247,6 +10696,7 @@ pyphen = "*" name = "thinc" version = "8.1.10" description = "A refreshing functional take on deep learning, compatible with your favorite libraries" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -10322,6 +10772,7 @@ torch = ["torch (>=1.6.0)"] name = "threadpoolctl" version = "3.1.0" description = "threadpoolctl" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -10333,6 +10784,7 @@ files = [ name = "tigrisdb" version = "1.0.0b6" description = "Python SDK for Tigris " +category = "main" optional = true python-versions = ">=3.8,<4.0" files = [ @@ -10348,6 +10800,7 @@ protobuf = ">=3.19.6" name = "tiktoken" version = "0.3.3" description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -10393,6 +10846,7 @@ blobfile = ["blobfile (>=2)"] name = "tinycss2" version = "1.2.1" description = "A tiny CSS parser" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -10411,6 +10865,7 @@ test = ["flake8", "isort", "pytest"] name = "tokenizers" version = "0.13.3" description = "Fast and Customizable Tokenizers" +category = "main" optional = false python-versions = "*" files = [ @@ -10465,6 +10920,7 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -10476,6 +10932,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -10487,6 +10944,7 @@ files = [ name = "toolz" version = "0.12.0" description = "List processing tools and functional utilities" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -10498,6 +10956,7 @@ files = [ name = "torch" version = "1.13.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -10538,6 +10997,7 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "torchvision" version = "0.14.1" description = "image and video datasets and models for torch deep learning" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -10564,7 +11024,7 @@ files = [ [package.dependencies] numpy = "*" -pillow = ">=5.3.0,<8.3.dev0 || >=8.4.dev0" +pillow = ">=5.3.0,<8.3.0 || >=8.4.0" requests = "*" torch = "1.13.1" typing-extensions = "*" @@ -10576,6 +11036,7 @@ scipy = ["scipy"] name = "tornado" version = "6.3.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "main" optional = false python-versions = ">= 3.8" files = [ @@ -10596,6 +11057,7 @@ files = [ name = "tqdm" version = "4.65.0" description = "Fast, Extensible Progress Meter" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -10616,6 +11078,7 @@ telegram = ["requests"] name = "traitlets" version = "5.9.0" description = "Traitlets Python configuration system" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -10631,6 +11094,7 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] name = "transformers" version = "4.30.2" description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -10704,6 +11168,7 @@ vision = ["Pillow"] name = "typer" version = "0.7.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -10724,6 +11189,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "types-chardet" version = "5.0.4.6" description = "Typing stubs for chardet" +category = "dev" optional = false python-versions = "*" files = [ @@ -10735,6 +11201,7 @@ files = [ name = "types-protobuf" version = "4.23.0.1" description = "Typing stubs for protobuf" +category = "dev" optional = false python-versions = "*" files = [ @@ -10746,6 +11213,7 @@ files = [ name = "types-pyopenssl" version = "23.2.0.0" description = "Typing stubs for pyOpenSSL" +category = "dev" optional = false python-versions = "*" files = [ @@ -10760,6 +11228,7 @@ cryptography = ">=35.0.0" name = "types-pytz" version = "2023.3.0.0" description = "Typing stubs for pytz" +category = "dev" optional = false python-versions = "*" files = [ @@ -10771,6 +11240,7 @@ files = [ name = "types-pyyaml" version = "6.0.12.10" description = "Typing stubs for PyYAML" +category = "main" optional = false python-versions = "*" files = [ @@ -10782,6 +11252,7 @@ files = [ name = "types-redis" version = "4.5.5.2" description = "Typing stubs for redis" +category = "dev" optional = false python-versions = "*" files = [ @@ -10797,6 +11268,7 @@ types-pyOpenSSL = "*" name = "types-requests" version = "2.31.0.1" description = "Typing stubs for requests" +category = "main" optional = false python-versions = "*" files = [ @@ -10811,6 +11283,7 @@ types-urllib3 = "*" name = "types-toml" version = "0.10.8.6" description = "Typing stubs for toml" +category = "dev" optional = false python-versions = "*" files = [ @@ -10822,6 +11295,7 @@ files = [ name = "types-urllib3" version = "1.26.25.13" description = "Typing stubs for urllib3" +category = "main" optional = false python-versions = "*" files = [ @@ -10833,6 +11307,7 @@ files = [ name = "typing-extensions" version = "4.6.3" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -10844,6 +11319,7 @@ files = [ name = "typing-inspect" version = "0.9.0" description = "Runtime inspection utilities for typing module." +category = "main" optional = false python-versions = "*" files = [ @@ -10859,6 +11335,7 @@ typing-extensions = ">=3.7.4" name = "tzdata" version = "2023.3" description = "Provider of IANA time zone data" +category = "main" optional = false python-versions = ">=2" files = [ @@ -10870,6 +11347,7 @@ files = [ name = "tzlocal" version = "4.3" description = "tzinfo object for the local timezone" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -10889,6 +11367,7 @@ devenv = ["black", "check-manifest", "flake8", "pyroma", "pytest (>=4.3)", "pyte name = "uri-template" version = "1.2.0" description = "RFC 6570 URI Template Processor" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -10903,6 +11382,7 @@ dev = ["flake8 (<4.0.0)", "flake8-annotations", "flake8-bugbear", "flake8-commas name = "uritemplate" version = "4.1.1" description = "Implementation of RFC 6570 URI Templates" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -10914,6 +11394,7 @@ files = [ name = "urllib3" version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -10930,6 +11411,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "uvicorn" version = "0.22.0" description = "The lightning-fast ASGI server." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -10944,7 +11426,7 @@ h11 = ">=0.8" httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} -uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} @@ -10955,6 +11437,7 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", name = "uvloop" version = "0.17.0" description = "Fast implementation of asyncio event loop on top of libuv" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -10999,6 +11482,7 @@ test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "my name = "validators" version = "0.20.0" description = "Python Data Validation for Humans™." +category = "main" optional = false python-versions = ">=3.4" files = [ @@ -11015,6 +11499,7 @@ test = ["flake8 (>=2.4.0)", "isort (>=4.2.2)", "pytest (>=2.2.3)"] name = "vcrpy" version = "4.3.1" description = "Automatically mock your HTTP interactions to simplify and speed up testing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -11033,6 +11518,7 @@ yarl = "*" name = "wasabi" version = "1.1.2" description = "A lightweight console printing and formatting toolkit" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -11047,6 +11533,7 @@ colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\" and python name = "watchdog" version = "3.0.0" description = "Filesystem events monitoring" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11086,6 +11573,7 @@ watchmedo = ["PyYAML (>=3.10)"] name = "watchfiles" version = "0.19.0" description = "Simple, modern and high performance file watching and code reload in python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11120,6 +11608,7 @@ anyio = ">=3.0.0" name = "wcwidth" version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" +category = "main" optional = false python-versions = "*" files = [ @@ -11131,6 +11620,7 @@ files = [ name = "weaviate-client" version = "3.20.1" description = "A python native Weaviate client" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -11151,6 +11641,7 @@ grpc = ["grpcio", "grpcio-tools"] name = "webcolors" version = "1.13" description = "A library for working with the color formats defined by HTML and CSS." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -11166,6 +11657,7 @@ tests = ["pytest", "pytest-cov"] name = "webencodings" version = "0.5.1" description = "Character encoding aliases for legacy web content" +category = "dev" optional = false python-versions = "*" files = [ @@ -11177,6 +11669,7 @@ files = [ name = "websocket-client" version = "1.6.0" description = "WebSocket client for Python with low level API options" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11193,6 +11686,7 @@ test = ["websockets"] name = "websockets" version = "11.0.3" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11272,6 +11766,7 @@ files = [ name = "werkzeug" version = "2.3.6" description = "The comprehensive WSGI web application library." +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -11289,6 +11784,7 @@ watchdog = ["watchdog (>=2.3)"] name = "wget" version = "3.2" description = "pure python download utility" +category = "main" optional = false python-versions = "*" files = [ @@ -11299,6 +11795,7 @@ files = [ name = "wheel" version = "0.40.0" description = "A built-package format for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11313,6 +11810,7 @@ test = ["pytest (>=6.0.0)"] name = "whylabs-client" version = "0.5.1" description = "WhyLabs API client" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -11328,6 +11826,7 @@ urllib3 = ">=1.25.3" name = "whylogs" version = "1.1.45.dev6" description = "Profile and monitor your ML data pipeline end-to-end" +category = "main" optional = true python-versions = ">=3.7.1,<4" files = [ @@ -11361,6 +11860,7 @@ viz = ["Pillow (>=9.2.0,<10.0.0)", "ipython", "numpy", "numpy (>=1.23.2)", "pyba name = "whylogs-sketching" version = "3.4.1.dev3" description = "sketching library of whylogs" +category = "main" optional = true python-versions = "*" files = [ @@ -11401,6 +11901,7 @@ files = [ name = "widgetsnbextension" version = "4.0.7" description = "Jupyter interactive widgets for Jupyter Notebook" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -11412,6 +11913,7 @@ files = [ name = "wikipedia" version = "1.4.0" description = "Wikipedia API for Python" +category = "main" optional = false python-versions = "*" files = [ @@ -11426,6 +11928,7 @@ requests = ">=2.0.0,<3.0.0" name = "win32-setctime" version = "1.1.0" description = "A small Python utility to set file creation time on Windows" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -11440,6 +11943,7 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] name = "wolframalpha" version = "5.0.0" description = "Wolfram|Alpha 2.0 API client" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -11460,6 +11964,7 @@ testing = ["keyring", "pmxbot", "pytest (>=3.5,!=3.7.3)", "pytest-black (>=0.3.7 name = "wonderwords" version = "2.2.0" description = "A python package for random words and sentences in the english language" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -11474,6 +11979,7 @@ cli = ["rich (==9.10.0)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -11558,6 +12064,7 @@ files = [ name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" +category = "main" optional = true python-versions = ">=3.4" files = [ @@ -11569,6 +12076,7 @@ files = [ name = "xxhash" version = "3.2.0" description = "Python binding for xxHash" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -11676,6 +12184,7 @@ files = [ name = "yarl" version = "1.9.2" description = "Yet another URL library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11763,6 +12272,7 @@ multidict = ">=4.0" name = "zep-python" version = "0.32" description = "Zep stores, manages, enriches, indexes, and searches long-term memory for conversational AI applications. This is the Python client for the Zep service." +category = "main" optional = true python-versions = ">=3.8,<4.0" files = [ @@ -11778,6 +12288,7 @@ pydantic = ">=1.10.7,<2.0.0" name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -11793,6 +12304,7 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more name = "zstandard" version = "0.21.0" description = "Zstandard bindings for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ diff --git a/tests/unit_tests/callbacks/fake_callback_handler.py b/tests/unit_tests/callbacks/fake_callback_handler.py index e167a70ffa..f607b3c708 100644 --- a/tests/unit_tests/callbacks/fake_callback_handler.py +++ b/tests/unit_tests/callbacks/fake_callback_handler.py @@ -19,6 +19,7 @@ class BaseFakeCallbackHandler(BaseModel): ignore_llm_: bool = False ignore_chain_: bool = False ignore_agent_: bool = False + ignore_retriever_: bool = False ignore_chat_model_: bool = False # add finer-grained counters for easier debugging of failing tests @@ -32,6 +33,9 @@ class BaseFakeCallbackHandler(BaseModel): agent_actions: int = 0 agent_ends: int = 0 chat_model_starts: int = 0 + retriever_starts: int = 0 + retriever_ends: int = 0 + retriever_errors: int = 0 class BaseFakeCallbackHandlerMixin(BaseFakeCallbackHandler): @@ -52,7 +56,7 @@ class BaseFakeCallbackHandlerMixin(BaseFakeCallbackHandler): self.llm_streams += 1 def on_chain_start_common(self) -> None: - print("CHAIN START") + ("CHAIN START") self.chain_starts += 1 self.starts += 1 @@ -91,6 +95,18 @@ class BaseFakeCallbackHandlerMixin(BaseFakeCallbackHandler): def on_text_common(self) -> None: self.text += 1 + def on_retriever_start_common(self) -> None: + self.starts += 1 + self.retriever_starts += 1 + + def on_retriever_end_common(self) -> None: + self.ends += 1 + self.retriever_ends += 1 + + def on_retriever_error_common(self) -> None: + self.errors += 1 + self.retriever_errors += 1 + class FakeCallbackHandler(BaseCallbackHandler, BaseFakeCallbackHandlerMixin): """Fake callback handler for testing.""" @@ -110,6 +126,11 @@ class FakeCallbackHandler(BaseCallbackHandler, BaseFakeCallbackHandlerMixin): """Whether to ignore agent callbacks.""" return self.ignore_agent_ + @property + def ignore_retriever(self) -> bool: + """Whether to ignore retriever callbacks.""" + return self.ignore_retriever_ + def on_llm_start( self, *args: Any, @@ -201,6 +222,27 @@ class FakeCallbackHandler(BaseCallbackHandler, BaseFakeCallbackHandlerMixin): ) -> Any: self.on_text_common() + def on_retriever_start( + self, + *args: Any, + **kwargs: Any, + ) -> Any: + self.on_retriever_start_common() + + def on_retriever_end( + self, + *args: Any, + **kwargs: Any, + ) -> Any: + self.on_retriever_end_common() + + def on_retriever_error( + self, + *args: Any, + **kwargs: Any, + ) -> Any: + self.on_retriever_error_common() + def __deepcopy__(self, memo: dict) -> "FakeCallbackHandler": return self diff --git a/tests/unit_tests/callbacks/test_callback_manager.py b/tests/unit_tests/callbacks/test_callback_manager.py index 73d38233ba..5793e71d14 100644 --- a/tests/unit_tests/callbacks/test_callback_manager.py +++ b/tests/unit_tests/callbacks/test_callback_manager.py @@ -141,6 +141,22 @@ def test_ignore_agent() -> None: assert handler2.errors == 1 +def test_ignore_retriever() -> None: + """Test the ignore retriever param for callback handlers.""" + handler1 = FakeCallbackHandler(ignore_retriever_=True) + handler2 = FakeCallbackHandler() + manager = CallbackManager(handlers=[handler1, handler2]) + run_manager = manager.on_retriever_start("") + run_manager.on_retriever_end([]) + run_manager.on_retriever_error(Exception()) + assert handler1.starts == 0 + assert handler1.ends == 0 + assert handler1.errors == 0 + assert handler2.starts == 1 + assert handler2.ends == 1 + assert handler2.errors == 1 + + @pytest.mark.asyncio async def test_async_callback_manager() -> None: """Test the AsyncCallbackManager.""" diff --git a/tests/unit_tests/retrievers/test_base.py b/tests/unit_tests/retrievers/test_base.py new file mode 100644 index 0000000000..e3e15ff4e9 --- /dev/null +++ b/tests/unit_tests/retrievers/test_base.py @@ -0,0 +1,220 @@ +"""Test Base Retriever logic.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import pytest + +from langchain.callbacks.manager import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) +from langchain.schema import BaseRetriever, Document +from tests.unit_tests.callbacks.fake_callback_handler import FakeCallbackHandler + + +@pytest.fixture +def fake_retriever_v1() -> BaseRetriever: + with pytest.warns( + DeprecationWarning, + match="Retrievers must implement abstract " + "`_get_relevant_documents` method instead of `get_relevant_documents`", + ): + + class FakeRetrieverV1(BaseRetriever): + def get_relevant_documents( # type: ignore[override] + self, + query: str, + ) -> List[Document]: + assert isinstance(self, FakeRetrieverV1) + return [ + Document(page_content=query, metadata={"uuid": "1234"}), + ] + + async def aget_relevant_documents( # type: ignore[override] + self, + query: str, + ) -> List[Document]: + assert isinstance(self, FakeRetrieverV1) + return [ + Document( + page_content=f"Async query {query}", metadata={"uuid": "1234"} + ), + ] + + return FakeRetrieverV1() # type: ignore[abstract] + + +def test_fake_retriever_v1_upgrade(fake_retriever_v1: BaseRetriever) -> None: + callbacks = FakeCallbackHandler() + assert fake_retriever_v1._new_arg_supported is False + assert fake_retriever_v1._expects_other_args is False + results: List[Document] = fake_retriever_v1.get_relevant_documents( + "Foo", callbacks=[callbacks] + ) + assert results[0].page_content == "Foo" + assert callbacks.retriever_starts == 1 + assert callbacks.retriever_ends == 1 + assert callbacks.retriever_errors == 0 + + +@pytest.mark.asyncio +async def test_fake_retriever_v1_upgrade_async( + fake_retriever_v1: BaseRetriever, +) -> None: + callbacks = FakeCallbackHandler() + assert fake_retriever_v1._new_arg_supported is False + assert fake_retriever_v1._expects_other_args is False + results: List[Document] = await fake_retriever_v1.aget_relevant_documents( + "Foo", callbacks=[callbacks] + ) + assert results[0].page_content == "Async query Foo" + assert callbacks.retriever_starts == 1 + assert callbacks.retriever_ends == 1 + assert callbacks.retriever_errors == 0 + + +@pytest.fixture +def fake_retriever_v1_with_kwargs() -> BaseRetriever: + # Test for things like the Weaviate V1 Retriever. + with pytest.warns( + DeprecationWarning, + match="Retrievers must implement abstract " + "`_get_relevant_documents` method instead of `get_relevant_documents`", + ): + + class FakeRetrieverV1(BaseRetriever): + def get_relevant_documents( # type: ignore[override] + self, query: str, where_filter: Optional[Dict[str, object]] = None + ) -> List[Document]: + assert isinstance(self, FakeRetrieverV1) + return [ + Document(page_content=query, metadata=where_filter or {}), + ] + + async def aget_relevant_documents( # type: ignore[override] + self, query: str, where_filter: Optional[Dict[str, object]] = None + ) -> List[Document]: + assert isinstance(self, FakeRetrieverV1) + return [ + Document( + page_content=f"Async query {query}", metadata=where_filter or {} + ), + ] + + return FakeRetrieverV1() # type: ignore[abstract] + + +def test_fake_retriever_v1_with_kwargs_upgrade( + fake_retriever_v1_with_kwargs: BaseRetriever, +) -> None: + callbacks = FakeCallbackHandler() + assert fake_retriever_v1_with_kwargs._new_arg_supported is False + assert fake_retriever_v1_with_kwargs._expects_other_args is True + results: List[Document] = fake_retriever_v1_with_kwargs.get_relevant_documents( + "Foo", callbacks=[callbacks], where_filter={"foo": "bar"} + ) + assert results[0].page_content == "Foo" + assert results[0].metadata == {"foo": "bar"} + assert callbacks.retriever_starts == 1 + assert callbacks.retriever_ends == 1 + assert callbacks.retriever_errors == 0 + + +@pytest.mark.asyncio +async def test_fake_retriever_v1_with_kwargs_upgrade_async( + fake_retriever_v1_with_kwargs: BaseRetriever, +) -> None: + callbacks = FakeCallbackHandler() + assert fake_retriever_v1_with_kwargs._new_arg_supported is False + assert fake_retriever_v1_with_kwargs._expects_other_args is True + results: List[ + Document + ] = await fake_retriever_v1_with_kwargs.aget_relevant_documents( + "Foo", callbacks=[callbacks], where_filter={"foo": "bar"} + ) + assert results[0].page_content == "Async query Foo" + assert results[0].metadata == {"foo": "bar"} + assert callbacks.retriever_starts == 1 + assert callbacks.retriever_ends == 1 + assert callbacks.retriever_errors == 0 + + +@pytest.fixture +def fake_retriever_v2() -> BaseRetriever: + class FakeRetrieverV2(BaseRetriever): + def _get_relevant_documents( + self, + query: str, + *, + run_manager: CallbackManagerForRetrieverRun | None = None, + **kwargs: Any, + ) -> List[Document]: + assert isinstance(self, FakeRetrieverV2) + assert run_manager is not None + assert isinstance(run_manager, CallbackManagerForRetrieverRun) + if "throw_error" in kwargs: + raise ValueError("Test error") + return [ + Document(page_content=query, metadata=kwargs), + ] + + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun | None = None, + **kwargs: Any, + ) -> List[Document]: + assert isinstance(self, FakeRetrieverV2) + assert run_manager is not None + assert isinstance(run_manager, AsyncCallbackManagerForRetrieverRun) + if "throw_error" in kwargs: + raise ValueError("Test error") + return [ + Document(page_content=f"Async query {query}", metadata=kwargs), + ] + + return FakeRetrieverV2() # type: ignore[abstract] + + +def test_fake_retriever_v2(fake_retriever_v2: BaseRetriever) -> None: + callbacks = FakeCallbackHandler() + assert fake_retriever_v2._new_arg_supported is True + results = fake_retriever_v2.get_relevant_documents("Foo", callbacks=[callbacks]) + assert results[0].page_content == "Foo" + assert callbacks.retriever_starts == 1 + assert callbacks.retriever_ends == 1 + assert callbacks.retriever_errors == 0 + results2 = fake_retriever_v2.get_relevant_documents( + "Foo", callbacks=[callbacks], foo="bar" + ) + assert results2[0].metadata == {"foo": "bar"} + + with pytest.raises(ValueError, match="Test error"): + fake_retriever_v2.get_relevant_documents( + "Foo", callbacks=[callbacks], throw_error=True + ) + assert callbacks.retriever_errors == 1 + + +@pytest.mark.asyncio +async def test_fake_retriever_v2_async(fake_retriever_v2: BaseRetriever) -> None: + callbacks = FakeCallbackHandler() + assert fake_retriever_v2._new_arg_supported is True + results = await fake_retriever_v2.aget_relevant_documents( + "Foo", callbacks=[callbacks] + ) + assert results[0].page_content == "Async query Foo" + assert callbacks.retriever_starts == 1 + assert callbacks.retriever_ends == 1 + assert callbacks.retriever_errors == 0 + results2 = await fake_retriever_v2.aget_relevant_documents( + "Foo", callbacks=[callbacks], foo="bar" + ) + assert results2[0].metadata == {"foo": "bar"} + with pytest.raises(ValueError, match="Test error"): + await fake_retriever_v2.aget_relevant_documents( + "Foo", callbacks=[callbacks], throw_error=True + )