diff --git a/docs/docs_skeleton/docs/integrations/memory/momento_chat_message_history.ipynb b/docs/docs_skeleton/docs/integrations/memory/momento_chat_message_history.ipynb index c58fa408e5..362ccf8d78 100644 --- a/docs/docs_skeleton/docs/integrations/memory/momento_chat_message_history.ipynb +++ b/docs/docs_skeleton/docs/integrations/memory/momento_chat_message_history.ipynb @@ -15,7 +15,7 @@ "\n", "Note that, by default we will create a cache if one with the given name doesn't already exist.\n", "\n", - "You'll need to get a Momento auth token to use this class. This can either be passed in to a momento.CacheClient if you'd like to instantiate that directly, as a named parameter `auth_token` to `MomentoChatMessageHistory.from_client_params`, or can just be set as an environment variable `MOMENTO_AUTH_TOKEN`." + "You'll need to get a Momento API key to use this class. This can either be passed in to a momento.CacheClient if you'd like to instantiate that directly, as a named parameter `api_key` to `MomentoChatMessageHistory.from_client_params`, or can just be set as an environment variable `MOMENTO_API_KEY`." ] }, { diff --git a/docs/docs_skeleton/docs/integrations/providers/momento.mdx b/docs/docs_skeleton/docs/integrations/providers/momento.mdx index 2317c80cd7..1437dbd294 100644 --- a/docs/docs_skeleton/docs/integrations/providers/momento.mdx +++ b/docs/docs_skeleton/docs/integrations/providers/momento.mdx @@ -1,31 +1,30 @@ # Momento ->[Momento Cache](https://docs.momentohq.com/) is the world's first truly serverless caching service. It provides instant elasticity, scale-to-zero -> capability, and blazing-fast performance. -> With Momento Cache, you grab the SDK, you get an end point, input a few lines into your code, and you're off and running. +> [Momento Cache](https://docs.momentohq.com/) is the world's first truly serverless caching service, offering instant elasticity, scale-to-zero +> capability, and blazing-fast performance. +> +> [Momento Vector Index](https://docs.momentohq.com/vector-index) stands out as the most productive, easiest-to-use, fully serverless vector index. +> +> For both services, simply grab the SDK, obtain an API key, input a few lines into your code, and you're set to go. Together, they provide a comprehensive solution for your LLM data needs. This page covers how to use the [Momento](https://gomomento.com) ecosystem within LangChain. ## Installation and Setup -- Sign up for a free account [here](https://docs.momentohq.com/getting-started) and get an auth token +- Sign up for a free account [here](https://console.momentohq.com) to get an API key - Install the Momento Python SDK with `pip install momento` - ## Cache -The Cache wrapper allows for [Momento](https://gomomento.com) to be used as a serverless, distributed, low-latency cache for LLM prompts and responses. - - -The standard cache is the go-to use case for [Momento](https://gomomento.com) users in any environment. +Use Momento as a serverless, distributed, low-latency cache for LLM prompts and responses. The standard cache is the primary use case for Momento users in any environment. -Import the cache as follows: +To integrate Momento Cache into your application: ```python from langchain.cache import MomentoCache ``` -And set up like so: +Then, set it up with the following code: ```python from datetime import timedelta @@ -35,7 +34,7 @@ import langchain # Instantiate the Momento client cache_client = CacheClient( Configurations.Laptop.v1(), - CredentialProvider.from_environment_variable("MOMENTO_AUTH_TOKEN"), + CredentialProvider.from_environment_variable("MOMENTO_API_KEY"), default_ttl=timedelta(days=1)) # Choose a Momento cache name of your choice @@ -52,3 +51,9 @@ Momento can be used as a distributed memory store for LLMs. ### Chat Message History Memory See [this notebook](/docs/integrations/memory/momento_chat_message_history.html) for a walkthrough of how to use Momento as a memory store for chat message history. + +## Vector Store + +Momento Vector Index (MVI) can be used as a vector store. + +See [this notebook](/docs/integrations/vectorstores/momento_vector_index.html) for a walkthrough of how to use MVI as a vector store. diff --git a/docs/docs_skeleton/docs/integrations/vectorstores/momento_vector_index.ipynb b/docs/docs_skeleton/docs/integrations/vectorstores/momento_vector_index.ipynb new file mode 100644 index 0000000000..e35633e79c --- /dev/null +++ b/docs/docs_skeleton/docs/integrations/vectorstores/momento_vector_index.ipynb @@ -0,0 +1,474 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "683953b3", + "metadata": {}, + "source": [ + "# Momento Vector Index (MVI)\n", + "\n", + ">[MVI](https://gomomento.com): the most productive, easiest to use, serverless vector index for your data. To get started with MVI, simply sign up for an account. There's no need to handle infrastructure, manage servers, or be concerned about scaling. MVI is a service that scales automatically to meet your needs.\n", + "\n", + "To sign up and access MVI, visit the [Momento Console](https://console.gomomento.com)." + ] + }, + { + "cell_type": "markdown", + "id": "82581e78", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "markdown", + "id": "3120d063", + "metadata": {}, + "source": [ + "## Install prerequisites" + ] + }, + { + "cell_type": "markdown", + "id": "9d7e5fd5", + "metadata": {}, + "source": [ + "You will need:\n", + "- the [`momento`](https://pypi.org/project/momento/) package for interacting with MVI, and\n", + "- the openai package for interacting with the OpenAI API.\n", + "- the tiktoken package for tokenizing text." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a62cff8a-bcf7-4e33-bbbc-76999c2e3e20", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "!pip install momento openai tiktoken\n" + ] + }, + { + "cell_type": "markdown", + "id": "8317b9df", + "metadata": {}, + "source": [ + "## Enter API keys" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4b96eed5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import getpass\n" + ] + }, + { + "cell_type": "markdown", + "id": "7ce4b6f7", + "metadata": {}, + "source": [ + "### Momento: for indexing data" + ] + }, + { + "cell_type": "markdown", + "id": "78b8b2ee", + "metadata": {}, + "source": [ + "Visit the [Momento Console](https://console.gomomento.com) to get your API key." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "211407a8", + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"MOMENTO_API_KEY\"] = getpass.getpass(\"Momento API Key:\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "08148c5f", + "metadata": {}, + "source": [ + "### OpenAI: for text embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "8b6ed9cd-81b9-46e5-9c20-5aafca2844d0", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "347932a6", + "metadata": {}, + "source": [ + "# Load your data" + ] + }, + { + "cell_type": "markdown", + "id": "2cfa2538", + "metadata": {}, + "source": [ + "Here we use the example dataset from Langchain, the state of the union address.\n", + "\n", + "First we load relevant modules:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "aac9563e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from langchain.embeddings.openai import OpenAIEmbeddings\n", + "from langchain.text_splitter import CharacterTextSplitter\n", + "from langchain.vectorstores import MomentoVectorIndex\n", + "from langchain.document_loaders import TextLoader\n" + ] + }, + { + "cell_type": "markdown", + "id": "f75e1221", + "metadata": {}, + "source": [ + "Then we load the data:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "a3c3999a", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "loader = TextLoader(\"../../modules/state_of_the_union.txt\")\n", + "documents = loader.load()\n", + "len(documents)\n" + ] + }, + { + "cell_type": "markdown", + "id": "31a90e56", + "metadata": {}, + "source": [ + "Note the data is one large file, hence there is only one document:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "1926aaae", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "38539" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(documents[0].page_content)\n" + ] + }, + { + "cell_type": "markdown", + "id": "1ff35d84", + "metadata": {}, + "source": [ + "Because this is one large text file, we split it into chunks for question answering. That way, user questions will be answered from the most relevant chunk." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "1de69459", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "42" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n", + "docs = text_splitter.split_documents(documents)\n", + "len(docs)\n" + ] + }, + { + "cell_type": "markdown", + "id": "cb7854c1", + "metadata": {}, + "source": [ + "# Index your data" + ] + }, + { + "cell_type": "markdown", + "id": "42059ec1", + "metadata": {}, + "source": [ + "Indexing your data is as simple as instantiating the `MomentoVectorIndex` object. Here we use the `from_documents` helper to both instantiate and index the data:" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "dcf88bdf", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "vector_db = MomentoVectorIndex.from_documents(\n", + " docs,\n", + " OpenAIEmbeddings(),\n", + " index_name=\"sotu\"\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "id": "225cd0e2", + "metadata": {}, + "source": [ + "This connects to the Momento Vector Index service using your API key and indexes the data. If the index did not exist before, this process creates it for you. The data is now searchable." + ] + }, + { + "cell_type": "markdown", + "id": "ffb2c44e", + "metadata": {}, + "source": [ + "# Query your data" + ] + }, + { + "cell_type": "markdown", + "id": "e705a976", + "metadata": {}, + "source": [ + "## Ask a question directly against the index" + ] + }, + { + "cell_type": "markdown", + "id": "173185cd", + "metadata": {}, + "source": [ + "The most direct way to query the data is to search against the index. We can do that as follows using the `VectorStore` API:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a8c513ab", + "metadata": {}, + "outputs": [], + "source": [ + "query = \"What did the president say about Ketanji Brown Jackson\"\n", + "docs = vector_db.similarity_search(query)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "fc516993", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \\n\\nTonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \\n\\nOne of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \\n\\nAnd I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.'" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "docs[0].page_content\n" + ] + }, + { + "cell_type": "markdown", + "id": "f03b7cf6", + "metadata": {}, + "source": [ + "While this does contain relevant information about Ketanji Brown Jackson, we don't have a concise, human-readable answer. We'll tackle that in the next section." + ] + }, + { + "cell_type": "markdown", + "id": "a8979022", + "metadata": {}, + "source": [ + "## Use an LLM to generate fluent answers" + ] + }, + { + "cell_type": "markdown", + "id": "d5d51797", + "metadata": {}, + "source": [ + "With the data indexed in MVI, we can integrate with any chain that leverages vector similarity search. Here we use the `RetrievalQA` chain to demonstrate how to answer questions from the indexed data." + ] + }, + { + "cell_type": "markdown", + "id": "9f5a4386", + "metadata": {}, + "source": [ + "First we load the relevant modules:" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "7fa71a86", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.chat_models import ChatOpenAI\n", + "from langchain.chains import RetrievalQA\n" + ] + }, + { + "cell_type": "markdown", + "id": "026cffe3", + "metadata": {}, + "source": [ + "Then we instantiate the retrieval QA chain:" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "3c0a7b2b", + "metadata": {}, + "outputs": [], + "source": [ + "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n", + "qa_chain = RetrievalQA.from_chain_type(llm, retriever=vector_db.as_retriever())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "19c1a3c8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'query': 'What did the president say about Ketanji Brown Jackson?',\n", + " 'result': \"The President said that he nominated Circuit Court of Appeals Judge Ketanji Brown Jackson to serve on the United States Supreme Court. He described her as one of the nation's top legal minds and mentioned that she has received broad support from various groups, including the Fraternal Order of Police and former judges appointed by Democrats and Republicans.\"}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "qa_chain({\"query\": \"What did the president say about Ketanji Brown Jackson?\"})\n" + ] + }, + { + "cell_type": "markdown", + "id": "d120efcf", + "metadata": {}, + "source": [ + "# Next Steps" + ] + }, + { + "cell_type": "markdown", + "id": "96a0cc66", + "metadata": {}, + "source": [ + "That's it! You've now indexed your data and can query it using the Momento Vector Index. You can use the same index to query your data from any chain that supports vector similarity search.\n", + "\n", + "With Momento you can not only index your vector data, but also cache your API calls and store your chat message history. Check out the other Momento langchain integrations to learn more.\n", + "\n", + "To learn more about the Momento Vector Index, visit the [Momento Documentation](https://docs.gomomento.com).\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "7f18e7b7", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/libs/langchain/langchain/cache.py b/libs/langchain/langchain/cache.py index e77528924c..bbcc87bd39 100644 --- a/libs/langchain/langchain/cache.py +++ b/libs/langchain/langchain/cache.py @@ -684,7 +684,8 @@ class MomentoCache(BaseCache): ttl: timedelta, *, configuration: Optional[momento.config.Configuration] = None, - auth_token: Optional[str] = None, + api_key: Optional[str] = None, + auth_token: Optional[str] = None, # for backwards compatibility **kwargs: Any, ) -> MomentoCache: """Construct cache from CacheClient parameters.""" @@ -697,8 +698,13 @@ class MomentoCache(BaseCache): ) if configuration is None: configuration = Configurations.Laptop.v1() - auth_token = auth_token or get_from_env("auth_token", "MOMENTO_AUTH_TOKEN") - credentials = CredentialProvider.from_string(auth_token) + + # Try checking `MOMENTO_AUTH_TOKEN` first for backwards compatibility + try: + api_key = auth_token or get_from_env("auth_token", "MOMENTO_AUTH_TOKEN") + except ValueError: + api_key = api_key or get_from_env("api_key", "MOMENTO_API_KEY") + credentials = CredentialProvider.from_string(api_key) cache_client = CacheClient(configuration, credentials, default_ttl=ttl) return cls(cache_client, cache_name, ttl=ttl, **kwargs) diff --git a/libs/langchain/langchain/memory/chat_message_histories/momento.py b/libs/langchain/langchain/memory/chat_message_histories/momento.py index 448801db81..12ffbaf4c2 100644 --- a/libs/langchain/langchain/memory/chat_message_histories/momento.py +++ b/libs/langchain/langchain/memory/chat_message_histories/momento.py @@ -97,7 +97,8 @@ class MomentoChatMessageHistory(BaseChatMessageHistory): ttl: timedelta, *, configuration: Optional[momento.config.Configuration] = None, - auth_token: Optional[str] = None, + api_key: Optional[str] = None, + auth_token: Optional[str] = None, # for backwards compatibility **kwargs: Any, ) -> MomentoChatMessageHistory: """Construct cache from CacheClient parameters.""" @@ -110,8 +111,13 @@ class MomentoChatMessageHistory(BaseChatMessageHistory): ) if configuration is None: configuration = Configurations.Laptop.v1() - auth_token = auth_token or get_from_env("auth_token", "MOMENTO_AUTH_TOKEN") - credentials = CredentialProvider.from_string(auth_token) + + # Try checking `MOMENTO_AUTH_TOKEN` first for backwards compatibility + try: + api_key = auth_token or get_from_env("auth_token", "MOMENTO_AUTH_TOKEN") + except ValueError: + api_key = api_key or get_from_env("api_key", "MOMENTO_API_KEY") + credentials = CredentialProvider.from_string(api_key) cache_client = CacheClient(configuration, credentials, default_ttl=ttl) return cls(session_id, cache_client, cache_name, ttl=ttl, **kwargs) diff --git a/libs/langchain/langchain/vectorstores/__init__.py b/libs/langchain/langchain/vectorstores/__init__.py index ac2eb0fb57..7de91d32db 100644 --- a/libs/langchain/langchain/vectorstores/__init__.py +++ b/libs/langchain/langchain/vectorstores/__init__.py @@ -205,6 +205,12 @@ def _import_milvus() -> Any: return Milvus +def _import_momento_vector_index() -> Any: + from langchain.vectorstores.momento_vector_index import MomentoVectorIndex + + return MomentoVectorIndex + + def _import_mongodb_atlas() -> Any: from langchain.vectorstores.mongodb_atlas import MongoDBAtlasVectorSearch @@ -446,6 +452,8 @@ def __getattr__(name: str) -> Any: return _import_meilisearch() elif name == "Milvus": return _import_milvus() + elif name == "MomentoVectorIndex": + return _import_momento_vector_index() elif name == "MongoDBAtlasVectorSearch": return _import_mongodb_atlas() elif name == "MyScaleSettings": @@ -545,6 +553,7 @@ __all__ = [ "MatchingEngine", "Meilisearch", "Milvus", + "MomentoVectorIndex", "MongoDBAtlasVectorSearch", "MyScale", "MyScaleSettings", diff --git a/libs/langchain/langchain/vectorstores/momento_vector_index.py b/libs/langchain/langchain/vectorstores/momento_vector_index.py new file mode 100644 index 0000000000..be56363cbe --- /dev/null +++ b/libs/langchain/langchain/vectorstores/momento_vector_index.py @@ -0,0 +1,389 @@ +from typing import ( + TYPE_CHECKING, + Any, + Iterable, + List, + Optional, + Tuple, + Type, + TypeVar, + cast, +) +from uuid import uuid4 + +from langchain.docstore.document import Document +from langchain.schema.embeddings import Embeddings +from langchain.schema.vectorstore import VectorStore +from langchain.utils import get_from_env +from langchain.vectorstores.utils import DistanceStrategy + +VST = TypeVar("VST", bound="VectorStore") + + +if TYPE_CHECKING: + from momento import PreviewVectorIndexClient + + +class MomentoVectorIndex(VectorStore): + """`Momento Vector Index` (MVI) vector store. + + Momento Vector Index is a serverless vector index that can be used to store and + search vectors. To use you should have the ``momento`` python package installed. + + Example: + .. code-block:: python + + from langchain.embeddings import OpenAIEmbeddings + from langchain.vectorstores import MomentoVectorIndex + from momento import ( + CredentialProvider, + PreviewVectorIndexClient, + VectorIndexConfigurations, + ) + + vectorstore = MomentoVectorIndex( + embedding=OpenAIEmbeddings(), + client=PreviewVectorIndexClient( + VectorIndexConfigurations.Default.latest(), + credential_provider=CredentialProvider.from_environment_variable( + "MOMENTO_API_KEY" + ), + ), + index_name="my-index", + ) + """ + + def __init__( + self, + embedding: Embeddings, + client: "PreviewVectorIndexClient", + index_name: str = "default", + distance_strategy: DistanceStrategy = DistanceStrategy.COSINE, + text_field: str = "text", + ensure_index_exists: bool = True, + **kwargs: Any, + ): + """Initialize a Vector Store backed by Momento Vector Index. + + Args: + embedding (Embeddings): The embedding function to use. + configuration (VectorIndexConfiguration): The configuration to initialize + the Vector Index with. + credential_provider (CredentialProvider): The credential provider to + authenticate the Vector Index with. + index_name (str, optional): The name of the index to store the documents in. + Defaults to "default". + distance_strategy (DistanceStrategy, optional): The distance strategy to + use. Defaults to DistanceStrategy.COSINE. If you select + DistanceStrategy.EUCLIDEAN_DISTANCE, Momento uses the squared + Euclidean distance. + text_field (str, optional): The name of the metadata field to store the + original text in. Defaults to "text". + ensure_index_exists (bool, optional): Whether to ensure that the index + exists before adding documents to it. Defaults to True. + """ + try: + from momento import PreviewVectorIndexClient + except ImportError: + raise ImportError( + "Could not import momento python package. " + "Please install it with `pip install momento`." + ) + + self._client: PreviewVectorIndexClient = client + self._embedding = embedding + self.index_name = index_name + self.__validate_distance_strategy(distance_strategy) + self.distance_strategy = distance_strategy + self.text_field = text_field + self._ensure_index_exists = ensure_index_exists + + @staticmethod + def __validate_distance_strategy(distance_strategy: DistanceStrategy) -> None: + if distance_strategy not in [ + DistanceStrategy.COSINE, + DistanceStrategy.MAX_INNER_PRODUCT, + DistanceStrategy.MAX_INNER_PRODUCT, + ]: + raise ValueError(f"Distance strategy {distance_strategy} not implemented.") + + @property + def embeddings(self) -> Embeddings: + return self._embedding + + def _create_index_if_not_exists(self, num_dimensions: int) -> bool: + """Create index if it does not exist.""" + from momento.requests.vector_index import SimilarityMetric + from momento.responses.vector_index import CreateIndex + + similarity_metric = None + if self.distance_strategy == DistanceStrategy.COSINE: + similarity_metric = SimilarityMetric.COSINE_SIMILARITY + elif self.distance_strategy == DistanceStrategy.MAX_INNER_PRODUCT: + similarity_metric = SimilarityMetric.INNER_PRODUCT + elif self.distance_strategy == DistanceStrategy.EUCLIDEAN_DISTANCE: + similarity_metric = SimilarityMetric.EUCLIDEAN_SIMILARITY + else: + raise ValueError( + f"Distance strategy {self.distance_strategy} not implemented." + ) + + response = self._client.create_index( + self.index_name, num_dimensions, similarity_metric + ) + if isinstance(response, CreateIndex.Success): + return True + elif isinstance(response, CreateIndex.IndexAlreadyExists): + return False + elif isinstance(response, CreateIndex.Error): + raise response.inner_exception + else: + raise Exception(f"Unexpected response: {response}") + + def add_texts( + self, + texts: Iterable[str], + metadatas: Optional[List[dict]] = None, + **kwargs: Any, + ) -> List[str]: + """Run more texts through the embeddings and add to the vectorstore. + + Args: + texts (Iterable[str]): Iterable of strings to add to the vectorstore. + metadatas (Optional[List[dict]]): Optional list of metadatas associated with + the texts. + kwargs (Any): Other optional parameters. Specifically: + - ids (List[str], optional): List of ids to use for the texts. + Defaults to None, in which case uuids are generated. + + Returns: + List[str]: List of ids from adding the texts into the vectorstore. + """ + from momento.requests.vector_index import Item + from momento.responses.vector_index import UpsertItemBatch + + texts = list(texts) + + if len(texts) == 0: + return [] + + if metadatas is not None: + for metadata, text in zip(metadatas, texts): + metadata[self.text_field] = text + else: + metadatas = [{self.text_field: text} for text in texts] + + try: + embeddings = self._embedding.embed_documents(texts) + except NotImplementedError: + embeddings = [self._embedding.embed_query(x) for x in texts] + + # Create index if it does not exist. + # We assume that if it does exist, then it was created with the desired number + # of dimensions and similarity metric. + if self._ensure_index_exists: + self._create_index_if_not_exists(len(embeddings[0])) + + if "ids" in kwargs: + ids = kwargs["ids"] + if len(ids) != len(embeddings): + raise ValueError("Number of ids must match number of texts") + else: + ids = [str(uuid4()) for _ in range(len(embeddings))] + + batch_size = 128 + for i in range(0, len(embeddings), batch_size): + start = i + end = min(i + batch_size, len(embeddings)) + items = [ + Item(id=id, vector=vector, metadata=metadata) + for id, vector, metadata in zip( + ids[start:end], + embeddings[start:end], + metadatas[start:end], + ) + ] + + response = self._client.upsert_item_batch(self.index_name, items) + if isinstance(response, UpsertItemBatch.Success): + pass + elif isinstance(response, UpsertItemBatch.Error): + raise response.inner_exception + else: + raise Exception(f"Unexpected response: {response}") + + return ids + + def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]: + """Delete by vector ID. + + Args: + ids (List[str]): List of ids to delete. + kwargs (Any): Other optional parameters (unused) + + Returns: + Optional[bool]: True if deletion is successful, + False otherwise, None if not implemented. + """ + from momento.responses.vector_index import DeleteItemBatch + + if ids is None: + return True + response = self._client.delete_item_batch(self.index_name, ids) + return isinstance(response, DeleteItemBatch.Success) + + def similarity_search( + self, query: str, k: int = 4, **kwargs: Any + ) -> List[Document]: + """Search for similar documents to the query string. + + Args: + query (str): The query string to search for. + k (int, optional): The number of results to return. Defaults to 4. + + Returns: + List[Document]: A list of documents that are similar to the query. + """ + res = self.similarity_search_with_score(query=query, k=k, **kwargs) + return [doc for doc, _ in res] + + def similarity_search_with_score( + self, + query: str, + k: int = 4, + **kwargs: Any, + ) -> List[Tuple[Document, float]]: + """Search for similar documents to the query string. + + Args: + query (str): The query string to search for. + k (int, optional): The number of results to return. Defaults to 4. + kwargs (Any): Vector Store specific search parameters. The following are + forwarded to the Momento Vector Index: + - top_k (int, optional): The number of results to return. + + Returns: + List[Tuple[Document, float]]: A list of tuples of the form + (Document, score). + """ + embedding = self._embedding.embed_query(query) + + results = self.similarity_search_with_score_by_vector( + embedding=embedding, k=k, **kwargs + ) + return results + + def similarity_search_with_score_by_vector( + self, + embedding: List[float], + k: int = 4, + **kwargs: Any, + ) -> List[Tuple[Document, float]]: + """Search for similar documents to the query vector. + + Args: + embedding (List[float]): The query vector to search for. + k (int, optional): The number of results to return. Defaults to 4. + kwargs (Any): Vector Store specific search parameters. The following are + forwarded to the Momento Vector Index: + - top_k (int, optional): The number of results to return. + + Returns: + List[Tuple[Document, float]]: A list of tuples of the form + (Document, score). + """ + from momento.requests.vector_index import ALL_METADATA + from momento.responses.vector_index import Search + + if "top_k" in kwargs: + k = kwargs["k"] + response = self._client.search( + self.index_name, embedding, top_k=k, metadata_fields=ALL_METADATA + ) + + if not isinstance(response, Search.Success): + return [] + + results = [] + for hit in response.hits: + text = cast(str, hit.metadata.pop(self.text_field)) + doc = Document(page_content=text, metadata=hit.metadata) + pair = (doc, hit.distance) + results.append(pair) + + return results + + def similarity_search_by_vector( + self, embedding: List[float], k: int = 4, **kwargs: Any + ) -> List[Document]: + """Search for similar documents to the query vector. + + Args: + embedding (List[float]): The query vector to search for. + k (int, optional): The number of results to return. Defaults to 4. + + Returns: + List[Document]: A list of documents that are similar to the query. + """ + results = self.similarity_search_with_score_by_vector( + embedding=embedding, k=k, **kwargs + ) + return [doc for doc, _ in results] + + @classmethod + def from_texts( + cls: Type[VST], + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + **kwargs: Any, + ) -> VST: + """Return the Vector Store initialized from texts and embeddings. + + Args: + cls (Type[VST]): The Vector Store class to use to initialize + the Vector Store. + texts (List[str]): The texts to initialize the Vector Store with. + embedding (Embeddings): The embedding function to use. + metadatas (Optional[List[dict]], optional): The metadata associated with + the texts. Defaults to None. + kwargs (Any): Vector Store specific parameters. The following are forwarded + to the Vector Store constructor and required: + - index_name (str, optional): The name of the index to store the documents + in. Defaults to "default". + - text_field (str, optional): The name of the metadata field to store the + original text in. Defaults to "text". + - distance_strategy (DistanceStrategy, optional): The distance strategy to + use. Defaults to DistanceStrategy.COSINE. If you select + DistanceStrategy.EUCLIDEAN_DISTANCE, Momento uses the squared + Euclidean distance. + - ensure_index_exists (bool, optional): Whether to ensure that the index + exists before adding documents to it. Defaults to True. + Additionally you can either pass in a client or an API key + - client (PreviewVectorIndexClient): The Momento Vector Index client to use. + - api_key (Optional[str]): The configuration to use to initialize + the Vector Index with. Defaults to None. If None, the configuration + is initialized from the environment variable `MOMENTO_API_KEY`. + + Returns: + VST: Momento Vector Index vector store initialized from texts and + embeddings. + """ + from momento import ( + CredentialProvider, + PreviewVectorIndexClient, + VectorIndexConfigurations, + ) + + if "client" in kwargs: + client = kwargs.pop("client") + else: + supplied_api_key = kwargs.pop("api_key", None) + api_key = supplied_api_key or get_from_env("api_key", "MOMENTO_API_KEY") + client = PreviewVectorIndexClient( + configuration=VectorIndexConfigurations.Default.latest(), + credential_provider=CredentialProvider.from_string(api_key), + ) + vector_db = cls(embedding=embedding, client=client, **kwargs) # type: ignore + vector_db.add_texts(texts=texts, metadatas=metadatas, **kwargs) + return vector_db diff --git a/libs/langchain/poetry.lock b/libs/langchain/poetry.lock index 891448a6a6..2565ffb006 100644 --- a/libs/langchain/poetry.lock +++ b/libs/langchain/poetry.lock @@ -2884,7 +2884,6 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -2893,7 +2892,6 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -2923,7 +2921,6 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -2932,7 +2929,6 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -3726,7 +3722,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" files = [ {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, - {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, ] [[package]] @@ -4534,16 +4529,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -4696,29 +4681,29 @@ files = [ [[package]] name = "momento" -version = "1.9.2" +version = "1.10.1" description = "SDK for Momento" optional = true python-versions = ">=3.7,<4.0" files = [ - {file = "momento-1.9.2-py3-none-any.whl", hash = "sha256:8d6cb266c867e80ad9019e6ad1997c5e77be65ffb0182cfe9e55da20e06183ad"}, - {file = "momento-1.9.2.tar.gz", hash = "sha256:9f5e9401de04aec620f6acd4f8b61f0e76ba8aca18c72e2b3f1b9008dd8054d7"}, + {file = "momento-1.10.1-py3-none-any.whl", hash = "sha256:a97893bb3542499a429aa67c66d9330cd64b8f1a4f64c33dba4adc72cc3e9d01"}, + {file = "momento-1.10.1.tar.gz", hash = "sha256:c9cc21d06194371881c9749a20f3a6db68190427ff29dfd414aa94b90b27ab89"}, ] [package.dependencies] grpcio = ">=1.46.0,<2.0.0" -momento-wire-types = ">=0.75.0,<0.76.0" +momento-wire-types = ">=0.84.0,<0.85.0" pyjwt = ">=2.4.0,<3.0.0" [[package]] name = "momento-wire-types" -version = "0.75.0" +version = "0.84.2" description = "Momento Client Proto Generated Files" optional = true python-versions = ">=3.7,<4.0" files = [ - {file = "momento_wire_types-0.75.0-py3-none-any.whl", hash = "sha256:dce824584bde6d4896fbb4c010c146b68dac6d10bf6b77c5487edf43ff12ff75"}, - {file = "momento_wire_types-0.75.0.tar.gz", hash = "sha256:eb70d549bdcc28a0926b273737cd60f37eededbae3456db43e7dfa67ca83446f"}, + {file = "momento_wire_types-0.84.2-py3-none-any.whl", hash = "sha256:1b260556f1220bc3d5b3ffd7491a7d6de6654fef87e8a17f2b7cae53e9e30a23"}, + {file = "momento_wire_types-0.84.2.tar.gz", hash = "sha256:0560180af916cc588598ac0ee0fe04872ad30f428971c81e2d54b146a9f6d5fc"}, ] [package.dependencies] @@ -7643,7 +7628,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -7651,15 +7635,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -7676,7 +7653,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -7684,7 +7660,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -9015,14 +8990,6 @@ files = [ {file = "SQLAlchemy-2.0.21-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b69f1f754d92eb1cc6b50938359dead36b96a1dcf11a8670bff65fd9b21a4b09"}, {file = "SQLAlchemy-2.0.21-cp311-cp311-win32.whl", hash = "sha256:af520a730d523eab77d754f5cf44cc7dd7ad2d54907adeb3233177eeb22f271b"}, {file = "SQLAlchemy-2.0.21-cp311-cp311-win_amd64.whl", hash = "sha256:141675dae56522126986fa4ca713739d00ed3a6f08f3c2eb92c39c6dfec463ce"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:56628ca27aa17b5890391ded4e385bf0480209726f198799b7e980c6bd473bd7"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db726be58837fe5ac39859e0fa40baafe54c6d54c02aba1d47d25536170b690f"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7421c1bfdbb7214313919472307be650bd45c4dc2fcb317d64d078993de045b"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:632784f7a6f12cfa0e84bf2a5003b07660addccf5563c132cd23b7cc1d7371a9"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f6f7276cf26145a888f2182a98f204541b519d9ea358a65d82095d9c9e22f917"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2a1f7ffac934bc0ea717fa1596f938483fb8c402233f9b26679b4f7b38d6ab6e"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-win32.whl", hash = "sha256:bfece2f7cec502ec5f759bbc09ce711445372deeac3628f6fa1c16b7fb45b682"}, - {file = "SQLAlchemy-2.0.21-cp312-cp312-win_amd64.whl", hash = "sha256:526b869a0f4f000d8d8ee3409d0becca30ae73f494cbb48801da0129601f72c6"}, {file = "SQLAlchemy-2.0.21-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7614f1eab4336df7dd6bee05bc974f2b02c38d3d0c78060c5faa4cd1ca2af3b8"}, {file = "SQLAlchemy-2.0.21-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d59cb9e20d79686aa473e0302e4a82882d7118744d30bb1dfb62d3c47141b3ec"}, {file = "SQLAlchemy-2.0.21-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a95aa0672e3065d43c8aa80080cdd5cc40fe92dc873749e6c1cf23914c4b83af"}, @@ -10886,4 +10853,4 @@ text-helpers = ["chardet"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "e25dfb2dbe4090086ccaa9f483d75bed50fa65bb196dd5e63fd9f819d71d03f6" +content-hash = "de11d7f6257615dd61f579d5137d5a6b9ee5433e0d9b3cac3feb8a759ad70393" diff --git a/libs/langchain/pyproject.toml b/libs/langchain/pyproject.toml index 145966bd46..75503f96df 100644 --- a/libs/langchain/pyproject.toml +++ b/libs/langchain/pyproject.toml @@ -97,7 +97,7 @@ azure-ai-formrecognizer = {version = "^3.2.1", optional = true} azure-ai-vision = {version = "^0.11.1b1", optional = true} azure-cognitiveservices-speech = {version = "^1.28.0", optional = true} py-trello = {version = "^0.19.0", optional = true} -momento = {version = "^1.5.0", optional = true} +momento = {version = "^1.10.1", optional = true} bibtexparser = {version = "^1.4.0", optional = true} singlestoredb = {version = "^0.7.1", optional = true} pyspark = {version = "^3.4.0", optional = true} diff --git a/libs/langchain/tests/integration_tests/cache/test_momento_cache.py b/libs/langchain/tests/integration_tests/cache/test_momento_cache.py index 09517305c0..75d4a22921 100644 --- a/libs/langchain/tests/integration_tests/cache/test_momento_cache.py +++ b/libs/langchain/tests/integration_tests/cache/test_momento_cache.py @@ -29,7 +29,7 @@ def momento_cache() -> Iterator[MomentoCache]: cache_name = f"langchain-test-cache-{random_string()}" client = CacheClient( Configurations.Laptop.v1(), - CredentialProvider.from_environment_variable("MOMENTO_AUTH_TOKEN"), + CredentialProvider.from_environment_variable("MOMENTO_API_KEY"), default_ttl=timedelta(seconds=30), ) try: @@ -45,7 +45,7 @@ def test_invalid_ttl() -> None: client = CacheClient( Configurations.Laptop.v1(), - CredentialProvider.from_environment_variable("MOMENTO_AUTH_TOKEN"), + CredentialProvider.from_environment_variable("MOMENTO_API_KEY"), default_ttl=timedelta(seconds=30), ) with pytest.raises(ValueError): diff --git a/libs/langchain/tests/integration_tests/memory/test_momento.py b/libs/langchain/tests/integration_tests/memory/test_momento.py index 395232689b..0e20c4d183 100644 --- a/libs/langchain/tests/integration_tests/memory/test_momento.py +++ b/libs/langchain/tests/integration_tests/memory/test_momento.py @@ -27,7 +27,7 @@ def message_history() -> Iterator[MomentoChatMessageHistory]: cache_name = f"langchain-test-cache-{random_string()}" client = CacheClient( Configurations.Laptop.v1(), - CredentialProvider.from_environment_variable("MOMENTO_AUTH_TOKEN"), + CredentialProvider.from_environment_variable("MOMENTO_API_KEY"), default_ttl=timedelta(seconds=30), ) try: diff --git a/libs/langchain/tests/integration_tests/vectorstores/test_momento_vector_index.py b/libs/langchain/tests/integration_tests/vectorstores/test_momento_vector_index.py new file mode 100644 index 0000000000..59e70aaa80 --- /dev/null +++ b/libs/langchain/tests/integration_tests/vectorstores/test_momento_vector_index.py @@ -0,0 +1,164 @@ +import time +import uuid +from typing import Iterator, List + +import pytest + +from langchain.docstore.document import Document +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import MomentoVectorIndex + +API_KEY_ENV_VAR = "MOMENTO_API_KEY" + + +def random_string() -> str: + return str(uuid.uuid4()) + + +@pytest.fixture(scope="function") +def random_index_name() -> str: + return f"langchain-test-index-{random_string()}" + + +def wait() -> None: + time.sleep(1) + + +@pytest.fixture(scope="function") +def vector_store( + embedding_openai: OpenAIEmbeddings, random_index_name: str +) -> Iterator[MomentoVectorIndex]: + from momento import ( + CredentialProvider, + PreviewVectorIndexClient, + VectorIndexConfigurations, + ) + + vector_store = None + try: + client = PreviewVectorIndexClient( + VectorIndexConfigurations.Default.latest(), + credential_provider=CredentialProvider.from_environment_variable( + API_KEY_ENV_VAR + ), + ) + vector_store = MomentoVectorIndex( + embedding=embedding_openai, + client=client, + index_name=random_index_name, + ) + yield vector_store + finally: + if vector_store is not None: + vector_store._client.delete_index(random_index_name) + + +def test_from_texts( + random_index_name: str, embedding_openai: OpenAIEmbeddings, texts: List[str] +) -> None: + from momento import ( + CredentialProvider, + VectorIndexConfigurations, + ) + + random_text = random_string() + random_document = f"Hello world {random_text} goodbye world!" + texts.insert(0, random_document) + + vector_store = None + try: + vector_store = MomentoVectorIndex.from_texts( + texts=texts, + embedding=embedding_openai, + index_name=random_index_name, + configuration=VectorIndexConfigurations.Default.latest(), + credential_provider=CredentialProvider.from_environment_variable( + "MOMENTO_API_KEY" + ), + ) + wait() + + documents = vector_store.similarity_search(query=random_text, k=1) + assert documents == [Document(page_content=random_document)] + finally: + if vector_store is not None: + vector_store._client.delete_index(random_index_name) + + +def test_from_texts_with_metadatas( + random_index_name: str, embedding_openai: OpenAIEmbeddings, texts: List[str] +) -> None: + """Test end to end construction and search.""" + from momento import ( + CredentialProvider, + VectorIndexConfigurations, + ) + + random_text = random_string() + random_document = f"Hello world {random_text} goodbye world!" + texts.insert(0, random_document) + metadatas = [{"page": f"{i}", "source": "user"} for i in range(len(texts))] + + vector_store = None + try: + vector_store = MomentoVectorIndex.from_texts( + texts=texts, + embedding=embedding_openai, + index_name=random_index_name, + metadatas=metadatas, + configuration=VectorIndexConfigurations.Default.latest(), + credential_provider=CredentialProvider.from_environment_variable( + API_KEY_ENV_VAR + ), + ) + + wait() + documents = vector_store.similarity_search(query=random_text, k=1) + assert documents == [ + Document( + page_content=random_document, metadata={"page": "0", "source": "user"} + ) + ] + finally: + if vector_store is not None: + vector_store._client.delete_index(random_index_name) + + +def test_from_texts_with_scores(vector_store: MomentoVectorIndex) -> None: + # """Test end to end construction and search with scores and IDs.""" + texts = ["apple", "orange", "hammer"] + metadatas = [{"page": f"{i}"} for i in range(len(texts))] + + vector_store.add_texts(texts, metadatas) + wait() + search_results = vector_store.similarity_search_with_score("apple", k=3) + docs = [o[0] for o in search_results] + scores = [o[1] for o in search_results] + + assert docs == [ + Document(page_content="apple", metadata={"page": "0"}), + Document(page_content="orange", metadata={"page": "1"}), + Document(page_content="hammer", metadata={"page": "2"}), + ] + assert scores[0] > scores[1] > scores[2] + + +def test_add_documents_with_ids(vector_store: MomentoVectorIndex) -> None: + """Test end to end construction and search with scores and IDs.""" + from momento.responses.vector_index import Search + + texts = ["apple", "orange", "hammer"] + ids = [random_string() for _ in range(len(texts))] + metadatas = [{"page": f"{i}"} for i in range(len(texts))] + + # Add texts with metadata and ids + stored_ids = vector_store.add_texts(texts, metadatas, ids=ids) + assert stored_ids == ids + wait() + + # Verify that the ids are in the index + response = vector_store._client.search( + vector_store.index_name, vector_store.embeddings.embed_query("apple") + ) + assert isinstance(response, Search.Success) + assert [hit.id for hit in response.hits] == ids