diff --git a/docs/integrations/sklearn.md b/docs/integrations/sklearn.md new file mode 100644 index 00000000..76076232 --- /dev/null +++ b/docs/integrations/sklearn.md @@ -0,0 +1,23 @@ +# scikit-learn + +This page covers how to use the scikit-learn package within LangChain. +It is broken into two parts: installation and setup, and then references to specific scikit-learn wrappers. + +## Installation and Setup + +- Install the Python package with `pip install scikit-learn` + +## Wrappers + +### VectorStore + +`SKLearnVectorStore` provides a simple wrapper around the nearest neighbor implementation in the +scikit-learn package, allowing you to use it as a vectorstore. + +To import this vectorstore: + +```python +from langchain.vectorstores import SKLearnVectorStore +``` + +For a more detailed walkthrough of the SKLearnVectorStore wrapper, see [this notebook](../modules/indexes/vectorstores/examples/sklearn.ipynb). diff --git a/docs/modules/indexes/vectorstores/examples/sklearn.ipynb b/docs/modules/indexes/vectorstores/examples/sklearn.ipynb new file mode 100644 index 00000000..dc31ea01 --- /dev/null +++ b/docs/modules/indexes/vectorstores/examples/sklearn.ipynb @@ -0,0 +1,233 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SKLearnVectorStore\n", + "\n", + "[scikit-learn](https://scikit-learn.org/stable/) is an open source collection of machine learning algorithms, including some implementations of the [k nearest neighbors](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestNeighbors.html). `SKLearnVectorStore` wraps this implementation and adds the possibility to persist the vector store in json, bson (binary json) or Apache Parquet format.\n", + "\n", + "This notebook shows how to use the `SKLearnVectorStore` vector database." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install scikit-learn\n", + "\n", + "# # if you plan to use bson serialization, install also:\n", + "# %pip install bson\n", + "\n", + "# # if you plan to use parquet serialization, install also:\n", + "%pip install pandas pyarrow" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use OpenAI embeddings, you will need an OpenAI key. You can get one at https://platform.openai.com/account/api-keys or feel free to use any other embeddings." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "os.environ['OPENAI_API_KEY'] = getpass('Enter your OpenAI key:')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic usage\n", + "\n", + "### Load a sample document corpus" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.embeddings.openai import OpenAIEmbeddings\n", + "from langchain.text_splitter import CharacterTextSplitter\n", + "from langchain.vectorstores import SKLearnVectorStore\n", + "from langchain.document_loaders import TextLoader\n", + "\n", + "loader = TextLoader('../../../state_of_the_union.txt')\n", + "documents = loader.load()\n", + "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n", + "docs = text_splitter.split_documents(documents)\n", + "embeddings = OpenAIEmbeddings()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create the SKLearnVectorStore, index the document corpus and run a sample query" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n", + "\n", + "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n", + "\n", + "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n", + "\n", + "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.\n" + ] + } + ], + "source": [ + "import tempfile\n", + "persist_path = os.path.join(tempfile.gettempdir(), 'union.parquet')\n", + "\n", + "vector_store = SKLearnVectorStore.from_documents(\n", + " documents=docs, \n", + " embedding=embeddings,\n", + " persist_path=persist_path, # persist_path and serializer are optional\n", + " serializer='parquet'\n", + ")\n", + "\n", + "query = \"What did the president say about Ketanji Brown Jackson\"\n", + "docs = vector_store.similarity_search(query)\n", + "print(docs[0].page_content)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Saving and loading a vector store" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Vector store was persisted to /var/folders/6r/wc15p6m13nl_nl_n_xfqpc5c0000gp/T/union.parquet\n" + ] + } + ], + "source": [ + "vector_store.persist()\n", + "print('Vector store was persisted to', persist_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A new instance of vector store was loaded from /var/folders/6r/wc15p6m13nl_nl_n_xfqpc5c0000gp/T/union.parquet\n" + ] + } + ], + "source": [ + "vector_store2 = SKLearnVectorStore(\n", + " embedding=embeddings,\n", + " persist_path=persist_path,\n", + " serializer='parquet'\n", + ")\n", + "print('A new instance of vector store was loaded from', persist_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n", + "\n", + "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n", + "\n", + "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n", + "\n", + "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence.\n" + ] + } + ], + "source": [ + "docs = vector_store2.similarity_search(query)\n", + "print(docs[0].page_content)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean-up" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(persist_path)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sofia", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/langchain/vectorstores/__init__.py b/langchain/vectorstores/__init__.py index 6807800a..23fe02ac 100644 --- a/langchain/vectorstores/__init__.py +++ b/langchain/vectorstores/__init__.py @@ -15,6 +15,7 @@ from langchain.vectorstores.opensearch_vector_search import OpenSearchVectorSear from langchain.vectorstores.pinecone import Pinecone from langchain.vectorstores.qdrant import Qdrant from langchain.vectorstores.redis import Redis +from langchain.vectorstores.sklearn import SKLearnVectorStore from langchain.vectorstores.supabase import SupabaseVectorStore from langchain.vectorstores.tair import Tair from langchain.vectorstores.typesense import Typesense @@ -39,6 +40,7 @@ __all__ = [ "Annoy", "MyScale", "MyScaleSettings", + "SKLearnVectorStore", "SupabaseVectorStore", "AnalyticDB", "Vectara", diff --git a/langchain/vectorstores/sklearn.py b/langchain/vectorstores/sklearn.py new file mode 100644 index 00000000..153a523f --- /dev/null +++ b/langchain/vectorstores/sklearn.py @@ -0,0 +1,271 @@ +""" Wrapper around scikit-learn NearestNeighbors implementation. + +The vector store can be persisted in json, bson or parquet format. +""" + +import importlib +import json +import math +import os +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, List, Literal, Optional, Tuple, Type +from uuid import uuid4 + +from langchain.docstore.document import Document +from langchain.embeddings.base import Embeddings +from langchain.vectorstores.base import VectorStore + + +def guard_import( + module_name: str, *, pip_name: Optional[str] = None, package: Optional[str] = None +) -> Any: + """Dynamically imports a module and raises a helpful exception if the module is not + installed.""" + try: + module = importlib.import_module(module_name, package) + except ImportError: + raise ImportError( + f"Could not import {module_name} python package. " + f"Please install it with `pip install {pip_name or module_name}`." + ) + return module + + +class BaseSerializer(ABC): + """Abstract base class for saving and loading data.""" + + def __init__(self, persist_path: str) -> None: + self.persist_path = persist_path + + @classmethod + @abstractmethod + def extension(cls) -> str: + """The file extension suggested by this serializer (without dot).""" + + @abstractmethod + def save(self, data: Any) -> None: + """Saves the data to the persist_path""" + + @abstractmethod + def load(self) -> Any: + """Loads the data from the persist_path""" + + +class JsonSerializer(BaseSerializer): + """Serializes data in json using the json package from python standard library.""" + + @classmethod + def extension(cls) -> str: + return "json" + + def save(self, data: Any) -> None: + with open(self.persist_path, "w") as fp: + json.dump(data, fp) + + def load(self) -> Any: + with open(self.persist_path, "r") as fp: + return json.load(fp) + + +class BsonSerializer(BaseSerializer): + """Serializes data in binary json using the bson python package.""" + + def __init__(self, persist_path: str) -> None: + super().__init__(persist_path) + self.bson = guard_import("bson") + + @classmethod + def extension(cls) -> str: + return "bson" + + def save(self, data: Any) -> None: + with open(self.persist_path, "wb") as fp: + fp.write(self.bson.dumps(data)) + + def load(self) -> Any: + with open(self.persist_path, "rb") as fp: + return self.bson.loads(fp.read()) + + +class ParquetSerializer(BaseSerializer): + """Serializes data in Apache Parquet format using the pyarrow package.""" + + def __init__(self, persist_path: str) -> None: + super().__init__(persist_path) + self.pd = guard_import("pandas") + self.pa = guard_import("pyarrow") + self.pq = guard_import("pyarrow.parquet") + + @classmethod + def extension(cls) -> str: + return "parquet" + + def save(self, data: Any) -> None: + df = self.pd.DataFrame(data) + table = self.pa.Table.from_pandas(df) + if os.path.exists(self.persist_path): + backup_path = str(self.persist_path) + "-backup" + os.rename(self.persist_path, backup_path) + try: + self.pq.write_table(table, self.persist_path) + except Exception as exc: + os.rename(backup_path, self.persist_path) + raise exc + else: + os.remove(backup_path) + else: + self.pq.write_table(table, self.persist_path) + + def load(self) -> Any: + table = self.pq.read_table(self.persist_path) + df = table.to_pandas() + return {col: series.tolist() for col, series in df.items()} + + +SERIALIZER_MAP: Dict[str, Type[BaseSerializer]] = { + "json": JsonSerializer, + "bson": BsonSerializer, + "parquet": ParquetSerializer, +} + + +class SKLearnVectorStoreException(RuntimeError): + pass + + +class SKLearnVectorStore(VectorStore): + """A simple in-memory vector store based on the scikit-learn library + NearestNeighbors implementation.""" + + def __init__( + self, + embedding: Embeddings, + *, + persist_path: Optional[str] = None, + serializer: Literal["json", "bson", "parquet"] = "json", + metric: str = "cosine", + **kwargs: Any, + ) -> None: + np = guard_import("numpy") + sklearn_neighbors = guard_import("sklearn.neighbors", pip_name="scikit-learn") + + # non-persistent properties + self._np = np + self._neighbors = sklearn_neighbors.NearestNeighbors(metric=metric, **kwargs) + self._neighbors_fitted = False + self._embedding_function = embedding + self._persist_path = persist_path + self._serializer: Optional[BaseSerializer] = None + if self._persist_path is not None: + serializer_cls = SERIALIZER_MAP[serializer] + self._serializer = serializer_cls(persist_path=self._persist_path) + + # data properties + self._embeddings: List[List[float]] = [] + self._texts: List[str] = [] + self._metadatas: List[dict] = [] + self._ids: List[str] = [] + + # cache properties + self._embeddings_np: Any = np.asarray([]) + + if self._persist_path is not None and os.path.isfile(self._persist_path): + self._load() + + def persist(self) -> None: + if self._serializer is None: + raise SKLearnVectorStoreException( + "You must specify a persist_path on creation to persist the " + "collection." + ) + data = { + "ids": self._ids, + "texts": self._texts, + "metadatas": self._metadatas, + "embeddings": self._embeddings, + } + self._serializer.save(data) + + def _load(self) -> None: + if self._serializer is None: + raise SKLearnVectorStoreException( + "You must specify a persist_path on creation to load the " "collection." + ) + data = self._serializer.load() + self._embeddings = data["embeddings"] + self._texts = data["texts"] + self._metadatas = data["metadatas"] + self._ids = data["ids"] + self._update_neighbors() + + def add_texts( + self, + texts: Iterable[str], + metadatas: Optional[List[dict]] = None, + ids: Optional[List[str]] = None, + **kwargs: Any, + ) -> List[str]: + _texts = list(texts) + _ids = ids or [str(uuid4()) for _ in _texts] + self._texts.extend(_texts) + self._embeddings.extend(self._embedding_function.embed_documents(_texts)) + self._metadatas.extend(metadatas or ([{}] * len(_texts))) + self._ids.extend(_ids) + self._update_neighbors() + return _ids + + def _update_neighbors(self) -> None: + if len(self._embeddings) == 0: + raise SKLearnVectorStoreException( + "No data was added to SKLearnVectorStore." + ) + self._embeddings_np = self._np.asarray(self._embeddings) + self._neighbors.fit(self._embeddings_np) + self._neighbors_fitted = True + + def similarity_search_with_score( + self, query: str, *, k: int = 4, **kwargs: Any + ) -> List[Tuple[Document, float]]: + if not self._neighbors_fitted: + raise SKLearnVectorStoreException( + "No data was added to SKLearnVectorStore." + ) + query_embedding = self._embedding_function.embed_query(query) + neigh_dists, neigh_idxs = self._neighbors.kneighbors( + [query_embedding], n_neighbors=k + ) + res = [] + for idx, dist in zip(neigh_idxs[0], neigh_dists[0]): + _idx = int(idx) + metadata = {"id": self._ids[_idx], **self._metadatas[_idx]} + doc = Document(page_content=self._texts[_idx], metadata=metadata) + res.append((doc, dist)) + return res + + def similarity_search( + self, query: str, k: int = 4, **kwargs: Any + ) -> List[Document]: + docs_scores = self.similarity_search_with_score(query, k=k, **kwargs) + return [doc for doc, _ in docs_scores] + + def _similarity_search_with_relevance_scores( + self, query: str, k: int = 4, **kwargs: Any + ) -> List[Tuple[Document, float]]: + docs_dists = self.similarity_search_with_score(query=query, k=k, **kwargs) + docs, dists = zip(*docs_dists) + scores = [1 / math.exp(dist) for dist in dists] + return list(zip(list(docs), scores)) + + @classmethod + def from_texts( + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + ids: Optional[List[str]] = None, + persist_path: Optional[str] = None, + **kwargs: Any, + ) -> "SKLearnVectorStore": + vs = SKLearnVectorStore(embedding, persist_path=persist_path, **kwargs) + vs.add_texts(texts, metadatas=metadatas, ids=ids) + return vs diff --git a/tests/unit_tests/vectorstores/test_sklearn.py b/tests/unit_tests/vectorstores/test_sklearn.py new file mode 100644 index 00000000..b1f423c7 --- /dev/null +++ b/tests/unit_tests/vectorstores/test_sklearn.py @@ -0,0 +1,76 @@ +"""Test SKLearnVectorStore functionality.""" +from pathlib import Path + +import pytest + +from langchain.vectorstores import SKLearnVectorStore +from tests.integration_tests.vectorstores.fake_embeddings import FakeEmbeddings + + +@pytest.mark.requires("numpy", "sklearn") +def test_sklearn() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + docsearch = SKLearnVectorStore.from_texts(texts, embedding=FakeEmbeddings()) + output = docsearch.similarity_search("foo", k=1) + assert len(output) == 1 + assert output[0].page_content == "foo" + + +@pytest.mark.requires("numpy", "sklearn") +def test_sklearn_with_metadatas() -> None: + """Test end to end construction and search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = SKLearnVectorStore.from_texts( + texts, + embedding=FakeEmbeddings(), + metadatas=metadatas, + ) + output = docsearch.similarity_search("foo", k=1) + assert output[0].metadata["page"] == "0" + + +@pytest.mark.requires("numpy", "sklearn") +def test_sklearn_with_metadatas_with_scores() -> None: + """Test end to end construction and scored search.""" + texts = ["foo", "bar", "baz"] + metadatas = [{"page": str(i)} for i in range(len(texts))] + docsearch = SKLearnVectorStore.from_texts( + texts, + embedding=FakeEmbeddings(), + metadatas=metadatas, + ) + output = docsearch.similarity_search_with_relevance_scores("foo", k=1) + assert len(output) == 1 + doc, score = output[0] + assert doc.page_content == "foo" + assert doc.metadata["page"] == "0" + assert score == 1 + + +@pytest.mark.requires("numpy", "sklearn") +def test_sklearn_with_persistence(tmpdir: Path) -> None: + """Test end to end construction and search, with persistence.""" + persist_path = tmpdir / "foo.parquet" + texts = ["foo", "bar", "baz"] + docsearch = SKLearnVectorStore.from_texts( + texts, + FakeEmbeddings(), + persist_path=str(persist_path), + serializer="json", + ) + + output = docsearch.similarity_search("foo", k=1) + assert len(output) == 1 + assert output[0].page_content == "foo" + + docsearch.persist() + + # Get a new VectorStore from the persisted directory + docsearch = SKLearnVectorStore( + embedding=FakeEmbeddings(), persist_path=str(persist_path), serializer="json" + ) + output = docsearch.similarity_search("foo", k=1) + assert len(output) == 1 + assert output[0].page_content == "foo"