mirror of https://github.com/hwchase17/langchain
Add embeddings for AwaEmbedding (#8353)
- Description: Adds AwaEmbeddings class for embeddings, which provides users with a convenient way to do fine-tuning, as well as the potential need for multimodality - Tag maintainer: @baskaryan Create `Awa.ipynb`: an example notebook for AwaEmbeddings class Modify `embeddings/__init__.py`: Import the class Create `embeddings/awa.py`: The embedding class Create `embeddings/test_awa.py`: The test file. --------- Co-authored-by: taozhiwang <taozhiwa@gmail.com>pull/8038/head
parent
ba4e82bb47
commit
594f195e54
@ -0,0 +1,109 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "b14a24db",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# AwaEmbedding\n",
|
||||||
|
"\n",
|
||||||
|
"This notebook explains how to use AwaEmbedding, which is included in [awadb](https://github.com/awa-ai/awadb), to embedding texts in langchain."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"id": "0ab948fc",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# pip install awadb"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "67c637ca",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## import the library"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"id": "5709b030",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from langchain.embeddings import AwaEmbeddings"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"id": "1756b1ba",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"Embedding = AwaEmbeddings()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "4a2a098d",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Set embedding model\n",
|
||||||
|
"Users can use `Embedding.set_model()` to specify the embedding model. \\\n",
|
||||||
|
"The input of this function is a string which represents the model's name. \\\n",
|
||||||
|
"The list of currently supported models can be obtained [here](https://github.com/awa-ai/awadb) \\ \\ \n",
|
||||||
|
"\n",
|
||||||
|
"The **default model** is `all-mpnet-base-v2`, it can be used without setting."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"id": "584b9af5",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"text = \"our embedding test\"\n",
|
||||||
|
"\n",
|
||||||
|
"Embedding.set_model(\"all-mpnet-base-v2\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"id": "be18b873",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"res_query = Embedding.embed_query(\"The test information\")\n",
|
||||||
|
"res_document = Embedding.embed_documents([\"test1\", \"another test\"])"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.11.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
@ -0,0 +1,56 @@
|
|||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from pydantic import BaseModel, root_validator
|
||||||
|
|
||||||
|
from langchain.embeddings.base import Embeddings
|
||||||
|
|
||||||
|
|
||||||
|
class AwaEmbeddings(BaseModel, Embeddings):
|
||||||
|
client: Any #: :meta private:
|
||||||
|
model: str = "all-mpnet-base-v2"
|
||||||
|
|
||||||
|
@root_validator()
|
||||||
|
def validate_environment(cls, values: Dict) -> Dict:
|
||||||
|
"""Validate that awadb library is installed."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
from awadb import AwaEmbedding
|
||||||
|
except ImportError as exc:
|
||||||
|
raise ImportError(
|
||||||
|
"Could not import awadb library. "
|
||||||
|
"Please install it with `pip install awadb`"
|
||||||
|
) from exc
|
||||||
|
values["client"] = AwaEmbedding()
|
||||||
|
return values
|
||||||
|
|
||||||
|
def set_model(self, model_name: str) -> None:
|
||||||
|
"""Set the model used for embedding.
|
||||||
|
The default model used is all-mpnet-base-v2
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model_name: A string which represents the name of model.
|
||||||
|
"""
|
||||||
|
self.model = model_name
|
||||||
|
self.client.model_name = model_name
|
||||||
|
|
||||||
|
def embed_documents(self, texts: List[str]) -> List[List[float]]:
|
||||||
|
"""Embed a list of documents using AwaEmbedding.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
texts: The list of texts need to be embedded
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of embeddings, one for each text.
|
||||||
|
"""
|
||||||
|
return self.client.EmbeddingBatch(texts)
|
||||||
|
|
||||||
|
def embed_query(self, text: str) -> List[float]:
|
||||||
|
"""Compute query embeddings using AwaEmbedding.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: The text to embed.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Embeddings for the text.
|
||||||
|
"""
|
||||||
|
return self.client.Embedding(text)
|
@ -0,0 +1,19 @@
|
|||||||
|
"""Test Awa Embedding"""
|
||||||
|
from langchain.embeddings.awa import AwaEmbeddings
|
||||||
|
|
||||||
|
|
||||||
|
def test_awa_embedding_documents() -> None:
|
||||||
|
"""Test Awa embeddings for documents."""
|
||||||
|
documents = ["foo bar", "test document"]
|
||||||
|
embedding = AwaEmbeddings()
|
||||||
|
output = embedding.embed_documents(documents)
|
||||||
|
assert len(output) == 2
|
||||||
|
assert len(output[0]) == 768
|
||||||
|
|
||||||
|
|
||||||
|
def test_awa_embedding_query() -> None:
|
||||||
|
"""Test Awa embeddings for query."""
|
||||||
|
document = "foo bar"
|
||||||
|
embedding = AwaEmbeddings()
|
||||||
|
output = embedding.embed_query(document)
|
||||||
|
assert len(output) == 768
|
Loading…
Reference in New Issue