langchain/docs/extras/integrations/vectorstores/vearch.ipynb
zhanghexian 62fa2bc518
Add Vearch vectorstore (#9846)
---------

Co-authored-by: zhanghexian1 <zhanghexian1@jd.com>
Co-authored-by: Bagatur <baskaryan@gmail.com>
Co-authored-by: Harrison Chase <hw.chase.17@gmail.com>
2023-09-08 16:51:14 -07:00

414 lines
21 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/export/anaconda3/envs/langchainGLM6B/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n",
"INFO 2023-08-28 18:26:07,485-1d: \n",
"loading model config\n",
"llm device: cuda\n",
"embedding device: cuda\n",
"dir: /data/zhx/zhx/langchain-ChatGLM_new\n",
"flagging username: e2fc35b8e87c4de18d692e951a5f7c46\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"True\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Loading checkpoint shards: 100%|██████████| 7/7 [00:06<00:00, 1.01it/s]\n"
]
}
],
"source": [
"\n",
"import os, sys, torch\n",
"from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModel\n",
"from langchain import HuggingFacePipeline, ConversationChain\n",
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain.vectorstores.vearch import VearchDb\n",
"from langchain.document_loaders import TextLoader\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import RetrievalQA\n",
"from langchain.embeddings.huggingface import HuggingFaceEmbeddings\n",
"\n",
"# your local model path\n",
"model_path =\"/data/zhx/zhx/langchain-ChatGLM_new/chatglm2-6b\" \n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)\n",
"model = AutoModel.from_pretrained(model_path, trust_remote_code=True).half().cuda(0)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Human: 你好!\n",
"ChatGLM:你好👋!我是人工智能助手 ChatGLM2-6B很高兴见到你欢迎问我任何问题。\n",
"\n",
"Human: 你知道凌波微步吗,你知道都有谁学会了吗?\n",
"ChatGLM:凌波微步是一种步伐,最早出自于《倚天屠龙记》。在小说中,灭绝师太曾因与练习凌波微步的杨过的恩怨纠葛,而留下了一部经书,内容是记载凌波微步的起源和作用。后来,凌波微步便成为杨过和小龙女的感情象征。在现实生活中,凌波微步是一句口号,是清华大学学生社团“模型社”的社训。\n",
"\n"
]
}
],
"source": [
"query = \"你好!\"\n",
"response, history = model.chat(tokenizer, query, history=[])\n",
"print(f\"Human: {query}\\nChatGLM:{response}\\n\")\n",
"query = \"你知道凌波微步吗,你知道都有谁学会了吗?\"\n",
"response, history = model.chat(tokenizer, query, history=history)\n",
"print(f\"Human: {query}\\nChatGLM:{response}\\n\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO 2023-08-28 18:27:36,037-1d: Load pretrained SentenceTransformer: /data/zhx/zhx/langchain-ChatGLM_new/text2vec/text2vec-large-chinese\n",
"WARNING 2023-08-28 18:27:36,038-1d: No sentence-transformers model found with name /data/zhx/zhx/langchain-ChatGLM_new/text2vec/text2vec-large-chinese. Creating a new one with MEAN pooling.\n",
"INFO 2023-08-28 18:27:38,936-1d: Use pytorch device: cuda\n"
]
}
],
"source": [
"# Add your local knowledge files\n",
"file_path = \"/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/天龙八部/lingboweibu.txt\"#Your local file path\"\n",
"loader = TextLoader(file_path,encoding=\"utf-8\")\n",
"documents = loader.load()\n",
"\n",
"# split text into sentences and embedding the sentences\n",
"text_splitter = RecursiveCharacterTextSplitter(\n",
" chunk_size=500, chunk_overlap=100)\n",
"texts = text_splitter.split_documents(documents)\n",
"\n",
"#your model path\n",
"embedding_path = '/data/zhx/zhx/langchain-ChatGLM_new/text2vec/text2vec-large-chinese'\n",
"embeddings = HuggingFaceEmbeddings(model_name=embedding_path)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Batches: 100%|██████████| 1/1 [00:00<00:00, 4.56it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"['7aae36236f784105a0004d8ff3c7c3ad', '7e495d4e5962497db2080e84d52e75ed', '9a640124fc324a8abb0eaa31acb638b7']\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"#first add your document into vearch vectorstore\n",
"vearch_db = VearchDb.from_documents(texts,embeddings,table_name=\"your_table_name\",metadata_path=\"/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/your_table_name\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Batches: 100%|██████████| 1/1 [00:00<00:00, 22.49it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"####################第1段相关文档####################\n",
"\n",
"午饭过后,段誉又练“凌波微步”,走一步,吸一口气,走第二步时将气呼出,六十四卦走完,四肢全无麻痹之感,料想呼吸顺畅,便无害处。第二次再走时连走两步吸一口气,再走两步始行呼出。这“凌波微步”是以动功修习内功,脚步踏遍六十四卦一个周天,内息自然而然地也转了一个周天。因此他每走一遍,内力便有一分进益。\n",
"\n",
"这般练了几天,“凌波微步”已走得颇为纯熟,不须再数呼吸,纵然疾行,气息也已无所窒滞。心意既畅,跨步时渐渐想到《洛神赋》中那些与“凌波微步”有关的句子:“仿佛兮若轻云之蔽月,飘飘兮若流风之回雪”,“竦轻躯以鹤立,若将飞而未翔”,“体迅飞凫,飘忽若神”,“动无常则,若危若安。进止难期,若往若还”。\n",
"\n",
"\n",
"\n",
"百度简介\n",
"\n",
"凌波微步是「逍遥派」独门轻功身法,精妙异常。\n",
"\n",
"凌波微步乃是一门极上乘的轻功,所以列于卷轴之末,以易经八八六十四卦为基础,使用者按特定顺序踏着卦象方位行进,从第一步到最后一步正好行走一个大圈。此步法精妙异常,原是要待人练成「北冥神功」,吸人内力,自身内力已【颇为深厚】之后再练。\n",
"\n",
"####################第2段相关文档####################\n",
"\n",
"《天龙八部》第五回 微步縠纹生\n",
"\n",
"卷轴中此外诸种经脉修习之法甚多,皆是取人内力的法门,段誉虽自语宽解,总觉习之有违本性,单是贪多务得,便非好事,当下暂不理会。\n",
"\n",
"卷到卷轴末端,又见到了“凌波微步”那四字,登时便想起《洛神赋》中那些句子来:“凌波微步,罗袜生尘……转眄流精,光润玉颜。含辞未吐,气若幽兰。华容婀娜,令我忘餐。”曹子建那些千古名句,在脑海中缓缓流过:“秾纤得衷,修短合度,肩若削成,腰如约素。延颈秀项,皓质呈露。芳泽无加,铅华弗御。云髻峨峨,修眉连娟。丹唇外朗,皓齿内鲜。明眸善睐,靥辅承权。瑰姿艳逸,仪静体闲。柔情绰态,媚于语言……”这些句子用在木婉清身上,“这话倒也有理”;但如用之于神仙姊姊,只怕更为适合。想到神仙姊姊的姿容体态,“皎若太阳升朝霞,灼若芙蓉出绿波”,但觉依她吩咐行事,实为人生至乐,心想:“我先来练这‘凌波微步’,此乃逃命之妙法,非害人之手段也,练之有百利而无一害。”\n",
"\n",
"####################第3段相关文档####################\n",
"\n",
"《天龙八部》第二回 玉壁月华明\n",
"\n",
"再展帛卷,长卷上源源皆是裸女画像,或立或卧,或现前胸,或见后背。人像的面容都是一般,但或喜或愁,或含情凝眸,或轻嗔薄怒,神情各异。一共有三十六幅图像,每幅像上均有颜色细线,注明穴道部位及练功法诀。\n",
"\n",
"帛卷尽处题着“凌波微步”四字,其后绘的是无数足印,注明“妇妹”、“无妄”等等字样,尽是《易经》中的方位。段誉前几日还正全心全意地钻研《易经》,一见到这些名称,登时精神大振,便似遇到故交良友一般。只见足印密密麻麻,不知有几千百个,自一个足印至另一个足印均有绿线贯串,线上绘有箭头,最后写着一行字道:“步法神妙,保身避敌,待积内力,再取敌命。”\n",
"\n",
"段誉心道:“神仙姊姊所遗的步法,必定精妙之极,遇到强敌时脱身逃走,那就很好,‘再取敌命’也就不必了。”\n",
"卷好帛卷,对之作了两个揖,珍而重之地揣入怀中,转身对那玉像道:“神仙姊姊,你吩咐我朝午晚三次练功,段誉不敢有违。今后我对人加倍客气,别人不会来打我,我自然也不会去吸他内力。你这套‘凌波微步’我更要用心练熟,眼见不对,立刻溜之大吉,就吸不到他内力了。”至于“杀尽我逍遥派弟子”一节,却想也不敢去想。\n",
"\n",
"********ChatGLM:凌波微步是一种轻功身法,属于逍遥派独门轻功。它以《易经》中的六十四卦为基础,按照特定顺序踏着卦象方位行进,从第一步到最后一步正好行走一个大圈。凌波微步精妙异常,可以让人内力相助,自身内力颇为深厚之后再练。《天龙八部》第五回中有描述。\n",
"\n"
]
}
],
"source": [
"\n",
"res=vearch_db.similarity_search(query, 3)\n",
"query = \"你知道凌波微步吗,你知道都有谁会凌波微步?\"\n",
"for idx,tmp in enumerate(res): \n",
" print(f\"{'#'*20}第{idx+1}段相关文档{'#'*20}\\n\\n{tmp.page_content}\\n\")\n",
"\n",
"# combine your local knowleadge and query \n",
"context = \"\".join([tmp.page_content for tmp in res])\n",
"new_query = f\"基于以下信息,尽可能准确的来回答用户的问题。背景信息:\\n {context} \\n 回答用户这个问题:{query}\\n\\n\"\n",
"response, history = model.chat(tokenizer, new_query, history=[])\n",
"print(f\"********ChatGLM:{response}\\n\")\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Human: 你知道vearch是什么吗?\n",
"ChatGLM:是的,我知道 Vearch。Vearch 是一种矩阵分解 technique用于将矩阵分解为若干个不可约矩阵的乘积。它是由 Linus Torvalds 开发的,旨在提高 Linux 内核中矩阵操作的性能。\n",
"\n",
"Vearch 可以通过使用特殊的操作来对矩阵进行操作,从而避免了使用昂贵的矩阵操作库。它也被广泛用于其他操作系统中,如 FreeBSD 和 Solaris。\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Batches: 100%|██████████| 1/1 [00:00<00:00, 31.59it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"['04bc84fff5074b7b8990441e92e6df07', 'e221906153bb4e03bc7095dadea144de', '126034ba51934093920d8732860f340b']\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
},
{
"data": {
"text/plain": [
"['04bc84fff5074b7b8990441e92e6df07',\n",
" 'e221906153bb4e03bc7095dadea144de',\n",
" '126034ba51934093920d8732860f340b']"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"query = \"你知道vearch是什么吗?\"\n",
"response, history = model.chat(tokenizer, query, history=history)\n",
"print(f\"Human: {query}\\nChatGLM:{response}\\n\")\n",
"\n",
"\n",
"vearch_info = [\"Vearch 是一款存储大语言模型数据的向量数据库用于存储和快速搜索模型embedding后的向量可用于基于个人知识库的大模型应用\",\n",
" \"Vearch 支持OpenAI, Llama, ChatGLM等模型以及LangChain库\",\n",
" \"vearch 是基于C语言,go语言开发的并提供python接口可以直接通过pip安装\"]\n",
"vearch_source=[{'source': '/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/tlbb/three_body.txt'},{'source': '/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/tlbb/three_body.txt'},{'source': '/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/tlbb/three_body.txt'}]\n",
"vearch_db.add_texts(vearch_info,vearch_source)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Batches: 100%|██████████| 1/1 [00:00<00:00, 25.57it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"####################第1段相关文档####################\n",
"\n",
"Vearch 是一款存储大语言模型数据的向量数据库用于存储和快速搜索模型embedding后的向量可用于基于个人知识库的大模型应用\n",
"\n",
"####################第2段相关文档####################\n",
"\n",
"Vearch 支持OpenAI, Llama, ChatGLM等模型以及LangChain库\n",
"\n",
"####################第3段相关文档####################\n",
"\n",
"vearch 是基于C语言,go语言开发的并提供python接口可以直接通过pip安装\n",
"\n",
"***************ChatGLM:是的Varch是一个向量数据库旨在存储和快速搜索模型embedding后的向量。它支持OpenAI、Llama和ChatGLM等模型并可以直接通过pip安装。Varch是一个基于C语言和Go语言开发的项目并提供了Python接口。\n",
"\n"
]
}
],
"source": [
"query3 = \"你知道vearch是什么吗?\"\n",
"res1 = vearch_db.similarity_search(query3, 3)\n",
"for idx,tmp in enumerate(res1): \n",
" print(f\"{'#'*20}第{idx+1}段相关文档{'#'*20}\\n\\n{tmp.page_content}\\n\")\n",
"\n",
"context1 = \"\".join([tmp.page_content for tmp in res1])\n",
"new_query1 = f\"基于以下信息,尽可能准确的来回答用户的问题。背景信息:\\n {context1} \\n 回答用户这个问题:{query3}\\n\\n\"\n",
"response, history = model.chat(tokenizer, new_query1, history=[])\n",
"\n",
"print(f\"***************ChatGLM:{response}\\n\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"delete docid True\n",
"Human: 你知道vearch是什么吗?\n",
"ChatGLM:Vearch是一种高分子化合物,也称为聚合物、高分子材料或合成材料。它是由重复单元组成的大型聚合物,通常由一些重复单元组成,这些单元在聚合过程中结合在一起形成一个连续的高分子链。\n",
"\n",
"Vearch具有许多独特的性质,例如高强度、高刚性、耐磨、耐腐蚀、耐高温等。它们通常用于制造各种应用,例如塑料制品、橡胶、纤维、建筑材料等。\n",
"\n",
"after delete docid to query again: {}\n",
"get existed docid {'7aae36236f784105a0004d8ff3c7c3ad': Document(page_content='《天龙八部》第二回 玉壁月华明\\n\\n再展帛卷长卷上源源皆是裸女画像或立或卧或现前胸或见后背。人像的面容都是一般但或喜或愁或含情凝眸或轻嗔薄怒神情各异。一共有三十六幅图像每幅像上均有颜色细线注明穴道部位及练功法诀。\\n\\n帛卷尽处题着“凌波微步”四字其后绘的是无数足印注明“妇妹”、“无妄”等等字样尽是《易经》中的方位。段誉前几日还正全心全意地钻研《易经》一见到这些名称登时精神大振便似遇到故交良友一般。只见足印密密麻麻不知有几千百个自一个足印至另一个足印均有绿线贯串线上绘有箭头最后写着一行字道“步法神妙保身避敌待积内力再取敌命。”\\n\\n段誉心道“神仙姊姊所遗的步法必定精妙之极遇到强敌时脱身逃走那就很好再取敌命也就不必了。”\\n卷好帛卷对之作了两个揖珍而重之地揣入怀中转身对那玉像道“神仙姊姊你吩咐我朝午晚三次练功段誉不敢有违。今后我对人加倍客气别人不会来打我我自然也不会去吸他内力。你这套凌波微步我更要用心练熟眼见不对立刻溜之大吉就吸不到他内力了。”至于“杀尽我逍遥派弟子”一节却想也不敢去想。', metadata={'source': '/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/天龙八部/lingboweibu.txt'}), '7e495d4e5962497db2080e84d52e75ed': Document(page_content='《天龙八部》第五回 微步縠纹生\\n\\n卷轴中此外诸种经脉修习之法甚多皆是取人内力的法门段誉虽自语宽解总觉习之有违本性单是贪多务得便非好事当下暂不理会。\\n\\n卷到卷轴末端又见到了“凌波微步”那四字登时便想起《洛神赋》中那些句子来“凌波微步罗袜生尘……转眄流精光润玉颜。含辞未吐气若幽兰。华容婀娜令我忘餐。”曹子建那些千古名句在脑海中缓缓流过“秾纤得衷修短合度肩若削成腰如约素。延颈秀项皓质呈露。芳泽无加铅华弗御。云髻峨峨修眉连娟。丹唇外朗皓齿内鲜。明眸善睐靥辅承权。瑰姿艳逸仪静体闲。柔情绰态媚于语言……”这些句子用在木婉清身上“这话倒也有理”但如用之于神仙姊姊只怕更为适合。想到神仙姊姊的姿容体态“皎若太阳升朝霞灼若芙蓉出绿波”但觉依她吩咐行事实为人生至乐心想“我先来练这凌波微步此乃逃命之妙法非害人之手段也练之有百利而无一害。”', metadata={'source': '/data/zhx/zhx/langchain-ChatGLM_new/knowledge_base/天龙八部/lingboweibu.txt'})}\n"
]
}
],
"source": [
"##delete and get function need to maintian docids \n",
"##your docid\n",
"res_d=vearch_db.delete(['04bc84fff5074b7b8990441e92e6df07', 'e221906153bb4e03bc7095dadea144de', '126034ba51934093920d8732860f340b'])\n",
"print(\"delete docid\",res_d)\n",
"query = \"你知道vearch是什么吗?\"\n",
"response, history = model.chat(tokenizer, query, history=[])\n",
"print(f\"Human: {query}\\nChatGLM:{response}\\n\")\n",
"get_id_doc=vearch_db.get(['04bc84fff5074b7b8990441e92e6df07'])\n",
"print(\"after delete docid to query again:\",get_id_doc)\n",
"get_delet_doc=vearch_db.get(['7aae36236f784105a0004d8ff3c7c3ad', '7e495d4e5962497db2080e84d52e75ed'])\n",
"print(\"get existed docid\",get_delet_doc)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.10.12 ('langchainGLM6B')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "1fd24e7ef183310e43cbf656d21568350c6a30580b6df7fe3b34654b3770f74d"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}