@ -60,14 +60,14 @@
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 30.7 ms, sys: 18.6 ms, total: 49.3 ms\n",
"Wall time: 791 m s\n"
"CPU times: user 14.2 ms, sys: 4.9 ms, total: 19.1 ms\n",
"Wall time: 1.1 s\n"
]
},
{
"data": {
"text/plain": [
"\"\\n\\nWhy couldn't the bicycle stand up by itself? Because it was...two tired!\" "
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.' "
]
},
"execution_count": 4,
@ -91,14 +91,14 @@
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 80 µs, sys: 0 ns, total: 80 µs\n",
"Wall time: 83.9 µs\n"
"CPU times: user 162 µs, sys: 7 µs, total: 169 µs\n",
"Wall time: 175 µs\n"
]
},
{
"data": {
"text/plain": [
"\"\\n\\nWhy couldn't the bicycle stand up by itself? Because it was...two tired!\" "
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.' "
]
},
"execution_count": 5,
@ -252,6 +252,249 @@
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "markdown",
"id": "684eab55",
"metadata": {},
"source": [
"## GPTCache\n",
"\n",
"We can use [GPTCache](https://github.com/zilliztech/GPTCache) for exact match caching OR to cache results based on semantic similarity\n",
"\n",
"Let's first start with an example of exact match"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "14a82124",
"metadata": {},
"outputs": [],
"source": [
"import gptcache\n",
"from gptcache.processor.pre import get_prompt\n",
"from gptcache.manager.factory import get_data_manager\n",
"from langchain.cache import GPTCache\n",
"\n",
"# Avoid multiple caches using the same file, causing different llm model caches to affect each other\n",
"i = 0\n",
"file_prefix = \"data_map\"\n",
"\n",
"def init_gptcache_map(cache_obj: gptcache.Cache):\n",
" global i\n",
" cache_path = f'{file_prefix}_{i}.txt'\n",
" cache_obj.init(\n",
" pre_embedding_func=get_prompt,\n",
" data_manager=get_data_manager(data_path=cache_path),\n",
" )\n",
" i += 1\n",
"\n",
"langchain.llm_cache = GPTCache(init_gptcache_map)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "9e4ecfd1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 8.6 ms, sys: 3.82 ms, total: 12.4 ms\n",
"Wall time: 881 ms\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The first time, it is not yet in cache, so it should take longer\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "c98bbe3b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 286 µs, sys: 21 µs, total: 307 µs\n",
"Wall time: 316 µs\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The second time it is, so it goes faster\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "markdown",
"id": "502b6076",
"metadata": {},
"source": [
"Let's now show an example of similarity caching"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "b3c663bb",
"metadata": {},
"outputs": [],
"source": [
"import gptcache\n",
"from gptcache.processor.pre import get_prompt\n",
"from gptcache.manager.factory import get_data_manager\n",
"from langchain.cache import GPTCache\n",
"from gptcache.manager import get_data_manager, CacheBase, VectorBase\n",
"from gptcache import Cache\n",
"from gptcache.embedding import Onnx\n",
"from gptcache.similarity_evaluation.distance import SearchDistanceEvaluation\n",
"\n",
"# Avoid multiple caches using the same file, causing different llm model caches to affect each other\n",
"i = 0\n",
"file_prefix = \"data_map\"\n",
"llm_cache = Cache()\n",
"\n",
"\n",
"def init_gptcache_map(cache_obj: gptcache.Cache):\n",
" global i\n",
" cache_path = f'{file_prefix}_{i}.txt'\n",
" onnx = Onnx()\n",
" cache_base = CacheBase('sqlite')\n",
" vector_base = VectorBase('faiss', dimension=onnx.dimension)\n",
" data_manager = get_data_manager(cache_base, vector_base, max_size=10, clean_size=2)\n",
" cache_obj.init(\n",
" pre_embedding_func=get_prompt,\n",
" embedding_func=onnx.to_embeddings,\n",
" data_manager=data_manager,\n",
" similarity_evaluation=SearchDistanceEvaluation(),\n",
" )\n",
" i += 1\n",
"\n",
"langchain.llm_cache = GPTCache(init_gptcache_map)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "8c273ced",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 1.01 s, sys: 153 ms, total: 1.16 s\n",
"Wall time: 2.49 s\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# The first time, it is not yet in cache, so it should take longer\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "93e21a5f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 745 ms, sys: 13.2 ms, total: 758 ms\n",
"Wall time: 136 ms\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# This is an exact match, so it finds it in the cache\n",
"llm(\"Tell me a joke\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "c4bb024b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 737 ms, sys: 7.79 ms, total: 745 ms\n",
"Wall time: 135 ms\n"
]
},
{
"data": {
"text/plain": [
"'\\n\\nWhy did the chicken cross the road?\\n\\nTo get to the other side.'"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%%time\n",
"# This is not an exact match, but semantically within distance so it hits!\n",
"llm(\"Tell me joke\")"
]
},
{
"cell_type": "markdown",
"id": "934943dc",