Archives
/
langchain
mirror of https://github.com/hwchase17/langchain
2
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

feat: grammar-based sampling in llama-cpp (#9712)

Browse Source 
## Description 

The following PR enables the [grammar-based
sampling](https://github.com/ggerganov/llama.cpp/tree/master/grammars)
in llama-cpp LLM.

In short, loading file with formal grammar definition will constrain
model outputs. For instance, one can force the model to generate valid
JSON or generate only python lists.

In the follow-up PR we will add:
* docs with some description why it is cool and how it works
* maybe some code sample for some task such as in llama repo

---------

Co-authored-by: Lance Martin <lance@langchain.dev>
Co-authored-by: Bagatur <baskaryan@gmail.com>
pull/9874/head
eryk-dsai 1 year ago committed by GitHub
parent cb642ef658
commit 7f5713b80a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 620 additions and 48 deletions
Show Stats Download Patch File Download Diff File

563
docs/extras/integrations/llms/llamacpp.ipynb
Unescape Escape View File

@ -171,7 +171,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 1,
"metadata": {
"tags": []
},
@ -192,7 +192,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 2,
"metadata": {
"tags": []
},
@ -207,7 +207,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 3,
"metadata": {
"tags": []
},
@ -397,16 +397,96 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"llama.cpp: loading model from /Users/rlm/Desktop/Code/llama.cpp/llama-2-13b-chat.ggmlv3.q4_0.bin\n",
"llama_model_load_internal: format = ggjt v3 (latest)\n",
"llama_model_load_internal: n_vocab = 32000\n",
"llama_model_load_internal: n_ctx = 512\n",
"llama_model_load_internal: n_embd = 5120\n",
"llama_model_load_internal: n_mult = 256\n",
"llama_model_load_internal: n_head = 40\n",
"llama_model_load_internal: n_head_kv = 40\n",
"llama_model_load_internal: n_layer = 40\n",
"llama_model_load_internal: n_rot = 128\n",
"llama_model_load_internal: n_gqa = 1\n",
"llama_model_load_internal: rnorm_eps = 5.0e-06\n",
"llama_model_load_internal: n_ff = 13824\n",
"llama_model_load_internal: freq_base = 10000.0\n",
"llama_model_load_internal: freq_scale = 1\n",
"llama_model_load_internal: ftype = 2 (mostly Q4_0)\n",
"llama_model_load_internal: model size = 13B\n",
"llama_model_load_internal: ggml ctx size = 0.11 MB\n",
"llama_model_load_internal: mem required = 6983.72 MB (+ 400.00 MB per state)\n",
"llama_new_context_with_model: kv self size = 400.00 MB\n",
"ggml_metal_init: allocating\n",
"ggml_metal_init: loading '/Users/rlm/miniforge3/envs/llama2/lib/python3.9/site-packages/llama_cpp/ggml-metal.metal'\n",
"ggml_metal_init: loaded kernel_add 0x1405ed6b0\n",
"ggml_metal_init: loaded kernel_add_row 0x1405eee00\n",
"ggml_metal_init: loaded kernel_mul 0x1405ee650\n",
"ggml_metal_init: loaded kernel_mul_row 0x1405eda20\n",
"ggml_metal_init: loaded kernel_scale 0x121fc1d80\n",
"ggml_metal_init: loaded kernel_silu 0x121fc1fe0\n",
"ggml_metal_init: loaded kernel_relu 0x121fc2240\n",
"ggml_metal_init: loaded kernel_gelu 0x121fc24e0\n",
"ggml_metal_init: loaded kernel_soft_max 0x121fc2950\n",
"ggml_metal_init: loaded kernel_diag_mask_inf 0x121fc2d60\n",
"ggml_metal_init: loaded kernel_get_rows_f16 0x121fc3160\n",
"ggml_metal_init: loaded kernel_get_rows_q4_0 0x121fc3a20\n",
"ggml_metal_init: loaded kernel_get_rows_q4_1 0x121fc4170\n",
"ggml_metal_init: loaded kernel_get_rows_q2_K 0x121fc4890\n",
"ggml_metal_init: loaded kernel_get_rows_q3_K 0x121fc5010\n",
"ggml_metal_init: loaded kernel_get_rows_q4_K 0x121fc5750\n",
"ggml_metal_init: loaded kernel_get_rows_q5_K 0x121fc5e90\n",
"ggml_metal_init: loaded kernel_get_rows_q6_K 0x121fc65d0\n",
"ggml_metal_init: loaded kernel_rms_norm 0x121fc6d20\n",
"ggml_metal_init: loaded kernel_norm 0x121fc7460\n",
"ggml_metal_init: loaded kernel_mul_mat_f16_f32 0x121fc7dd0\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_0_f32 0x121fc8610\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_1_f32 0x121fc8e50\n",
"ggml_metal_init: loaded kernel_mul_mat_q2_K_f32 0x1405edc80\n",
"ggml_metal_init: loaded kernel_mul_mat_q3_K_f32 0x1405efdc0\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_K_f32 0x140306f30\n",
"ggml_metal_init: loaded kernel_mul_mat_q5_K_f32 0x1403073d0\n",
"ggml_metal_init: loaded kernel_mul_mat_q6_K_f32 0x140307aa0\n",
"ggml_metal_init: loaded kernel_mul_mm_f16_f32 0x140307f80\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_0_f32 0x140308460\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_1_f32 0x140308940\n",
"ggml_metal_init: loaded kernel_mul_mm_q2_K_f32 0x140308e20\n",
"ggml_metal_init: loaded kernel_mul_mm_q3_K_f32 0x140309300\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_K_f32 0x1403097e0\n",
"ggml_metal_init: loaded kernel_mul_mm_q5_K_f32 0x140309cc0\n",
"ggml_metal_init: loaded kernel_mul_mm_q6_K_f32 0x14030a1a0\n",
"ggml_metal_init: loaded kernel_rope 0x14030a400\n",
"ggml_metal_init: loaded kernel_alibi_f32 0x14030aa00\n",
"ggml_metal_init: loaded kernel_cpy_f32_f16 0x14030afd0\n",
"ggml_metal_init: loaded kernel_cpy_f32_f32 0x14030b5a0\n",
"ggml_metal_init: loaded kernel_cpy_f16_f16 0x14030bb70\n",
"ggml_metal_init: recommendedMaxWorkingSetSize = 21845.34 MB\n",
"ggml_metal_init: hasUnifiedMemory = true\n",
"ggml_metal_init: maxTransferRate = built-in GPU\n",
"llama_new_context_with_model: compute buffer total size = 91.35 MB\n",
"llama_new_context_with_model: max tensor size = 87.89 MB\n",
"ggml_metal_add_buffer: allocated 'data ' buffer, size = 6984.06 MB, ( 6984.50 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'eval ' buffer, size = 1.36 MB, ( 6985.86 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'kv ' buffer, size = 402.00 MB, ( 7387.86 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'alloc ' buffer, size = 90.02 MB, ( 7477.88 / 21845.34)\n",
"AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 | \n"
]
}
],
"source": [
"n_gpu_layers = 40 # Change this value based on your model and your GPU VRAM pool.\n",
"n_batch = 512 # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.\n",
"\n",
"# Make sure the model path is correct for your system!\n",
"llm = LlamaCpp(\n",
" model_path=\"./ggml-model-q4_0.bin\",\n",
" model_path=\"/Users/rlm/Desktop/Code/llama.cpp/llama-2-13b-chat.ggmlv3.q4_0.bin\",\n",
" n_gpu_layers=n_gpu_layers,\n",
" n_batch=n_batch,\n",
" callback_manager=callback_manager,\n",
@ -416,36 +496,20 @@
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"llm_chain = LLMChain(prompt=prompt, llm=llm)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" We are looking for an NFL team that won the Super Bowl when Justin Bieber (born March 1, 1994) was born. \n",
"\n",
"First, let's look up which year is closest to when Justin Bieber was born:\n",
"\n",
"* The year before he was born: 1993\n",
"* The year of his birth: 1994\n",
"* The year after he was born: 1995\n",
"\n",
"We want to know what NFL team won the Super Bowl in the year that is closest to when Justin Bieber was born. Therefore, we should look up the NFL team that won the Super Bowl in either 1993 or 1994.\n",
"Justin Bieber was born on March 1, 1994. The Super Bowl is played at the end of the NFL season which runs from September to February.\n",
"\n",
"Now let's find out which NFL team did win the Super Bowl in either of those years:\n",
"In 1994, the NFL season ended with Super Bowl XXVIII which was played on January 28th, 1994.\n",
"\n",
"* In 1993, the San Francisco 49ers won the Super Bowl against the Dallas Cowboys by a score of 20-16.\n",
"* In 1994, the San Francisco 49ers won the Super Bowl again, this time against the San Diego Chargers by a score of 49-26.\n"
"So, there was no Super Bowl in the year Justin Bieber was born. The Super Bowl has only been around since 1967 and is played annually between the champions of the National Football Conference (NFC) and the American Football Conference (AFC)."
]
},
{
@ -453,25 +517,27 @@
"output_type": "stream",
"text": [
"\n",
"llama_print_timings: load time = 238.10 ms\n",
"llama_print_timings: sample time = 84.23 ms / 256 runs ( 0.33 ms per token)\n",
"llama_print_timings: prompt eval time = 238.04 ms / 49 tokens ( 4.86 ms per token)\n",
"llama_print_timings: eval time = 10391.96 ms / 255 runs ( 40.75 ms per token)\n",
"llama_print_timings: total time = 15664.80 ms\n"
"llama_print_timings: load time = 427.90 ms\n",
"llama_print_timings: sample time = 98.36 ms / 133 runs ( 0.74 ms per token, 1352.18 tokens per second)\n",
"llama_print_timings: prompt eval time = 427.83 ms / 45 tokens ( 9.51 ms per token, 105.18 tokens per second)\n",
"llama_print_timings: eval time = 3687.12 ms / 132 runs ( 27.93 ms per token, 35.80 tokens per second)\n",
"llama_print_timings: total time = 4401.84 ms\n"
]
},
{
"data": {
"text/plain": [
"\" We are looking for an NFL team that won the Super Bowl when Justin Bieber (born March 1, 1994) was born. \\n\\nFirst, let's look up which year is closest to when Justin Bieber was born:\\n\\n* The year before he was born: 1993\\n* The year of his birth: 1994\\n* The year after he was born: 1995\\n\\nWe want to know what NFL team won the Super Bowl in the year that is closest to when Justin Bieber was born. Therefore, we should look up the NFL team that won the Super Bowl in either 1993 or 1994.\\n\\nNow let's find out which NFL team did win the Super Bowl in either of those years:\\n\\n* In 1993, the San Francisco 49ers won the Super Bowl against the Dallas Cowboys by a score of 20-16.\\n* In 1994, the San Francisco 49ers won the Super Bowl again, this time against the San Diego Chargers by a score of 49-26.\\n\""
"'\\n\\nJustin Bieber was born on March 1, 1994. The Super Bowl is played at the end of the NFL season which runs from September to February.\\n\\nIn 1994, the NFL season ended with Super Bowl XXVIII which was played on January 28th, 1994.\\n\\nSo, there was no Super Bowl in the year Justin Bieber was born. The Super Bowl has only been around since 1967 and is played annually between the champions of the National Football Conference (NFC) and the American Football Conference (AFC).'"
]
},
"execution_count": 8,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"llm_chain = LLMChain(prompt=prompt, llm=llm)\n",
"\n",
"question = \"What NFL team won the Super Bowl in the year Justin Bieber was born?\"\n",
"\n",
"llm_chain.run(question)"
@ -497,16 +563,96 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"llama.cpp: loading model from /Users/rlm/Desktop/Code/llama.cpp/llama-2-13b-chat.ggmlv3.q4_0.bin\n",
"llama_model_load_internal: format = ggjt v3 (latest)\n",
"llama_model_load_internal: n_vocab = 32000\n",
"llama_model_load_internal: n_ctx = 512\n",
"llama_model_load_internal: n_embd = 5120\n",
"llama_model_load_internal: n_mult = 256\n",
"llama_model_load_internal: n_head = 40\n",
"llama_model_load_internal: n_head_kv = 40\n",
"llama_model_load_internal: n_layer = 40\n",
"llama_model_load_internal: n_rot = 128\n",
"llama_model_load_internal: n_gqa = 1\n",
"llama_model_load_internal: rnorm_eps = 5.0e-06\n",
"llama_model_load_internal: n_ff = 13824\n",
"llama_model_load_internal: freq_base = 10000.0\n",
"llama_model_load_internal: freq_scale = 1\n",
"llama_model_load_internal: ftype = 2 (mostly Q4_0)\n",
"llama_model_load_internal: model size = 13B\n",
"llama_model_load_internal: ggml ctx size = 0.11 MB\n",
"llama_model_load_internal: mem required = 6983.72 MB (+ 400.00 MB per state)\n",
"llama_new_context_with_model: kv self size = 400.00 MB\n",
"ggml_metal_init: allocating\n",
"ggml_metal_init: loading '/Users/rlm/miniforge3/envs/llama2/lib/python3.9/site-packages/llama_cpp/ggml-metal.metal'\n",
"ggml_metal_init: loaded kernel_add 0x113b42480\n",
"ggml_metal_init: loaded kernel_add_row 0x113b44210\n",
"ggml_metal_init: loaded kernel_mul 0x113b43a80\n",
"ggml_metal_init: loaded kernel_mul_row 0x113b44880\n",
"ggml_metal_init: loaded kernel_scale 0x113b45010\n",
"ggml_metal_init: loaded kernel_silu 0x113b45650\n",
"ggml_metal_init: loaded kernel_relu 0x113b427f0\n",
"ggml_metal_init: loaded kernel_gelu 0x113b46300\n",
"ggml_metal_init: loaded kernel_soft_max 0x113b46980\n",
"ggml_metal_init: loaded kernel_diag_mask_inf 0x113b46e20\n",
"ggml_metal_init: loaded kernel_get_rows_f16 0x113b47860\n",
"ggml_metal_init: loaded kernel_get_rows_q4_0 0x113b48010\n",
"ggml_metal_init: loaded kernel_get_rows_q4_1 0x113b48880\n",
"ggml_metal_init: loaded kernel_get_rows_q2_K 0x113b48f70\n",
"ggml_metal_init: loaded kernel_get_rows_q3_K 0x113b49e00\n",
"ggml_metal_init: loaded kernel_get_rows_q4_K 0x113b4a530\n",
"ggml_metal_init: loaded kernel_get_rows_q5_K 0x113b4ac70\n",
"ggml_metal_init: loaded kernel_get_rows_q6_K 0x113b4b3b0\n",
"ggml_metal_init: loaded kernel_rms_norm 0x113b4bb00\n",
"ggml_metal_init: loaded kernel_norm 0x113b4c1a0\n",
"ggml_metal_init: loaded kernel_mul_mat_f16_f32 0x113b4cba0\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_0_f32 0x113b4d360\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_1_f32 0x113b4dba0\n",
"ggml_metal_init: loaded kernel_mul_mat_q2_K_f32 0x113b4e560\n",
"ggml_metal_init: loaded kernel_mul_mat_q3_K_f32 0x113b4ed10\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_K_f32 0x113b4f580\n",
"ggml_metal_init: loaded kernel_mul_mat_q5_K_f32 0x113b4fdc0\n",
"ggml_metal_init: loaded kernel_mul_mat_q6_K_f32 0x113b50740\n",
"ggml_metal_init: loaded kernel_mul_mm_f16_f32 0x113b51250\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_0_f32 0x113b51a80\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_1_f32 0x113b522b0\n",
"ggml_metal_init: loaded kernel_mul_mm_q2_K_f32 0x113b52ae0\n",
"ggml_metal_init: loaded kernel_mul_mm_q3_K_f32 0x113b53310\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_K_f32 0x113b53b40\n",
"ggml_metal_init: loaded kernel_mul_mm_q5_K_f32 0x113b54370\n",
"ggml_metal_init: loaded kernel_mul_mm_q6_K_f32 0x113b54ba0\n",
"ggml_metal_init: loaded kernel_rope 0x113b551a0\n",
"ggml_metal_init: loaded kernel_alibi_f32 0x113b55b10\n",
"ggml_metal_init: loaded kernel_cpy_f32_f16 0x113b56450\n",
"ggml_metal_init: loaded kernel_cpy_f32_f32 0x113b56dc0\n",
"ggml_metal_init: loaded kernel_cpy_f16_f16 0x113b576b0\n",
"ggml_metal_init: recommendedMaxWorkingSetSize = 21845.34 MB\n",
"ggml_metal_init: hasUnifiedMemory = true\n",
"ggml_metal_init: maxTransferRate = built-in GPU\n",
"llama_new_context_with_model: compute buffer total size = 91.35 MB\n",
"llama_new_context_with_model: max tensor size = 87.89 MB\n",
"ggml_metal_add_buffer: allocated 'data ' buffer, size = 6984.06 MB, ( 6984.50 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'eval ' buffer, size = 1.36 MB, ( 6985.86 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'kv ' buffer, size = 402.00 MB, ( 7387.86 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'alloc ' buffer, size = 90.02 MB, ( 7477.88 / 21845.34)AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 | \n",
"\n"
]
}
],
"source": [
"n_gpu_layers = 1 # Metal set to 1 is enough.\n",
"n_batch = 512 # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.\n",
"\n",
"# Make sure the model path is correct for your system!\n",
"llm = LlamaCpp(\n",
" model_path=\"./ggml-model-q4_0.bin\",\n",
" model_path=\"/Users/rlm/Desktop/Code/llama.cpp/llama-2-13b-chat.ggmlv3.q4_0.bin\",\n",
" n_gpu_layers=n_gpu_layers,\n",
" n_batch=n_batch,\n",
" f16_kv=True, # MUST set to True, otherwise you will run into problem after a couple of calls\n",
@ -531,6 +677,349 @@
"\n",
"For the first call to the LLM, the performance may be slow due to the model compilation in Metal GPU."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Grammars\n",
"\n",
"\n",
"We can specify [grammars](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md) to constrain model outputs.\n",
"\n",
"Supply the path to the specifed `json.gbnf` file."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"llama.cpp: loading model from /Users/rlm/Desktop/Code/llama.cpp/llama-2-13b-chat.ggmlv3.q4_0.bin\n",
"llama_model_load_internal: format = ggjt v3 (latest)\n",
"llama_model_load_internal: n_vocab = 32000\n",
"llama_model_load_internal: n_ctx = 512\n",
"llama_model_load_internal: n_embd = 5120\n",
"llama_model_load_internal: n_mult = 256\n",
"llama_model_load_internal: n_head = 40\n",
"llama_model_load_internal: n_head_kv = 40\n",
"llama_model_load_internal: n_layer = 40\n",
"llama_model_load_internal: n_rot = 128\n",
"llama_model_load_internal: n_gqa = 1\n",
"llama_model_load_internal: rnorm_eps = 5.0e-06\n",
"llama_model_load_internal: n_ff = 13824\n",
"llama_model_load_internal: freq_base = 10000.0\n",
"llama_model_load_internal: freq_scale = 1\n",
"llama_model_load_internal: ftype = 2 (mostly Q4_0)\n",
"llama_model_load_internal: model size = 13B\n",
"llama_model_load_internal: ggml ctx size = 0.11 MB\n",
"llama_model_load_internal: mem required = 6983.72 MB (+ 400.00 MB per state)\n",
"llama_new_context_with_model: kv self size = 400.00 MB\n",
"ggml_metal_init: allocating\n",
"ggml_metal_init: loading '/Users/rlm/miniforge3/envs/llama2/lib/python3.9/site-packages/llama_cpp/ggml-metal.metal'\n",
"ggml_metal_init: loaded kernel_add 0x1516fb530\n",
"ggml_metal_init: loaded kernel_add_row 0x1516fb790\n",
"ggml_metal_init: loaded kernel_mul 0x1516fb9f0\n",
"ggml_metal_init: loaded kernel_mul_row 0x1516fbc50\n",
"ggml_metal_init: loaded kernel_scale 0x1516fbeb0\n",
"ggml_metal_init: loaded kernel_silu 0x1516fc110\n",
"ggml_metal_init: loaded kernel_relu 0x1516fc370\n",
"ggml_metal_init: loaded kernel_gelu 0x1516fc5d0\n",
"ggml_metal_init: loaded kernel_soft_max 0x1516fc830\n",
"ggml_metal_init: loaded kernel_diag_mask_inf 0x1516fca90\n",
"ggml_metal_init: loaded kernel_get_rows_f16 0x1516fccf0\n",
"ggml_metal_init: loaded kernel_get_rows_q4_0 0x1516fcf50\n",
"ggml_metal_init: loaded kernel_get_rows_q4_1 0x1516fd1b0\n",
"ggml_metal_init: loaded kernel_get_rows_q2_K 0x1516fd410\n",
"ggml_metal_init: loaded kernel_get_rows_q3_K 0x1516fd670\n",
"ggml_metal_init: loaded kernel_get_rows_q4_K 0x1516fd8d0\n",
"ggml_metal_init: loaded kernel_get_rows_q5_K 0x1516fdb30\n",
"ggml_metal_init: loaded kernel_get_rows_q6_K 0x1516fdd90\n",
"ggml_metal_init: loaded kernel_rms_norm 0x1516fdff0\n",
"ggml_metal_init: loaded kernel_norm 0x1516fe250\n",
"ggml_metal_init: loaded kernel_mul_mat_f16_f32 0x1516fe4b0\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_0_f32 0x1516fe710\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_1_f32 0x1516fe970\n",
"ggml_metal_init: loaded kernel_mul_mat_q2_K_f32 0x1516febd0\n",
"ggml_metal_init: loaded kernel_mul_mat_q3_K_f32 0x1516fee30\n",
"ggml_metal_init: loaded kernel_mul_mat_q4_K_f32 0x1516ff090\n",
"ggml_metal_init: loaded kernel_mul_mat_q5_K_f32 0x1516ff2f0\n",
"ggml_metal_init: loaded kernel_mul_mat_q6_K_f32 0x1516ff550\n",
"ggml_metal_init: loaded kernel_mul_mm_f16_f32 0x1516ff7b0\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_0_f32 0x121fce650\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_1_f32 0x121fcdce0\n",
"ggml_metal_init: loaded kernel_mul_mm_q2_K_f32 0x121fceab0\n",
"ggml_metal_init: loaded kernel_mul_mm_q3_K_f32 0x121fced10\n",
"ggml_metal_init: loaded kernel_mul_mm_q4_K_f32 0x121fcef70\n",
"ggml_metal_init: loaded kernel_mul_mm_q5_K_f32 0x121fcf1d0\n",
"ggml_metal_init: loaded kernel_mul_mm_q6_K_f32 0x121fcf430\n",
"ggml_metal_init: loaded kernel_rope 0x121fcf690\n",
"ggml_metal_init: loaded kernel_alibi_f32 0x121fcf8f0\n",
"ggml_metal_init: loaded kernel_cpy_f32_f16 0x121fcfb50\n",
"ggml_metal_init: loaded kernel_cpy_f32_f32 0x121fcfdb0\n",
"ggml_metal_init: loaded kernel_cpy_f16_f16 0x121fd0010\n",
"ggml_metal_init: recommendedMaxWorkingSetSize = 21845.34 MB\n",
"ggml_metal_init: hasUnifiedMemory = true\n",
"ggml_metal_init: maxTransferRate = built-in GPU\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"root ::= object \n",
"object ::= [{] ws object_11 [}] \n",
"value ::= object | array | string | number | boolean | [n] [u] [l] [l] \n",
"array ::= [[] ws array_15 []] \n",
"string ::= [\"] string_18 [\"] ws \n",
"number ::= number_19 number_20 ws \n",
"boolean ::= boolean_21 ws \n",
"ws ::= ws_23 \n",
"object_8 ::= string [:] ws value object_10 \n",
"object_9 ::= [,] ws string [:] ws value \n",
"object_10 ::= object_9 object_10 | \n",
"object_11 ::= object_8 | \n",
"array_12 ::= value array_14 \n",
"array_13 ::= [,] ws value \n",
"array_14 ::= array_13 array_14 | \n",
"array_15 ::= array_12 | \n",
"string_16 ::= [^\"\\] | [\\] string_17 \n",
"string_17 ::= [\"\\/bfnrt] | [u] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] \n",
"string_18 ::= string_16 string_18 | \n",
"number_19 ::= [-] | \n",
"number_20 ::= [0-9] number_20 | [0-9] \n",
"boolean_21 ::= [t] [r] [u] [e] | [f] [a] [l] [s] [e] \n",
"ws_22 ::= [ <U+0009><U+000A>] ws \n",
"ws_23 ::= ws_22 | \n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"llama_new_context_with_model: compute buffer total size = 91.35 MB\n",
"llama_new_context_with_model: max tensor size = 87.89 MB\n",
"ggml_metal_add_buffer: allocated 'data ' buffer, size = 6984.06 MB, (14468.72 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'eval ' buffer, size = 1.36 MB, (14470.08 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'kv ' buffer, size = 402.00 MB, (14872.08 / 21845.34)\n",
"ggml_metal_add_buffer: allocated 'alloc ' buffer, size = 90.02 MB, (14962.09 / 21845.34)\n",
"AVX = 0 | AVX2 = 0 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 0 | NEON = 1 | ARM_FMA = 1 | F16C = 0 | FP16_VA = 1 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 0 | VSX = 0 | \n",
"from_string grammar:\n",
"\n"
]
}
],
"source": [
"n_gpu_layers = 1 \n",
"n_batch = 512\n",
"llm = LlamaCpp(\n",
" model_path=\"/Users/rlm/Desktop/Code/llama.cpp/llama-2-13b-chat.ggmlv3.q4_0.bin\",\n",
" n_gpu_layers=n_gpu_layers,\n",
" n_batch=n_batch,\n",
" f16_kv=True, # MUST set to True, otherwise you will run into problem after a couple of calls\n",
" callback_manager=callback_manager,\n",
" verbose=True,\n",
" grammar_path=\"/Users/rlm/Desktop/Code/langchain-main/langchain/libs/langchain/langchain/llms/grammars/json.gbnf\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Error in LangChainTracer.on_llm_start callback: ctypes objects containing pointers cannot be pickled\n",
"Exception ignored in: <function LlamaGrammar.__del__ at 0x1402b15e0>\n",
"Traceback (most recent call last):\n",
" File \"/Users/rlm/miniforge3/envs/llama2/lib/python3.9/site-packages/llama_cpp/llama_grammar.py\", line 46, in __del__\n",
" if self.grammar is not None:\n",
"AttributeError: 'LlamaGrammar' object has no attribute 'grammar'\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\"name\": \"John Doe\", \"age\": 30, \"gender\": \"male\"}"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"llama_print_timings: load time = 317.62 ms\n",
"llama_print_timings: sample time = 141.83 ms / 22 runs ( 6.45 ms per token, 155.11 tokens per second)\n",
"llama_print_timings: prompt eval time = 316.89 ms / 9 tokens ( 35.21 ms per token, 28.40 tokens per second)\n",
"llama_print_timings: eval time = 575.93 ms / 21 runs ( 27.43 ms per token, 36.46 tokens per second)\n",
"llama_print_timings: total time = 1087.31 ms\n",
"Error in LangChainTracer.on_llm_end callback: ctypes objects containing pointers cannot be pickled\n",
"Exception ignored in: <function LlamaGrammar.__del__ at 0x1402b15e0>\n",
"Traceback (most recent call last):\n",
" File \"/Users/rlm/miniforge3/envs/llama2/lib/python3.9/site-packages/llama_cpp/llama_grammar.py\", line 46, in __del__\n",
" if self.grammar is not None:\n",
"AttributeError: 'LlamaGrammar' object has no attribute 'grammar'\n"
]
}
],
"source": [
"result=llm(\"Describe a person in JSON format:\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'John Doe'"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"eval(result)[\"name\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can also try `list.gbnf`."
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"llama.cpp: loading model from /home/eryk/deepsense/llama-2-7b.ggmlv3.q4_0.bin\n",
"llama_model_load_internal: format = ggjt v3 (latest)\n",
"llama_model_load_internal: n_vocab = 32000\n",
"llama_model_load_internal: n_ctx = 512\n",
"llama_model_load_internal: n_embd = 4096\n",
"llama_model_load_internal: n_mult = 256\n",
"llama_model_load_internal: n_head = 32\n",
"llama_model_load_internal: n_head_kv = 32\n",
"llama_model_load_internal: n_layer = 32\n",
"llama_model_load_internal: n_rot = 128\n",
"llama_model_load_internal: n_gqa = 1\n",
"llama_model_load_internal: rnorm_eps = 5.0e-06\n",
"llama_model_load_internal: n_ff = 11008\n",
"llama_model_load_internal: freq_base = 10000.0\n",
"llama_model_load_internal: freq_scale = 1\n",
"llama_model_load_internal: ftype = 2 (mostly Q4_0)\n",
"llama_model_load_internal: model size = 7B\n",
"llama_model_load_internal: ggml ctx size = 0.08 MB\n",
"llama_model_load_internal: mem required = 3615.73 MB (+ 256.00 MB per state)\n",
"llama_new_context_with_model: kv self size = 256.00 MB\n",
"AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | VSX = 0 | \n",
"llama_new_context_with_model: compute buffer total size = 71.84 MB\n",
"from_string grammar:\n",
"\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"root ::= [[] items []] EOF \n",
"items ::= item items_7 \n",
"EOF ::= [<U+000A>] \n",
"item ::= string \n",
"items_4 ::= [,] items_6 item \n",
"ws ::= [ ] \n",
"items_6 ::= ws items_6 | \n",
"items_7 ::= items_4 items_7 | \n",
"string ::= [\"] word string_12 [\"] string_13 \n",
"word ::= word_14 \n",
"string_10 ::= string_11 word \n",
"string_11 ::= ws string_11 | ws \n",
"string_12 ::= string_10 string_12 | \n",
"string_13 ::= ws string_13 | \n",
"word_14 ::= [a-zA-Z] word_14 | [a-zA-Z] \n"
]
}
],
"source": [
"n_gpu_layers = 1 \n",
"n_batch = 512\n",
"llm = LlamaCpp(\n",
" model_path=\"/home/eryk/deepsense/llama-2-7b.ggmlv3.q4_0.bin\",\n",
" n_gpu_layers=n_gpu_layers,\n",
" n_batch=n_batch,\n",
" f16_kv=True, # MUST set to True, otherwise you will run into problem after a couple of calls\n",
" callback_manager=callback_manager,\n",
" verbose=True,\n",
" grammar_path=\"/home/eryk/deepsense/langchain/libs/langchain/langchain/llms/grammars/list.gbnf\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[\"Jane Eyre\" , \"Sense and Sensibility\" , \"A Tale of Two Cities\"]\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"llama_print_timings: load time = 1079.21 ms\n",
"llama_print_timings: sample time = 225.57 ms / 29 runs ( 7.78 ms per token, 128.56 tokens per second)\n",
"llama_print_timings: prompt eval time = 1078.34 ms / 11 tokens ( 98.03 ms per token, 10.20 tokens per second)\n",
"llama_print_timings: eval time = 4389.99 ms / 28 runs ( 156.79 ms per token, 6.38 tokens per second)\n",
"llama_print_timings: total time = 5807.84 ms\n"
]
}
],
"source": [
"result=llm(\"List of top-3 my favourite books:\")"
]
},
{
"cell_type": "code",
"execution_count": 85,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Jane Eyre', 'Sense and Sensibility', 'A Tale of Two Cities']"
]
},
"execution_count": 85,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"eval(result)"
]
}
],
"metadata": {
@ -549,7 +1038,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
"version": "3.9.16"
}
},
"nbformat": 4,

29
libs/langchain/langchain/llms/grammars/json.gbnf
Unescape Escape View File

@ -0,0 +1,29 @@
# Grammar for subset of JSON - doesn't support full string or number syntax
root ::= object
value ::= object | array | string | number | boolean | "null"
object ::=
"{" ws (
string ":" ws value
("," ws string ":" ws value)*
)? "}"
array ::=
"[" ws (
value
("," ws value)*
)? "]"
string ::=
"\"" (
[^"\\] |
"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
)* "\"" ws
# Only plain integers currently
number ::= "-"? [0-9]+ ws
boolean ::= ("true" | "false") ws
# Optional space: by convention, applied in this grammar after literal chars when allowed
ws ::= ([ \t\n] ws)?

14
libs/langchain/langchain/llms/grammars/list.gbnf
Unescape Escape View File

@ -0,0 +1,14 @@
root ::= "[" items "]" EOF
items ::= item ("," ws* item)*
item ::= string
string ::=
"\"" word (ws+ word)* "\"" ws*
word ::= [a-zA-Z]+
ws ::= " "
EOF ::= "\n"

62
libs/langchain/langchain/llms/llamacpp.py
Unescape Escape View File

@ -1,5 +1,8 @@
from __future__ import annotations
import logging
from typing import Any, Dict, Iterator, List, Optional
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union
from langchain.callbacks.manager import CallbackManagerForLLMRun
from langchain.llms.base import LLM
@ -8,6 +11,9 @@ from langchain.schema.output import GenerationChunk
from langchain.utils import get_pydantic_field_names
from langchain.utils.utils import build_extra_kwargs
if TYPE_CHECKING:
from llama_cpp import LlamaGrammar
logger = logging.getLogger(__name__)
@ -113,12 +119,35 @@ class LlamaCpp(LLM):
streaming: bool = True
"""Whether to stream the results, token by token."""
grammar_path: Optional[Union[str, Path]] = None
"""
grammar_path: Path to the .gbnf file that defines formal grammars
for constraining model outputs. For instance, the grammar can be used
to force the model to generate valid JSON or to speak exclusively in emojis. At most
one of grammar_path and grammar should be passed in.
"""
grammar: Optional[Union[str, LlamaGrammar]] = None
"""
grammar: formal grammar for constraining model outputs. For instance, the grammar
can be used to force the model to generate valid JSON or to speak exclusively in
emojis. At most one of grammar_path and grammar should be passed in.
"""
verbose: bool = True
"""Print verbose output to stderr."""
@root_validator()
def validate_environment(cls, values: Dict) -> Dict:
"""Validate that llama-cpp-python library is installed."""
try:
from llama_cpp import Llama, LlamaGrammar
except ImportError:
raise ImportError(
"Could not import llama-cpp-python library. "
"Please install the llama-cpp-python library to "
"use this embedding model: pip install llama-cpp-python"
)
model_path = values["model_path"]
model_param_names = [
"rope_freq_scale",
@ -146,21 +175,26 @@ class LlamaCpp(LLM):
model_params.update(values["model_kwargs"])
try:
from llama_cpp import Llama
values["client"] = Llama(model_path, **model_params)
except ImportError:
raise ImportError(
"Could not import llama-cpp-python library. "
"Please install the llama-cpp-python library to "
"use this embedding model: pip install llama-cpp-python"
)
except Exception as e:
raise ValueError(
f"Could not load Llama model from path: {model_path}. "
f"Received error {e}"
)
if values["grammar"] and values["grammar_path"]:
grammar = values["grammar"]
grammar_path = values["grammar_path"]
raise ValueError(
"Can only pass in one of grammar and grammar_path. Received "
f"{grammar=} and {grammar_path=}."
)
elif isinstance(values["grammar"], str):
values["grammar"] = LlamaGrammar.from_string(values["grammar"])
elif values["grammar_path"]:
values["grammar"] = LlamaGrammar.from_file(values["grammar_path"])
else:
pass
return values
@root_validator(pre=True)
@ -176,7 +210,7 @@ class LlamaCpp(LLM):
@property
def _default_params(self) -> Dict[str, Any]:
"""Get the default parameters for calling llama_cpp."""
return {
params = {
"suffix": self.suffix,
"max_tokens": self.max_tokens,
"temperature": self.temperature,
@ -187,6 +221,9 @@ class LlamaCpp(LLM):
"repeat_penalty": self.repeat_penalty,
"top_k": self.top_k,
}
if self.grammar:
params["grammar"] = self.grammar
return params
@property
def _identifying_params(self) -> Dict[str, Any]:
@ -252,7 +289,10 @@ class LlamaCpp(LLM):
# and return the combined strings from the first choices's text:
combined_text_output = ""
for chunk in self._stream(
prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
prompt=prompt,
stop=stop,
run_manager=run_manager,
**kwargs,
):
combined_text_output += chunk.text
return combined_text_output

Write Preview
Loading…
Cancel
Save