"n_gpu_layers = 40 # Change this value based on your model and your GPU VRAM pool.\n",
"n_gpu_layers = -1 # The number of layers to put on the GPU. The rest will be on the CPU. If you don't know how many layers there are, you can use -1 to move all to GPU.\n",
"n_batch = 512 # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.\n",
"\n",
"# Make sure the model path is correct for your system!\n",
@ -501,7 +501,7 @@
"metadata": {},
"outputs": [],
"source": [
"n_gpu_layers = 1 # Change this value based on your model and your GPU VRAM pool.\n",
"n_gpu_layers = 1 # The number of layers to put on the GPU. The rest will be on the CPU. If you don't know how many layers there are, you can use -1 to move all to GPU.\n",
"n_batch = 512 # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.\n",
"# Make sure the model path is correct for your system!\n",
"llm = LlamaCpp(\n",
@ -559,7 +559,7 @@
"metadata": {},
"outputs": [],
"source": [
"n_gpu_layers = 1 # Metal set to 1 is enough.\n",
"n_gpu_layers = 1 # The number of layers to put on the GPU. The rest will be on the CPU. If you don't know how many layers there are, you can use -1 to move all to GPU.\n",
"n_batch = 512 # Should be between 1 and n_ctx, consider the amount of RAM of your Apple Silicon Chip.\n",
"# Make sure the model path is correct for your system!\n",