From e0de62f6dab55ae5dbf7e752813f60a0410e9a4c Mon Sep 17 00:00:00 2001 From: Jeffrey Wang <64715088+imjwang@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:42:41 -0400 Subject: [PATCH] Add RoPE Scaling params from llamacpp (#8422) Description: Just adding parameters from `llama-python-cpp` that support RoPE scaling. @hwchase17, @baskaryan sources: papers and explanation: https://kaiokendev.github.io/context llamacpp conversation: https://github.com/ggerganov/llama.cpp/discussions/1965 Supports models like: https://huggingface.co/conceptofmind/LLongMA-2-13b --- libs/langchain/langchain/llms/llamacpp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/langchain/langchain/llms/llamacpp.py b/libs/langchain/langchain/llms/llamacpp.py index be79076f16..c3a00b6861 100644 --- a/libs/langchain/langchain/llms/llamacpp.py +++ b/libs/langchain/langchain/llms/llamacpp.py @@ -100,6 +100,12 @@ class LlamaCpp(LLM): use_mmap: Optional[bool] = True """Whether to keep the model loaded in RAM""" + rope_freq_scale: float = 1.0 + """Scale factor for rope sampling.""" + + rope_freq_base: float = 10000.0 + """Base frequency for rope sampling.""" + streaming: bool = True """Whether to stream the results, token by token.""" @@ -111,6 +117,8 @@ class LlamaCpp(LLM): """Validate that llama-cpp-python library is installed.""" model_path = values["model_path"] model_param_names = [ + "rope_freq_scale", + "rope_freq_base", "lora_path", "lora_base", "n_ctx",