@ -6,6 +6,8 @@ from pydantic import Field, root_validator
from langchain . callbacks . manager import CallbackManagerForLLMRun
from langchain . callbacks . manager import CallbackManagerForLLMRun
from langchain . llms . base import LLM
from langchain . llms . base import LLM
from langchain . schema . output import GenerationChunk
from langchain . schema . output import GenerationChunk
from langchain . utils import get_pydantic_field_names
from langchain . utils . utils import build_extra_kwargs
logger = logging . getLogger ( __name__ )
logger = logging . getLogger ( __name__ )
@ -106,6 +108,9 @@ class LlamaCpp(LLM):
rope_freq_base : float = 10000.0
rope_freq_base : float = 10000.0
""" Base frequency for rope sampling. """
""" Base frequency for rope sampling. """
model_kwargs : Dict [ str , Any ] = Field ( default_factory = dict )
""" Any additional parameters to pass to llama_cpp.Llama. """
streaming : bool = True
streaming : bool = True
""" Whether to stream the results, token by token. """
""" Whether to stream the results, token by token. """
@ -139,6 +144,8 @@ class LlamaCpp(LLM):
if values [ " n_gpu_layers " ] is not None :
if values [ " n_gpu_layers " ] is not None :
model_params [ " n_gpu_layers " ] = values [ " n_gpu_layers " ]
model_params [ " n_gpu_layers " ] = values [ " n_gpu_layers " ]
model_params . update ( values [ " model_kwargs " ] )
try :
try :
from llama_cpp import Llama
from llama_cpp import Llama
@ -157,6 +164,16 @@ class LlamaCpp(LLM):
return values
return values
@root_validator ( pre = True )
def build_model_kwargs ( cls , values : Dict [ str , Any ] ) - > Dict [ str , Any ] :
""" Build extra kwargs from additional params that were passed in. """
all_required_field_names = get_pydantic_field_names ( cls )
extra = values . get ( " model_kwargs " , { } )
values [ " model_kwargs " ] = build_extra_kwargs (
extra , values , all_required_field_names
)
return values
@property
@property
def _default_params ( self ) - > Dict [ str , Any ] :
def _default_params ( self ) - > Dict [ str , Any ] :
""" Get the default parameters for calling llama_cpp. """
""" Get the default parameters for calling llama_cpp. """