|
|
@ -4,7 +4,7 @@ from application.core.settings import settings
|
|
|
|
|
|
|
|
|
|
|
|
class LlamaCpp(BaseLLM):
|
|
|
|
class LlamaCpp(BaseLLM):
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, api_key, llm_name=settings.MODEL_PATH, **kwargs):
|
|
|
|
def __init__(self, api_key, llm_name=settings.MODEL_PATH, *args, **kwargs):
|
|
|
|
global llama
|
|
|
|
global llama
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
from llama_cpp import Llama
|
|
|
|
from llama_cpp import Llama
|
|
|
@ -13,6 +13,8 @@ class LlamaCpp(BaseLLM):
|
|
|
|
"Please install llama_cpp using pip install llama-cpp-python"
|
|
|
|
"Please install llama_cpp using pip install llama-cpp-python"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
super().__init__(*args, **kwargs)
|
|
|
|
|
|
|
|
self.api_key = api_key
|
|
|
|
llama = Llama(model_path=llm_name, n_ctx=2048)
|
|
|
|
llama = Llama(model_path=llm_name, n_ctx=2048)
|
|
|
|
|
|
|
|
|
|
|
|
def _raw_gen(self, model, messages, stream=False, **kwargs):
|
|
|
|
def _raw_gen(self, model, messages, stream=False, **kwargs):
|
|
|
|