|
|
@ -1,15 +1,16 @@
|
|
|
|
from application.llm.base import BaseLLM
|
|
|
|
from application.llm.base import BaseLLM
|
|
|
|
|
|
|
|
from application.core.settings import settings
|
|
|
|
|
|
|
|
|
|
|
|
class LlamaCpp(BaseLLM):
|
|
|
|
class LlamaCpp(BaseLLM):
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, api_key, llm_name='/Users/pavel/Desktop/docsgpt/application/models/orca-test.bin'):
|
|
|
|
def __init__(self, api_key, llm_name=settings.MODEL_PATH, **kwargs):
|
|
|
|
global llama
|
|
|
|
global llama
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
from llama_cpp import Llama
|
|
|
|
from llama_cpp import Llama
|
|
|
|
except ImportError:
|
|
|
|
except ImportError:
|
|
|
|
raise ImportError("Please install llama_cpp using pip install llama-cpp-python")
|
|
|
|
raise ImportError("Please install llama_cpp using pip install llama-cpp-python")
|
|
|
|
|
|
|
|
|
|
|
|
llama = Llama(model_path=llm_name)
|
|
|
|
llama = Llama(model_path=llm_name, n_ctx=2048)
|
|
|
|
|
|
|
|
|
|
|
|
def gen(self, model, engine, messages, stream=False, **kwargs):
|
|
|
|
def gen(self, model, engine, messages, stream=False, **kwargs):
|
|
|
|
context = messages[0]['content']
|
|
|
|
context = messages[0]['content']
|
|
|
|