From 9a33bf22101df182a09e12a7cb5f2164560359ab Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 1 Oct 2023 19:16:13 +0100 Subject: [PATCH 1/3] script + cpu optimisations --- application/api/answer/routes.py | 2 ++ application/core/settings.py | 2 +- application/llm/llama_cpp.py | 5 +++-- docker-compose-local.yaml | 34 +------------------------------- frontend/src/Navigation.tsx | 3 ++- setup.sh | 4 +++- 6 files changed, 12 insertions(+), 38 deletions(-) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 3f48d33..b694c4a 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -118,6 +118,8 @@ def complete_stream(question, docsearch, chat_history, api_key, conversation_id) docs = docsearch.search(question, k=2) + if settings.LLM_NAME == "llama.cpp": + docs = [docs[0]] # join all page_content together with a newline docs_together = "\n".join([doc.page_content for doc in docs]) p_chat_combine = chat_combine_template.replace("{summaries}", docs_together) diff --git a/application/core/settings.py b/application/core/settings.py index 57a9f1c..a05fd00 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -2,7 +2,7 @@ from pathlib import Path import os from pydantic import BaseSettings -current_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) class Settings(BaseSettings): diff --git a/application/llm/llama_cpp.py b/application/llm/llama_cpp.py index ebd713c..f18d437 100644 --- a/application/llm/llama_cpp.py +++ b/application/llm/llama_cpp.py @@ -1,15 +1,16 @@ from application.llm.base import BaseLLM +from application.core.settings import settings class LlamaCpp(BaseLLM): - def __init__(self, api_key, llm_name='/Users/pavel/Desktop/docsgpt/application/models/orca-test.bin'): + def __init__(self, api_key, llm_name=settings.MODEL_PATH, **kwargs): global llama try: from llama_cpp import Llama except ImportError: raise ImportError("Please install llama_cpp using pip install llama-cpp-python") - llama = Llama(model_path=llm_name) + llama = Llama(model_path=llm_name, n_ctx=2048) def gen(self, model, engine, messages, stream=False, **kwargs): context = messages[0]['content'] diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index b24ed48..3aebe8b 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -6,42 +6,10 @@ services: environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING + - VITE_EMBEDDINGS_NAME=$EMBEDDINGS_NAME ports: - "5173:5173" - # backend: - # build: ./application - # environment: - # - LLM_NAME=$LLM_NAME - # - EMBEDDINGS_NAME=$EMBEDDINGS_NAME - # - CELERY_BROKER_URL=redis://redis:6379/0 - # - CELERY_RESULT_BACKEND=redis://redis:6379/1 - # - MONGO_URI=mongodb://mongo:27017/docsgpt - # ports: - # - "7091:7091" - # volumes: - # - ./application/indexes:/app/application/indexes - # - ./application/inputs:/app/application/inputs - # - ./application/vectors:/app/application/vectors - # - ./application/models:/app/application/models - # depends_on: - # - redis - # - mongo - - worker: - build: ./application - command: celery -A application.app.celery worker -l INFO - environment: - - LLM_NAME=$LLM_NAME - - EMBEDDINGS_NAME=$EMBEDDINGS_NAME - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - MONGO_URI=mongodb://mongo:27017/docsgpt - - API_URL=http://backend:7091 - depends_on: - - redis - - mongo - redis: image: redis:6-alpine ports: diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index c5488a5..9f84b97 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -59,6 +59,7 @@ export default function Navigation({ const navRef = useRef(null); const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; + const embeddingsName = import.meta.env.VITE_EMBEDDINGS_NAME || 'openai_text-embedding-ada-002'; useEffect(() => { if (!conversations) { @@ -253,7 +254,7 @@ export default function Navigation({
{docs ? ( docs.map((doc, index) => { - if (doc.model === 'openai_text-embedding-ada-002') { + if (doc.model === embeddingsName) { return (
Date: Sun, 1 Oct 2023 19:55:11 +0100 Subject: [PATCH 2/3] celery bugs --- application/api/user/routes.py | 2 +- setup.sh | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 2b1d505..fba818c 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -142,7 +142,7 @@ def upload_file(): def task_status(): """Get celery job status.""" task_id = request.args.get("task_id") - task = AsyncResult(task_id) + task = AsyncResult(task_id, backend=settings.CELERY_RESULT_BACKEND) task_meta = task.info return {"status": task.status, "result": task_meta} diff --git a/setup.sh b/setup.sh index fca9481..43e708d 100755 --- a/setup.sh +++ b/setup.sh @@ -23,23 +23,26 @@ download_locally() { # check if docsgpt-7b-f16.gguf does not exist if [ ! -f models/docsgpt-7b-f16.gguf ]; then echo "Downloading the model..." - wget -P models https://docsgpt.s3.eu-west-1.amazonaws.com/models/docsgpt-7b-f16.gguf + wget -P models https://d3dg1063dc54p9.cloudfront.net/models/docsgpt-7b-f16.gguf echo "Model downloaded to models directory." else echo "Model already exists." fi docker-compose -f docker-compose-local.yaml build && docker-compose -f docker-compose-local.yaml up -d - python -m venv venv - source venv/bin/activate - pip install -r application/requirements.txt - pip install llama-cpp-python - pip install sentence-transformers + #python -m venv venv + #source venv/bin/activate + #pip install -r application/requirements.txt + #pip install llama-cpp-python + #pip install sentence-transformers export FLASK_APP=application/app.py export FLASK_DEBUG=true + export CELERY_BROKER_URL=redis://localhost:6379/0 + export CELERY_RESULT_BACKEND=redis://localhost:6379/1 echo "The application is now running on http://localhost:5173" echo "You can stop the application by running the following command:" echo "Ctrl + C and then" + echo "Then pkill -f "flask run" and then" echo "docker-compose down" flask run --host=0.0.0.0 --port=7091 & celery -A application.app.celery worker -l INFO From cd9b03bdb9ddd336841f4fc79aaf3df73a5a06ee Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 1 Oct 2023 20:05:13 +0100 Subject: [PATCH 3/3] celery syncs --- application/api/user/routes.py | 4 ++-- application/celery.py | 2 +- setup.sh | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index fba818c..56b0201 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -6,7 +6,6 @@ from pymongo import MongoClient from bson.objectid import ObjectId from werkzeug.utils import secure_filename import http.client -from celery.result import AsyncResult from application.api.user.tasks import ingest @@ -142,7 +141,8 @@ def upload_file(): def task_status(): """Get celery job status.""" task_id = request.args.get("task_id") - task = AsyncResult(task_id, backend=settings.CELERY_RESULT_BACKEND) + from application.celery import celery + task = celery.AsyncResult(task_id) task_meta = task.info return {"status": task.status, "result": task_meta} diff --git a/application/celery.py b/application/celery.py index 3b4819e..c19c2e7 100644 --- a/application/celery.py +++ b/application/celery.py @@ -2,7 +2,7 @@ from celery import Celery from application.core.settings import settings def make_celery(app_name=__name__): - celery = Celery(app_name, broker=settings.CELERY_BROKER_URL) + celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND) celery.conf.update(settings) return celery diff --git a/setup.sh b/setup.sh index 43e708d..f9d3b87 100755 --- a/setup.sh +++ b/setup.sh @@ -35,6 +35,8 @@ download_locally() { #pip install -r application/requirements.txt #pip install llama-cpp-python #pip install sentence-transformers + export LLM_NAME=llama.cpp + export EMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2 export FLASK_APP=application/app.py export FLASK_DEBUG=true export CELERY_BROKER_URL=redis://localhost:6379/0 @@ -42,7 +44,7 @@ download_locally() { echo "The application is now running on http://localhost:5173" echo "You can stop the application by running the following command:" echo "Ctrl + C and then" - echo "Then pkill -f "flask run" and then" + echo "Then pkill -f 'flask run' and then" echo "docker-compose down" flask run --host=0.0.0.0 --port=7091 & celery -A application.app.celery worker -l INFO