From fce7d3417146e36d29efbfd6928bc763685512b3 Mon Sep 17 00:00:00 2001 From: Levente Csoke Date: Thu, 15 Jun 2023 17:02:52 +0200 Subject: [PATCH 1/4] Improve dev setup in README Add needed packages in for dev setup in requirements.txt --- README.md | 58 ++++++++++++++++++++++++------------ application/requirements.txt | 3 ++ 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index f2b8ef6..eb4036e 100644 --- a/README.md +++ b/README.md @@ -73,30 +73,50 @@ To stop just run Ctrl + C ## Development environments -Spin up only 2 containers from docker-compose.yaml (by deleting all services except for redis and mongo) - -Make sure you have python 3.10 or 3.11 installed - -1. Navigate to `/application` folder -2. Run `docker-compose -f docker-compose-dev.yaml build && docker-compose -f docker-compose-dev.yaml up -d` -3. Export required variables -`export CELERY_BROKER_URL=redis://localhost:6379/0` -`export CELERY_RESULT_BACKEND=redis://localhost:6379/1` -`export MONGO_URI=mongodb://localhost:27017/docsgpt` -4. Install dependencies -`pip install -r requirements.txt` -5. Prepare .env file -Copy .env_sample and create .env with your openai api token -6. Run the app +### Spin up mongo and redis +For development only 2 containers are used from docker-compose.yaml (by deleting all services except for redis and mongo). +See file [docker-compose-dev.yaml](./docker-compose-dev.yaml). + +Run +``` +docker compose -f docker-compose-dev.yaml build +docker compose -f docker-compose-dev.yaml up -d +``` + +### Run the backend + +Make sure you have Python 3.10 or 3.11 installed. + +1. Export required environment variables +```commandline +export CELERY_BROKER_URL=redis://localhost:6379/0 +export CELERY_RESULT_BACKEND=redis://localhost:6379/1 +export MONGO_URI=mongodb://localhost:27017/docsgpt +``` +2. Prepare .env file +Copy `.env_sample` and create `.env` with your OpenAI API token +3. (optional) Create a python virtual environment +```commandline +python -m venv venv +. venv/bin/activate +``` +4. Change to `application/` subdir and install dependencies for the backend +```commandline +cd application/ +pip install -r requirements.txt +``` +5 . Run the app `python wsgi.py` -7. Start worker with `celery -A app.celery worker -l INFO` +6. Start worker with `celery -A app.celery worker -l INFO` + +### Start frontend +Make sure you have Node version 14+ -To start frontend 1. Navigate to `/frontend` folder 2. Install dependencies `npm install` -3. Run the app -4. `npm run dev` +3. Run the app +`npm run dev` [How to install the Chrome extension](https://github.com/arc53/docsgpt/wiki#launch-chrome-extension) diff --git a/application/requirements.txt b/application/requirements.txt index f6ff97b..dda008c 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -10,6 +10,7 @@ billiard==3.6.4.0 blobfile==2.0.1 boto3==1.26.102 botocore==1.29.102 +celery cffi==1.15.1 charset-normalizer==3.1.0 click==8.1.3 @@ -52,6 +53,7 @@ multidict==6.0.4 multiprocess==0.70.14 mypy-extensions==1.0.0 networkx==3.0 +npx nltk==3.8.1 numcodecs==0.11.0 numpy==1.24.2 @@ -93,6 +95,7 @@ SQLAlchemy==1.4.46 sympy==1.11.1 tenacity==8.2.2 threadpoolctl==3.1.0 +tiktoken torch==2.0.0 torchvision==0.15.1 tqdm==4.65.0 From 63d99d6a57d54283890d77f3998c40fae95b63f9 Mon Sep 17 00:00:00 2001 From: Levente Csoke Date: Thu, 15 Jun 2023 17:04:05 +0200 Subject: [PATCH 2/4] Change min node version to v16 Husky won't run with Node v14 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eb4036e..c59fa70 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ pip install -r requirements.txt 6. Start worker with `celery -A app.celery worker -l INFO` ### Start frontend -Make sure you have Node version 14+ +Make sure you have Node version 16+ 1. Navigate to `/frontend` folder 2. Install dependencies From d4ff6d4d7a38d10380f26ae95f34dc6755b484cb Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 15 Jun 2023 17:40:58 +0100 Subject: [PATCH 3/4] minor fix --- application/Dockerfile | 10 ++++------ application/requirements.txt | 3 ++- docker-compose.yaml | 9 +++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/application/Dockerfile b/application/Dockerfile index bd8f09d..3e2b1d6 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -8,18 +8,16 @@ RUN pip install --upgrade pip && pip install tiktoken==0.3.3 COPY requirements.txt . RUN pip install -r requirements.txt - FROM python:3.10-slim-bullseye -# Copy pre-built packages from builder stage -COPY --from=builder /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/ -RUN pip install gunicorn==20.1.0 -RUN pip install celery==5.2.7 + +# Copy pre-built packages and binaries from builder stage +COPY --from=builder /usr/local/ /usr/local/ + WORKDIR /app COPY . /app ENV FLASK_APP=app.py ENV FLASK_DEBUG=true - EXPOSE 5001 CMD ["gunicorn", "-w", "2", "--timeout", "120", "--bind", "0.0.0.0:5001", "wsgi:app"] diff --git a/application/requirements.txt b/application/requirements.txt index dda008c..ad80334 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -10,7 +10,7 @@ billiard==3.6.4.0 blobfile==2.0.1 boto3==1.26.102 botocore==1.29.102 -celery +celery==5.2.7 cffi==1.15.1 charset-normalizer==3.1.0 click==8.1.3 @@ -31,6 +31,7 @@ Flask==2.2.3 Flask-Cors==3.0.10 frozenlist==1.3.3 geojson==2.5.0 +gunicorn==20.1.0 greenlet==2.0.2 gpt4all==0.1.7 hub==3.0.1 diff --git a/docker-compose.yaml b/docker-compose.yaml index f36fd10..c9557dc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,15 @@ version: "3.9" services: + frontend: + build: ./frontend + environment: + - VITE_API_HOST=http://localhost:5001 + - VITE_API_STREAMING=$VITE_API_STREAMING + ports: + - "5173:5173" + depends_on: + - backend backend: build: ./application From a57cdfff1eba0e12a5dfa1061f66870b28cde9bc Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 15 Jun 2023 17:52:16 +0100 Subject: [PATCH 4/4] Update requirements.txt --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index ad80334..e06dbaf 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -90,7 +90,7 @@ s3transfer==0.6.0 scikit-learn==1.2.2 scipy==1.10.1 sentence-transformers==2.2.2 -sentencepiece==0.1.97 +sentencepiece six==1.16.0 SQLAlchemy==1.4.46 sympy==1.11.1