diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ab20258c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,143 @@ +.vscode/ +.idea/ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints +notebooks/ + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +.venvs +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# macOS display setting files +.DS_Store + + + +# docker +docker/ +!docker/assets/ +.dockerignore diff --git a/.gitignore b/.gitignore index 2a7a13ed..23dcaf1d 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,7 @@ celerybeat.pid # Environments .env +!docker/.env .venv .venvs env/ @@ -134,3 +135,4 @@ dmypy.json # macOS display setting files .DS_Store +docker.build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab47c01b..8d8def95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,6 +151,37 @@ poetry run jupyter notebook When you run `poetry install`, the `langchain` package is installed as editable in the virtualenv, so your new logic can be imported into the notebook. +## Using Docker + +To quickly get started, run the command `make docker`. + +If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks. + +### Building the development image + +- use `make docker.run` will build the dev image if it does not exist. +- `make docker.build` + +#### Image caching + +The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code. + +### Examples + +All commands that in the python env are available by default in the container. + +A few examples: +```bash +# run jupyter notebook +docker run --rm -it IMG jupyter notebook + +# run ipython +docker run --rm -it IMG ipython + +# start web server +docker run --rm -p 8888:8888 IMG python -m http.server 8888 +``` + ## Documentation ### Contribute Documentation diff --git a/Makefile b/Makefile index 3cc2b846..0be5c081 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ .PHONY: all clean format lint test tests test_watch integration_tests help +GIT_HASH ?= $(shell git rev-parse --short HEAD) +LANGCHAIN_VERSION := $(shell grep '^version' pyproject.toml | cut -d '=' -f2 | tr -d '"') + all: help coverage: @@ -32,8 +35,7 @@ lint: test: poetry run pytest tests/unit_tests -tests: - poetry run pytest tests/unit_tests +tests: test test_watch: poetry run ptw --now . -- tests/unit_tests @@ -47,8 +49,22 @@ help: @echo 'docs_build - build the documentation' @echo 'docs_clean - clean the documentation build artifacts' @echo 'docs_linkcheck - run linkchecker on the documentation' +ifneq ($(shell command -v docker 2> /dev/null),) + @echo 'docker - build and run the docker dev image' + @echo 'docker.run - run the docker dev image' + @echo 'docker.build - build the docker dev image' + @echo 'docker.force_build - force a rebuild of the docker development image' +endif @echo 'format - run code formatters' @echo 'lint - run linters' @echo 'test - run unit tests' @echo 'test_watch - run unit tests in watch mode' @echo 'integration_tests - run integration tests' + +# include the following makefile if the docker executable is available +ifeq ($(shell command -v docker 2> /dev/null),) + $(info Docker not found, skipping docker-related targets) +else +include docker/Makefile +endif + diff --git a/docker/.env b/docker/.env new file mode 100644 index 00000000..10d0792b --- /dev/null +++ b/docker/.env @@ -0,0 +1,6 @@ +# python env +PYTHON_VERSION=3.11.2 + +# langchain env +OPENAI_API_KEY=${OPENAI_API_KEY:-} +SERPAPI_API_KEY=${SERPAPI_API_KEY:-} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..35c83c4e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,102 @@ +# vim: ft=dockerfile +# +# see also: https://github.com/python-poetry/poetry/discussions/1879 +# - with https://github.com/bneijt/poetry-lock-docker +# see https://github.com/thehale/docker-python-poetry +# see https://github.com/max-pfeiffer/uvicorn-poetry + +# use by default the slim version of python +ARG PYTHON_IMAGE_TAG=slim +ARG PYTHON_VERSION=${PYTHON_VERSION:-3.11.2} + +#################### +# Base Environment +#################### +FROM python:$PYTHON_VERSION-$PYTHON_IMAGE_TAG AS lchain-base + +ARG UID=1000 +ARG USERNAME=lchain + +ENV USERNAME=$USERNAME + +RUN groupadd -g ${UID} $USERNAME +RUN useradd -l -m -u ${UID} -g ${UID} $USERNAME + +# used for mounting source code +RUN mkdir /src +VOLUME /src + + +####################### +## Poetry Builder Image +####################### +FROM lchain-base AS lchain-base-builder + +ENV HOME=/root +ENV POETRY_HOME=/root/.poetry +ENV POETRY_VIRTUALENVS_IN_PROJECT=false +ENV POETRY_NO_INTERACTION=1 +ENV CACHE_DIR=$HOME/.cache +ENV POETRY_CACHE_DIR=$CACHE_DIR/pypoetry +ENV PATH="$POETRY_HOME/bin:$PATH" + +WORKDIR /root + +RUN apt-get update && \ + apt-get install -y \ + git \ + curl + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN mkdir -p $CACHE_DIR + +## setup poetry +RUN curl -sSL -o $CACHE_DIR/pypoetry-installer.py https://install.python-poetry.org/ +RUN python3 $CACHE_DIR/pypoetry-installer.py + + +# # Copy poetry files +COPY poetry.* pyproject.toml ./ + +RUN mkdir /pip-prefix + +RUN poetry export --with dev -f requirements.txt --output requirements.txt --without-hashes && \ + pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt + + +# add custom motd message +COPY docker/assets/etc/motd /tmp/motd +RUN cat /tmp/motd > /etc/motd + +RUN printf "\n%s\n%s\n" "$(poetry version)" "$(python --version)" >> /etc/motd + + + +################### +## Runtime Image +################### +FROM lchain-base AS lchain + +#jupyter port +EXPOSE 8888 + +COPY docker/assets/entry.sh /entry +RUN chmod +x /entry + +COPY --from=lchain-base-builder /etc/motd /etc/motd +COPY --from=lchain-base-builder /usr/bin/git /usr/bin/git + +USER ${USERNAME:-lchain} +ENV HOME /home/$USERNAME +WORKDIR /home/$USERNAME + +COPY --chown=lchain:lchain --from=lchain-base-builder /pip-prefix $HOME/.local/ + +COPY . . + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN pip install --no-deps --disable-pip-version-check --no-cache-dir -e . + + +entrypoint ["/entry"] diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000..16b855b1 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,47 @@ +#do not call this makefile it is included in the main Makefile +.PHONY: docker docker.jupyter docker.run docker.force_build + +# read python version from .env file +PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2) + + +DOCKER_SRC := $(shell find docker -type f -not -name .env) +DOCKER_IMAGE_NAME = langchain/dev + +# SRC is all files matched by the git ls-files command +SRC := $(shell git ls-files -- '*' ':!:docker/*') + +# set DOCKER_BUILD_PROGRESS=plain to see detailed build progress +DOCKER_BUILD_PROGRESS ?= auto + +# extra message to show when entering the docker container +DOCKER_MOTD := docker/assets/etc/motd + +ROOTDIR := $(shell git rev-parse --show-toplevel) + +docker: docker.run + +docker.run: docker.build + @echo "Docker image: $(DOCKER_IMAGE_NAME):$(GIT_HASH)" + @docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) + @# $(local source mounted at $(ROOTDIR) at /src) + +docker.jupyter: docker.build + @docker run --rm -it -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) jupyter notebook + +docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD) +ifdef $(DOCKER_BUILDKIT) + @docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ + --progress=$(DOCKER_BUILD_PROGRESS) \ + -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . +else + @docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ + -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . +endif + @docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest + @touch $@ # this avoids docker rebuilds the build dependencies have not changed + @ # you can remove the file docker_build to force a rebuild + +docker.force_build: $(DOCKER_SRC) + @docker build --no-cache -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . + diff --git a/docker/assets/entry.sh b/docker/assets/entry.sh new file mode 100644 index 00000000..46b81af1 --- /dev/null +++ b/docker/assets/entry.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +export PATH=$HOME/.local/bin:$PATH + +if [ -z "$1" ]; then + cat /etc/motd + exec /bin/bash +fi + +exec "$@" diff --git a/docker/assets/etc/motd b/docker/assets/etc/motd new file mode 100644 index 00000000..35207410 --- /dev/null +++ b/docker/assets/etc/motd @@ -0,0 +1,9 @@ +The dependencies have been installed in the current shell. +There is no virtualenv or a need for `poetry` inside the +container. + +Running the command `make docker.run` at the root directory +of the project will build the container the first time. On +the next runs, it will use the cached image or rebuild it if +a change is made on the source code. + diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..09a4e9ee --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,15 @@ +version: "3.7" + +services: + langchain: + hostname: langchain + image: langchain/dev:latest + build: + context: ../ + dockerfile: docker/Dockerfile + args: + PYTHON_VERSION: ${PYTHON_VERSION} + + restart: unless-stopped + ports: + - 127.0.0.1:8888:8888