From 3383c825b9d220513db2fa42ddffe2a58f3ed377 Mon Sep 17 00:00:00 2001 From: blob42 Date: Thu, 18 May 2023 20:09:55 +0200 Subject: [PATCH] linting in docker and parallel make jobs - linting can be run in docker in parallel with `make -j4 docker.lint` --- .dockerignore | 1 + Makefile | 10 ++++++---- docker/.env | 3 +++ docker/DOCKER.md | 32 +++++++++++++++++++++++--------- docker/Dockerfile | 6 +++--- docker/Makefile | 34 +++++++++++++++++++++++++++------- docker/docker-compose.yml | 2 ++ 7 files changed, 65 insertions(+), 23 deletions(-) diff --git a/.dockerignore b/.dockerignore index 708f09df..e4d6e968 100644 --- a/.dockerignore +++ b/.dockerignore @@ -144,3 +144,4 @@ dmypy.json docker/ !docker/assets/ .dockerignore +docker.build diff --git a/Makefile b/Makefile index a033ed2c..3268b7a7 100644 --- a/Makefile +++ b/Makefile @@ -70,13 +70,15 @@ help: @echo 'test_watch - run unit tests in watch mode' @echo 'integration_tests - run integration tests' ifneq ($(shell command -v docker 2> /dev/null),) - @echo 'docker_tests - run unit tests in docker' + @echo 'docker_tests - run unit tests in docker (test only image)' @echo 'docker - build and run the docker dev image' @echo 'docker.run - run the docker dev image' - @echo 'docker.jupyter - start a jupyter notebook inside container' + @echo 'docker.jupyter - start a jupyter notebook inside container' @echo 'docker.build - build the docker dev image' - @echo 'docker.force_build - force a rebuild of the docker development image' - @echo 'docker.clean - remove the docker dev image' + @echo 'docker.force_build - force a rebuild' + @echo 'docker.test - run unit tests in docker (full dev image)' + @echo 'docker.lint - run linters in docker' + @echo 'docker.clean - remove the docker dev image' endif # include the following makefile if the docker executable is available diff --git a/docker/.env b/docker/.env index f37256cc..5e6c745f 100644 --- a/docker/.env +++ b/docker/.env @@ -5,6 +5,9 @@ PYTHON_VERSION=3.10 # comment the following line to only install dev dependencies POETRY_EXTRA_PACKAGES="-E all" +# at least one group needed +POETRY_DEPENDENCIES="dev,test,lint,typing" + # langchain env. warning: these variables will be baked into the docker image ! OPENAI_API_KEY=${OPENAI_API_KEY:-} SERPAPI_API_KEY=${SERPAPI_API_KEY:-} diff --git a/docker/DOCKER.md b/docker/DOCKER.md index 382911c3..70ad03c7 100644 --- a/docker/DOCKER.md +++ b/docker/DOCKER.md @@ -1,15 +1,17 @@ -## Using Docker +# Using Docker To quickly get started, run the command `make docker`. -If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks. +If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of available tasks. -### Building the development image +There is a basic `docker-compose.yml` in the docker directory. -- use `make docker.run` will build the dev image if it does not exist. -- `make docker.build` +## Building the development image -#### Customizing the image and installed dependencies +Using `make docker` will build the dev image if it does not exist, then drops +you inside the container with the langchain environment available in the shell. + +### Customizing the image and installed dependencies The image is built with a default python version and all extras and dev dependencies. It can be customized by changing the variables in the [.env](/docker/.env) @@ -18,13 +20,13 @@ file. If you don't need all the `extra` dependencies a slimmer image can be obtained by commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file. -#### Image caching +### Image caching The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code. -### Example Usage +## Example Usage -All commands that in the python env are available by default in the container. +All commands from langchain's python environment are available by default in the container. A few examples: ```bash @@ -37,3 +39,15 @@ docker run --rm -it IMG ipython # start web server docker run --rm -p 8888:8888 IMG python -m http.server 8888 ``` + +## Testing / Linting + +Tests and lints are run using your local source directory that is mounted on the volume /src. + +Run unit tests in the container with `make docker.test`. + +Run the linting and formatting checks with `make docker.lint`. + +Note: this task can run in parallel using `make -j4 docker.lint`. + + diff --git a/docker/Dockerfile b/docker/Dockerfile index 1711430f..7cd58756 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,8 +32,8 @@ VOLUME /src ####################### FROM lchain-base AS lchain-base-builder -ARG POETRY_EXTRA_PACKAGES=${POETRY_EXTRA_PACKAGES} -ENV POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES +ARG POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES +ARG POETRY_DEPENDENCIES=$POETRY_DEPENDENCIES ENV HOME=/root ENV POETRY_HOME=/root/.poetry @@ -65,7 +65,7 @@ COPY poetry.* pyproject.toml ./ RUN mkdir /pip-prefix -RUN poetry export $POETRY_EXTRA_PACKAGES --with dev,test,lint -f requirements.txt --output requirements.txt --without-hashes && \ +RUN poetry export $POETRY_EXTRA_PACKAGES --with $POETRY_DEPENDENCIES -f requirements.txt --output requirements.txt --without-hashes && \ pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt diff --git a/docker/Makefile b/docker/Makefile index e44dec11..80449e34 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,9 +1,12 @@ #do not call this makefile it is included in the main Makefile -.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean +.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean \ + docker.test docker.lint docker.lint.mypy docker.lint.black \ + docker.lint.isort docker.lint.flake # read python version from .env file ignoring comments PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2) POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2) +POETRY_DEPENDENCIES := $(shell grep 'POETRY_DEPENDENCIES' docker/.env | cut -d '=' -f2) DOCKER_SRC := $(shell find docker -type f) @@ -20,6 +23,8 @@ DOCKER_MOTD := docker/assets/etc/motd ROOTDIR := $(shell git rev-parse --show-toplevel) +DOCKER_LINT_CMD = docker run --rm -i -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) + docker: docker.run docker.run: docker.build @@ -33,11 +38,13 @@ docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD) ifdef $(DOCKER_BUILDKIT) docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ --build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \ + --build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \ --progress=$(DOCKER_BUILD_PROGRESS) \ $(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . else docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ --build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \ + --build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \ $(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . endif docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest @@ -55,10 +62,23 @@ docker.test: docker.build docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) \ pytest /src/tests/unit_tests -docker.lint: docker.build - $(eval DOCKER_CMD = docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH)) +# this assumes that the docker image has been built +docker.lint: docker.lint.mypy docker.lint.black docker.lint.isort \ + docker.lint.flake - $(DOCKER_CMD) mypy /src - $(DOCKER_CMD) black /src --check - $(DOCKER_CMD) isort /src --check - $(DOCKER_CMD) flake8 /src --check +# these can run in parallel with -j[njobs] +docker.lint.mypy: + @$(DOCKER_LINT_CMD) mypy /src + @printf "\t%s\n" "mypy ... " + +docker.lint.black: + @$(DOCKER_LINT_CMD) black /src --check + @printf "\t%s\n" "black ... " + +docker.lint.isort: + @$(DOCKER_LINT_CMD) isort /src --check + @printf "\t%s\n" "isort ... " + +docker.lint.flake: + @$(DOCKER_LINT_CMD) flake8 /src + @printf "\t%s\n" "flake8 ... " diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 09a4e9ee..adef20f1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,6 +9,8 @@ services: dockerfile: docker/Dockerfile args: PYTHON_VERSION: ${PYTHON_VERSION} + POETRY_EXTRA_PACKAGES: ${POETRY_EXTRA_PACKAGES} + POETRY_DEPENDENCIES: ${POETRY_DEPENDENCIES} restart: unless-stopped ports: