linting in docker and parallel make jobs

- linting can be run in docker in parallel with `make -j4 docker.lint`
12 months ago · 3383c825b9
parent e1bf5d2f09
commit 3383c825b9
7 changed files with 65 additions and 23 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -144,3 +144,4 @@ dmypy.json
 docker/
 !docker/assets/
 .dockerignore
+docker.build
--- a/10
+++ b/10
@ -70,13 +70,15 @@ help:
 	@echo 'test_watch                   - run unit tests in watch mode'
 	@echo 'integration_tests            - run integration tests'
 ifneq ($(shell command -v docker 2> /dev/null),)
-	@echo 'docker_tests                 - run unit tests in docker'
+	@echo 'docker_tests                 - run unit tests in docker (test only image)'
 	@echo 'docker                       - build and run the docker dev image'
 	@echo 'docker.run                   - run the docker dev image'
-	@echo 'docker.jupyter 	            - start a jupyter notebook inside container'
+	@echo 'docker.jupyter               - start a jupyter notebook inside container'
 	@echo 'docker.build                 - build the docker dev image'
-	@echo 'docker.force_build           - force a rebuild of the docker development image'
-	@echo 'docker.clean 	            - remove the docker dev image'
+	@echo 'docker.force_build           - force a rebuild'
+	@echo 'docker.test                  - run unit tests in docker (full dev image)'
+	@echo 'docker.lint                  - run linters in docker'
+	@echo 'docker.clean                 - remove the docker dev image'
 endif

 # include the following makefile if the docker executable is available
--- a/docker/.env
+++ b/docker/.env
@ -5,6 +5,9 @@ PYTHON_VERSION=3.10
 # comment the following line to only install dev dependencies
 POETRY_EXTRA_PACKAGES="-E all"

+# at least one group needed
+POETRY_DEPENDENCIES="dev,test,lint,typing"
+
 # langchain env. warning: these variables will be baked into the docker image !
 OPENAI_API_KEY=${OPENAI_API_KEY:-}
 SERPAPI_API_KEY=${SERPAPI_API_KEY:-}
--- a/docker/DOCKER.md
+++ b/docker/DOCKER.md
@ -1,15 +1,17 @@
-## Using Docker
+# Using Docker

 To quickly get started, run the command `make docker`.

-If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks.
+If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of available tasks.

-### Building the development image
+There is a basic `docker-compose.yml` in the docker directory.

- use `make docker.run` will build the dev image if it does not exist.
- `make docker.build` 
+## Building the development image

-#### Customizing the image and installed dependencies
+Using `make docker` will build the dev image if it does not exist, then drops
+you inside the container with the langchain environment available in the shell.
+
+### Customizing the image and installed dependencies

 The image is built with a default python version and all extras and dev
 dependencies. It can be customized by changing the variables in the [.env](/docker/.env)
@ -18,13 +20,13 @@ file.
 If you don't need all the `extra` dependencies a slimmer image can be obtained by 
 commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file.

-#### Image caching
+### Image caching

 The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.

-### Example Usage
+## Example Usage

-All commands that in the python env are available by default in the container.
+All commands from langchain's python environment are available by default in the container.

 A few examples:
 ```bash
@ -37,3 +39,15 @@ docker run --rm -it IMG ipython
 # start web server
 docker run --rm -p 8888:8888 IMG python -m http.server 8888
 ```
+
+## Testing / Linting
+
+Tests and lints are run using your local source directory that is mounted on the volume /src.
+
+Run unit tests in the container with `make docker.test`.
+
+Run the linting and formatting checks with `make docker.lint`.
+
+Note: this task can run in parallel using `make -j4 docker.lint`.
+
+
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -32,8 +32,8 @@ VOLUME /src
 #######################
 FROM lchain-base AS lchain-base-builder

-ARG POETRY_EXTRA_PACKAGES=${POETRY_EXTRA_PACKAGES}
-ENV POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
+ARG POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
+ARG POETRY_DEPENDENCIES=$POETRY_DEPENDENCIES

 ENV HOME=/root
 ENV POETRY_HOME=/root/.poetry
@ -65,7 +65,7 @@ COPY poetry.* pyproject.toml ./

 RUN mkdir /pip-prefix

-RUN poetry export $POETRY_EXTRA_PACKAGES --with dev,test,lint -f requirements.txt --output requirements.txt --without-hashes && \
+RUN poetry export $POETRY_EXTRA_PACKAGES --with $POETRY_DEPENDENCIES -f requirements.txt --output requirements.txt --without-hashes && \
    pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt


--- a/docker/Makefile
+++ b/docker/Makefile
@ -1,9 +1,12 @@
 #do not call this makefile it is included in the main Makefile
-.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean
+.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean \
+	docker.test docker.lint docker.lint.mypy docker.lint.black \
+	docker.lint.isort docker.lint.flake

 # read python version from .env file ignoring comments
 PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2)
 POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2)
+POETRY_DEPENDENCIES := $(shell grep 'POETRY_DEPENDENCIES' docker/.env | cut -d '=' -f2)


 DOCKER_SRC := $(shell find docker -type f)
@ -20,6 +23,8 @@ DOCKER_MOTD := docker/assets/etc/motd

 ROOTDIR := $(shell git rev-parse --show-toplevel)

+DOCKER_LINT_CMD = docker run --rm -i -u lchain -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH)
+
 docker: docker.run

 docker.run: docker.build
@ -33,11 +38,13 @@ docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD)
 ifdef $(DOCKER_BUILDKIT)
 	docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
 			--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
+			--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
 			--progress=$(DOCKER_BUILD_PROGRESS) \
 			$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
 else
 	docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
 			--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
+			--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
 			$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
 endif
 	docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
@ -55,10 +62,23 @@ docker.test: docker.build
 	docker run --rm -it -u lchain -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH) \
 		pytest /src/tests/unit_tests

-docker.lint: docker.build
-	$(eval DOCKER_CMD = docker run --rm -it -u lchain -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH))
+# this assumes that the docker image has been built 
+docker.lint: docker.lint.mypy docker.lint.black docker.lint.isort \
+	docker.lint.flake
+
+# these can run in parallel with -j[njobs]
+docker.lint.mypy:
+	@$(DOCKER_LINT_CMD) mypy /src
+	@printf "\t%s\n" "mypy ... "
+
+docker.lint.black:
+	@$(DOCKER_LINT_CMD) black /src --check
+	@printf "\t%s\n" "black ... "
+
+docker.lint.isort:
+	@$(DOCKER_LINT_CMD) isort /src --check
+	@printf "\t%s\n" "isort ... "

-	$(DOCKER_CMD) mypy /src
-	$(DOCKER_CMD) black /src --check
-	$(DOCKER_CMD) isort /src --check
-	$(DOCKER_CMD) flake8 /src --check
+docker.lint.flake:
+	@$(DOCKER_LINT_CMD) flake8 /src
+	@printf "\t%s\n" "flake8 ... "
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@ -9,6 +9,8 @@ services:
      dockerfile: docker/Dockerfile
      args:
        PYTHON_VERSION: ${PYTHON_VERSION}
+        POETRY_EXTRA_PACKAGES: ${POETRY_EXTRA_PACKAGES}
+        POETRY_DEPENDENCIES: ${POETRY_DEPENDENCIES}

    restart: unless-stopped
    ports: