linting in docker and parallel make jobs

- linting can be run in docker in parallel with `make -j4 docker.lint`
This commit is contained in:
blob42 2023-05-18 20:09:55 +02:00
parent e1bf5d2f09
commit 3383c825b9
7 changed files with 65 additions and 23 deletions

View File

@ -144,3 +144,4 @@ dmypy.json
docker/ docker/
!docker/assets/ !docker/assets/
.dockerignore .dockerignore
docker.build

View File

@ -70,12 +70,14 @@ help:
@echo 'test_watch - run unit tests in watch mode' @echo 'test_watch - run unit tests in watch mode'
@echo 'integration_tests - run integration tests' @echo 'integration_tests - run integration tests'
ifneq ($(shell command -v docker 2> /dev/null),) ifneq ($(shell command -v docker 2> /dev/null),)
@echo 'docker_tests - run unit tests in docker' @echo 'docker_tests - run unit tests in docker (test only image)'
@echo 'docker - build and run the docker dev image' @echo 'docker - build and run the docker dev image'
@echo 'docker.run - run the docker dev image' @echo 'docker.run - run the docker dev image'
@echo 'docker.jupyter - start a jupyter notebook inside container' @echo 'docker.jupyter - start a jupyter notebook inside container'
@echo 'docker.build - build the docker dev image' @echo 'docker.build - build the docker dev image'
@echo 'docker.force_build - force a rebuild of the docker development image' @echo 'docker.force_build - force a rebuild'
@echo 'docker.test - run unit tests in docker (full dev image)'
@echo 'docker.lint - run linters in docker'
@echo 'docker.clean - remove the docker dev image' @echo 'docker.clean - remove the docker dev image'
endif endif

View File

@ -5,6 +5,9 @@ PYTHON_VERSION=3.10
# comment the following line to only install dev dependencies # comment the following line to only install dev dependencies
POETRY_EXTRA_PACKAGES="-E all" POETRY_EXTRA_PACKAGES="-E all"
# at least one group needed
POETRY_DEPENDENCIES="dev,test,lint,typing"
# langchain env. warning: these variables will be baked into the docker image ! # langchain env. warning: these variables will be baked into the docker image !
OPENAI_API_KEY=${OPENAI_API_KEY:-} OPENAI_API_KEY=${OPENAI_API_KEY:-}
SERPAPI_API_KEY=${SERPAPI_API_KEY:-} SERPAPI_API_KEY=${SERPAPI_API_KEY:-}

View File

@ -1,15 +1,17 @@
## Using Docker # Using Docker
To quickly get started, run the command `make docker`. To quickly get started, run the command `make docker`.
If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks. If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of available tasks.
### Building the development image There is a basic `docker-compose.yml` in the docker directory.
- use `make docker.run` will build the dev image if it does not exist. ## Building the development image
- `make docker.build`
#### Customizing the image and installed dependencies Using `make docker` will build the dev image if it does not exist, then drops
you inside the container with the langchain environment available in the shell.
### Customizing the image and installed dependencies
The image is built with a default python version and all extras and dev The image is built with a default python version and all extras and dev
dependencies. It can be customized by changing the variables in the [.env](/docker/.env) dependencies. It can be customized by changing the variables in the [.env](/docker/.env)
@ -18,13 +20,13 @@ file.
If you don't need all the `extra` dependencies a slimmer image can be obtained by If you don't need all the `extra` dependencies a slimmer image can be obtained by
commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file. commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file.
#### Image caching ### Image caching
The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code. The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.
### Example Usage ## Example Usage
All commands that in the python env are available by default in the container. All commands from langchain's python environment are available by default in the container.
A few examples: A few examples:
```bash ```bash
@ -37,3 +39,15 @@ docker run --rm -it IMG ipython
# start web server # start web server
docker run --rm -p 8888:8888 IMG python -m http.server 8888 docker run --rm -p 8888:8888 IMG python -m http.server 8888
``` ```
## Testing / Linting
Tests and lints are run using your local source directory that is mounted on the volume /src.
Run unit tests in the container with `make docker.test`.
Run the linting and formatting checks with `make docker.lint`.
Note: this task can run in parallel using `make -j4 docker.lint`.

View File

@ -32,8 +32,8 @@ VOLUME /src
####################### #######################
FROM lchain-base AS lchain-base-builder FROM lchain-base AS lchain-base-builder
ARG POETRY_EXTRA_PACKAGES=${POETRY_EXTRA_PACKAGES} ARG POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
ENV POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES ARG POETRY_DEPENDENCIES=$POETRY_DEPENDENCIES
ENV HOME=/root ENV HOME=/root
ENV POETRY_HOME=/root/.poetry ENV POETRY_HOME=/root/.poetry
@ -65,7 +65,7 @@ COPY poetry.* pyproject.toml ./
RUN mkdir /pip-prefix RUN mkdir /pip-prefix
RUN poetry export $POETRY_EXTRA_PACKAGES --with dev,test,lint -f requirements.txt --output requirements.txt --without-hashes && \ RUN poetry export $POETRY_EXTRA_PACKAGES --with $POETRY_DEPENDENCIES -f requirements.txt --output requirements.txt --without-hashes && \
pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt

View File

@ -1,9 +1,12 @@
#do not call this makefile it is included in the main Makefile #do not call this makefile it is included in the main Makefile
.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean .PHONY: docker docker.jupyter docker.run docker.force_build docker.clean \
docker.test docker.lint docker.lint.mypy docker.lint.black \
docker.lint.isort docker.lint.flake
# read python version from .env file ignoring comments # read python version from .env file ignoring comments
PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2) PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2)
POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2) POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2)
POETRY_DEPENDENCIES := $(shell grep 'POETRY_DEPENDENCIES' docker/.env | cut -d '=' -f2)
DOCKER_SRC := $(shell find docker -type f) DOCKER_SRC := $(shell find docker -type f)
@ -20,6 +23,8 @@ DOCKER_MOTD := docker/assets/etc/motd
ROOTDIR := $(shell git rev-parse --show-toplevel) ROOTDIR := $(shell git rev-parse --show-toplevel)
DOCKER_LINT_CMD = docker run --rm -i -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH)
docker: docker.run docker: docker.run
docker.run: docker.build docker.run: docker.build
@ -33,11 +38,13 @@ docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD)
ifdef $(DOCKER_BUILDKIT) ifdef $(DOCKER_BUILDKIT)
docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \ --build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
--progress=$(DOCKER_BUILD_PROGRESS) \ --progress=$(DOCKER_BUILD_PROGRESS) \
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . $(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
else else
docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \ docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \ --build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) . $(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
endif endif
docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
@ -55,10 +62,23 @@ docker.test: docker.build
docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) \ docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) \
pytest /src/tests/unit_tests pytest /src/tests/unit_tests
docker.lint: docker.build # this assumes that the docker image has been built
$(eval DOCKER_CMD = docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH)) docker.lint: docker.lint.mypy docker.lint.black docker.lint.isort \
docker.lint.flake
$(DOCKER_CMD) mypy /src # these can run in parallel with -j[njobs]
$(DOCKER_CMD) black /src --check docker.lint.mypy:
$(DOCKER_CMD) isort /src --check @$(DOCKER_LINT_CMD) mypy /src
$(DOCKER_CMD) flake8 /src --check @printf "\t%s\n" "mypy ... "
docker.lint.black:
@$(DOCKER_LINT_CMD) black /src --check
@printf "\t%s\n" "black ... "
docker.lint.isort:
@$(DOCKER_LINT_CMD) isort /src --check
@printf "\t%s\n" "isort ... "
docker.lint.flake:
@$(DOCKER_LINT_CMD) flake8 /src
@printf "\t%s\n" "flake8 ... "

View File

@ -9,6 +9,8 @@ services:
dockerfile: docker/Dockerfile dockerfile: docker/Dockerfile
args: args:
PYTHON_VERSION: ${PYTHON_VERSION} PYTHON_VERSION: ${PYTHON_VERSION}
POETRY_EXTRA_PACKAGES: ${POETRY_EXTRA_PACKAGES}
POETRY_DEPENDENCIES: ${POETRY_DEPENDENCIES}
restart: unless-stopped restart: unless-stopped
ports: ports: