forked from Archives/langchain
linting in docker and parallel make jobs
- linting can be run in docker in parallel with `make -j4 docker.lint`
This commit is contained in:
parent
e1bf5d2f09
commit
3383c825b9
@ -144,3 +144,4 @@ dmypy.json
|
|||||||
docker/
|
docker/
|
||||||
!docker/assets/
|
!docker/assets/
|
||||||
.dockerignore
|
.dockerignore
|
||||||
|
docker.build
|
||||||
|
6
Makefile
6
Makefile
@ -70,12 +70,14 @@ help:
|
|||||||
@echo 'test_watch - run unit tests in watch mode'
|
@echo 'test_watch - run unit tests in watch mode'
|
||||||
@echo 'integration_tests - run integration tests'
|
@echo 'integration_tests - run integration tests'
|
||||||
ifneq ($(shell command -v docker 2> /dev/null),)
|
ifneq ($(shell command -v docker 2> /dev/null),)
|
||||||
@echo 'docker_tests - run unit tests in docker'
|
@echo 'docker_tests - run unit tests in docker (test only image)'
|
||||||
@echo 'docker - build and run the docker dev image'
|
@echo 'docker - build and run the docker dev image'
|
||||||
@echo 'docker.run - run the docker dev image'
|
@echo 'docker.run - run the docker dev image'
|
||||||
@echo 'docker.jupyter - start a jupyter notebook inside container'
|
@echo 'docker.jupyter - start a jupyter notebook inside container'
|
||||||
@echo 'docker.build - build the docker dev image'
|
@echo 'docker.build - build the docker dev image'
|
||||||
@echo 'docker.force_build - force a rebuild of the docker development image'
|
@echo 'docker.force_build - force a rebuild'
|
||||||
|
@echo 'docker.test - run unit tests in docker (full dev image)'
|
||||||
|
@echo 'docker.lint - run linters in docker'
|
||||||
@echo 'docker.clean - remove the docker dev image'
|
@echo 'docker.clean - remove the docker dev image'
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
@ -5,6 +5,9 @@ PYTHON_VERSION=3.10
|
|||||||
# comment the following line to only install dev dependencies
|
# comment the following line to only install dev dependencies
|
||||||
POETRY_EXTRA_PACKAGES="-E all"
|
POETRY_EXTRA_PACKAGES="-E all"
|
||||||
|
|
||||||
|
# at least one group needed
|
||||||
|
POETRY_DEPENDENCIES="dev,test,lint,typing"
|
||||||
|
|
||||||
# langchain env. warning: these variables will be baked into the docker image !
|
# langchain env. warning: these variables will be baked into the docker image !
|
||||||
OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
||||||
SERPAPI_API_KEY=${SERPAPI_API_KEY:-}
|
SERPAPI_API_KEY=${SERPAPI_API_KEY:-}
|
||||||
|
@ -1,15 +1,17 @@
|
|||||||
## Using Docker
|
# Using Docker
|
||||||
|
|
||||||
To quickly get started, run the command `make docker`.
|
To quickly get started, run the command `make docker`.
|
||||||
|
|
||||||
If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks.
|
If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of available tasks.
|
||||||
|
|
||||||
### Building the development image
|
There is a basic `docker-compose.yml` in the docker directory.
|
||||||
|
|
||||||
- use `make docker.run` will build the dev image if it does not exist.
|
## Building the development image
|
||||||
- `make docker.build`
|
|
||||||
|
|
||||||
#### Customizing the image and installed dependencies
|
Using `make docker` will build the dev image if it does not exist, then drops
|
||||||
|
you inside the container with the langchain environment available in the shell.
|
||||||
|
|
||||||
|
### Customizing the image and installed dependencies
|
||||||
|
|
||||||
The image is built with a default python version and all extras and dev
|
The image is built with a default python version and all extras and dev
|
||||||
dependencies. It can be customized by changing the variables in the [.env](/docker/.env)
|
dependencies. It can be customized by changing the variables in the [.env](/docker/.env)
|
||||||
@ -18,13 +20,13 @@ file.
|
|||||||
If you don't need all the `extra` dependencies a slimmer image can be obtained by
|
If you don't need all the `extra` dependencies a slimmer image can be obtained by
|
||||||
commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file.
|
commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file.
|
||||||
|
|
||||||
#### Image caching
|
### Image caching
|
||||||
|
|
||||||
The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.
|
The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.
|
||||||
|
|
||||||
### Example Usage
|
## Example Usage
|
||||||
|
|
||||||
All commands that in the python env are available by default in the container.
|
All commands from langchain's python environment are available by default in the container.
|
||||||
|
|
||||||
A few examples:
|
A few examples:
|
||||||
```bash
|
```bash
|
||||||
@ -37,3 +39,15 @@ docker run --rm -it IMG ipython
|
|||||||
# start web server
|
# start web server
|
||||||
docker run --rm -p 8888:8888 IMG python -m http.server 8888
|
docker run --rm -p 8888:8888 IMG python -m http.server 8888
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Testing / Linting
|
||||||
|
|
||||||
|
Tests and lints are run using your local source directory that is mounted on the volume /src.
|
||||||
|
|
||||||
|
Run unit tests in the container with `make docker.test`.
|
||||||
|
|
||||||
|
Run the linting and formatting checks with `make docker.lint`.
|
||||||
|
|
||||||
|
Note: this task can run in parallel using `make -j4 docker.lint`.
|
||||||
|
|
||||||
|
|
||||||
|
@ -32,8 +32,8 @@ VOLUME /src
|
|||||||
#######################
|
#######################
|
||||||
FROM lchain-base AS lchain-base-builder
|
FROM lchain-base AS lchain-base-builder
|
||||||
|
|
||||||
ARG POETRY_EXTRA_PACKAGES=${POETRY_EXTRA_PACKAGES}
|
ARG POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
|
||||||
ENV POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
|
ARG POETRY_DEPENDENCIES=$POETRY_DEPENDENCIES
|
||||||
|
|
||||||
ENV HOME=/root
|
ENV HOME=/root
|
||||||
ENV POETRY_HOME=/root/.poetry
|
ENV POETRY_HOME=/root/.poetry
|
||||||
@ -65,7 +65,7 @@ COPY poetry.* pyproject.toml ./
|
|||||||
|
|
||||||
RUN mkdir /pip-prefix
|
RUN mkdir /pip-prefix
|
||||||
|
|
||||||
RUN poetry export $POETRY_EXTRA_PACKAGES --with dev,test,lint -f requirements.txt --output requirements.txt --without-hashes && \
|
RUN poetry export $POETRY_EXTRA_PACKAGES --with $POETRY_DEPENDENCIES -f requirements.txt --output requirements.txt --without-hashes && \
|
||||||
pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt
|
pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
#do not call this makefile it is included in the main Makefile
|
#do not call this makefile it is included in the main Makefile
|
||||||
.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean
|
.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean \
|
||||||
|
docker.test docker.lint docker.lint.mypy docker.lint.black \
|
||||||
|
docker.lint.isort docker.lint.flake
|
||||||
|
|
||||||
# read python version from .env file ignoring comments
|
# read python version from .env file ignoring comments
|
||||||
PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2)
|
PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2)
|
||||||
POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2)
|
POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2)
|
||||||
|
POETRY_DEPENDENCIES := $(shell grep 'POETRY_DEPENDENCIES' docker/.env | cut -d '=' -f2)
|
||||||
|
|
||||||
|
|
||||||
DOCKER_SRC := $(shell find docker -type f)
|
DOCKER_SRC := $(shell find docker -type f)
|
||||||
@ -20,6 +23,8 @@ DOCKER_MOTD := docker/assets/etc/motd
|
|||||||
|
|
||||||
ROOTDIR := $(shell git rev-parse --show-toplevel)
|
ROOTDIR := $(shell git rev-parse --show-toplevel)
|
||||||
|
|
||||||
|
DOCKER_LINT_CMD = docker run --rm -i -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH)
|
||||||
|
|
||||||
docker: docker.run
|
docker: docker.run
|
||||||
|
|
||||||
docker.run: docker.build
|
docker.run: docker.build
|
||||||
@ -33,11 +38,13 @@ docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD)
|
|||||||
ifdef $(DOCKER_BUILDKIT)
|
ifdef $(DOCKER_BUILDKIT)
|
||||||
docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
|
docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
|
||||||
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
|
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
|
||||||
|
--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
|
||||||
--progress=$(DOCKER_BUILD_PROGRESS) \
|
--progress=$(DOCKER_BUILD_PROGRESS) \
|
||||||
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
|
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
|
||||||
else
|
else
|
||||||
docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
|
docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
|
||||||
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
|
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
|
||||||
|
--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
|
||||||
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
|
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
|
||||||
endif
|
endif
|
||||||
docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
|
docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
|
||||||
@ -55,10 +62,23 @@ docker.test: docker.build
|
|||||||
docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) \
|
docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) \
|
||||||
pytest /src/tests/unit_tests
|
pytest /src/tests/unit_tests
|
||||||
|
|
||||||
docker.lint: docker.build
|
# this assumes that the docker image has been built
|
||||||
$(eval DOCKER_CMD = docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH))
|
docker.lint: docker.lint.mypy docker.lint.black docker.lint.isort \
|
||||||
|
docker.lint.flake
|
||||||
|
|
||||||
$(DOCKER_CMD) mypy /src
|
# these can run in parallel with -j[njobs]
|
||||||
$(DOCKER_CMD) black /src --check
|
docker.lint.mypy:
|
||||||
$(DOCKER_CMD) isort /src --check
|
@$(DOCKER_LINT_CMD) mypy /src
|
||||||
$(DOCKER_CMD) flake8 /src --check
|
@printf "\t%s\n" "mypy ... "
|
||||||
|
|
||||||
|
docker.lint.black:
|
||||||
|
@$(DOCKER_LINT_CMD) black /src --check
|
||||||
|
@printf "\t%s\n" "black ... "
|
||||||
|
|
||||||
|
docker.lint.isort:
|
||||||
|
@$(DOCKER_LINT_CMD) isort /src --check
|
||||||
|
@printf "\t%s\n" "isort ... "
|
||||||
|
|
||||||
|
docker.lint.flake:
|
||||||
|
@$(DOCKER_LINT_CMD) flake8 /src
|
||||||
|
@printf "\t%s\n" "flake8 ... "
|
||||||
|
@ -9,6 +9,8 @@ services:
|
|||||||
dockerfile: docker/Dockerfile
|
dockerfile: docker/Dockerfile
|
||||||
args:
|
args:
|
||||||
PYTHON_VERSION: ${PYTHON_VERSION}
|
PYTHON_VERSION: ${PYTHON_VERSION}
|
||||||
|
POETRY_EXTRA_PACKAGES: ${POETRY_EXTRA_PACKAGES}
|
||||||
|
POETRY_DEPENDENCIES: ${POETRY_DEPENDENCIES}
|
||||||
|
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
ports:
|
ports:
|
||||||
|
Loading…
Reference in New Issue
Block a user