added all extra dependencies to dev image + customized builds

- downgraded to python 3.10 to accomadate installing all dependencies
- by default installs all dev + extra dependencies
- option to install only dev dependencies by customizing .env file

CONTRIBUTING.md

@@ -153,34 +153,7 @@ When you run `poetry install`, the `langchain` package is installed as editable
 
 ## Using Docker
 
-To quickly get started, run the command `make docker`.
+Refer to [DOCKER.md](docker/DOCKER.md) for more information.
-
-If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks.
-
-### Building the development image
-
-- use `make docker.run` will build the dev image if it does not exist.
-- `make docker.build` 
-
-#### Image caching
-
-The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.
-
-### Examples
-
-All commands that in the python env are available by default in the container.
-
-A few examples:
-```bash
-# run jupyter notebook
-docker run --rm -it IMG jupyter notebook
-
-# run ipython
-docker run --rm -it IMG ipython
-
-# start web server
-docker run --rm -p 8888:8888 IMG python -m http.server 8888
-```
 
 ## Documentation
 

Makefile

@@ -51,8 +51,10 @@ help:
 ifneq ($(shell command -v docker 2> /dev/null),)
 	@echo 'docker              - build and run the docker dev image'
 	@echo 'docker.run          - run the docker dev image'
+	@echo 'docker.jupyter 	   - start a jupyter notebook inside container'
 	@echo 'docker.build        - build the docker dev image'
 	@echo 'docker.force_build  - force a rebuild of the docker development image'
+	@echo 'docker.clean 	   - remove the docker dev image'
 endif
 	@echo 'format              - run code formatters'
 	@echo 'lint                - run linters'

docker/.env

@@ -1,6 +1,10 @@
 # python env
-PYTHON_VERSION=3.11.2
+PYTHON_VERSION=3.10
 
-# langchain env
+# -E flag is required
+# comment the following line to only install dev dependencies
+POETRY_EXTRA_PACKAGES="-E all"
+
+# langchain env. warning: these variables will be baked into the docker image !
 OPENAI_API_KEY=${OPENAI_API_KEY:-}
 SERPAPI_API_KEY=${SERPAPI_API_KEY:-}

docker/DOCKER.md

@@ -0,0 +1,39 @@
+## Using Docker
+
+To quickly get started, run the command `make docker`.
+
+If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of common tasks.
+
+### Building the development image
+
+- use `make docker.run` will build the dev image if it does not exist.
+- `make docker.build` 
+
+#### Customizing the image and installed dependencies
+
+The image is built with a default python version and all extras and dev
+dependencies. It can be customized by changing the variables in the [.env](/docker/.env)
+file. 
+
+If you don't need all the `extra` dependencies a slimmer image can be obtained by 
+commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file.
+
+#### Image caching
+
+The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.
+
+### Example Usage
+
+All commands that in the python env are available by default in the container.
+
+A few examples:
+```bash
+# run jupyter notebook
+docker run --rm -it IMG jupyter notebook
+
+# run ipython
+docker run --rm -it IMG ipython
+
+# start web server
+docker run --rm -p 8888:8888 IMG python -m http.server 8888
+```

docker/Dockerfile

@@ -32,6 +32,9 @@ VOLUME /src
 #######################
 FROM lchain-base AS lchain-base-builder
 
+ARG POETRY_EXTRA_PACKAGES=${POETRY_EXTRA_PACKAGES}
+ENV POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
+
 ENV HOME=/root
 ENV POETRY_HOME=/root/.poetry
 ENV POETRY_VIRTUALENVS_IN_PROJECT=false
@@ -44,6 +47,7 @@ WORKDIR /root
 
 RUN apt-get update && \
     apt-get install -y \
+    build-essential \
     git \
     curl
 
@@ -61,7 +65,7 @@ COPY poetry.* pyproject.toml ./
 
 RUN mkdir /pip-prefix
 
-RUN poetry export --with dev -f requirements.txt --output requirements.txt --without-hashes && \
+RUN poetry export $POETRY_EXTRA_PACKAGES --with dev -f requirements.txt --output requirements.txt --without-hashes && \
     pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt
 
 
@@ -71,8 +75,6 @@ RUN cat /tmp/motd > /etc/motd
 
 RUN printf "\n%s\n%s\n" "$(poetry version)" "$(python --version)" >> /etc/motd
 
-
-
 ###################
 ## Runtime Image
 ###################

docker/Makefile

@@ -1,11 +1,12 @@
 #do not call this makefile it is included in the main Makefile
-.PHONY: docker docker.jupyter docker.run docker.force_build
+.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean
 
-# read python version from .env file
+# read python version from .env file ignoring comments
 PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2)
+POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2)
 
 
-DOCKER_SRC := $(shell find docker -type f -not -name .env)
+DOCKER_SRC := $(shell find docker -type f)
 DOCKER_IMAGE_NAME = langchain/dev
 
 # SRC is all files matched by the git ls-files command
@@ -23,25 +24,29 @@ docker: docker.run
 
 docker.run: docker.build
 	@echo "Docker image: $(DOCKER_IMAGE_NAME):$(GIT_HASH)"
-	@docker run --rm -it -u lchain -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH)
+	docker run --rm -it -u lchain -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH)
-	@# $(local source mounted at $(ROOTDIR) at /src)
 
 docker.jupyter: docker.build
-	@docker run --rm -it -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH) jupyter notebook
+	docker run --rm -it -v $(ROOTDIR):/src  $(DOCKER_IMAGE_NAME):$(GIT_HASH) jupyter notebook
 
 docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD)
 ifdef $(DOCKER_BUILDKIT)
-	@docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
+	docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
+			--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
 			--progress=$(DOCKER_BUILD_PROGRESS) \
-			-f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
+			$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
 else
-	@docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
+	docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
-			-f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
+			--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
+			$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
 endif
-	@docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
+	docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
-	@touch $@ # this avoids docker rebuilds the build dependencies have not changed
+	@touch $@ # this prevents docker from rebuilding dependencies that have not 
-	@		  # you can remove the file docker_build to force a rebuild
+	@         #  changed. Remove the file `docker/docker.build` to force a rebuild.
 
 docker.force_build: $(DOCKER_SRC)
-	@docker build --no-cache -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
+	@rm -f docker.build
+	@$(MAKE) docker.build BUILD_FLAGS=--no-cache
 
+docker.clean:
+	docker rmi $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest

docker/assets/etc/motd

@@ -1,9 +1,8 @@
-The dependencies have been installed in the current shell.
+All dependencies have been installed in the current shell. There is no
-There is no virtualenv or a need for `poetry` inside the
+virtualenv or a need for `poetry` inside the container.
-container.
 
-Running the command `make docker.run` at the root directory
+Running the command `make docker.run` at the root directory of the project will
-of the project will build the container the first time. On
+build the container the first time. On the next runs it will use the cached
-the next runs, it will use the cached image or rebuild it if
+image. A rebuild will happen when changes are made to the source code.
-a change is made on the source code.
 
+You local source directory has been mounted to the /src directory.