forked from Archives/langchain
Compare commits
1 Commits
main
...
docker-uti
| Author | SHA1 | Date |
|---|---|---|
blob42
|
9b19eb03aa | 1 year ago |
@ -1,144 +0,0 @@
|
||||
.vscode/
|
||||
.idea/
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
notebooks/
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
.venvs
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# macOS display setting files
|
||||
.DS_Store
|
||||
|
||||
|
||||
|
||||
# docker
|
||||
docker/
|
||||
!docker/assets/
|
||||
.dockerignore
|
||||
docker.build
|
||||
@ -1,13 +0,0 @@
|
||||
# python env
|
||||
PYTHON_VERSION=3.10
|
||||
|
||||
# -E flag is required
|
||||
# comment the following line to only install dev dependencies
|
||||
POETRY_EXTRA_PACKAGES="-E all"
|
||||
|
||||
# at least one group needed
|
||||
POETRY_DEPENDENCIES="dev,test,lint,typing"
|
||||
|
||||
# langchain env. warning: these variables will be baked into the docker image !
|
||||
OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
||||
SERPAPI_API_KEY=${SERPAPI_API_KEY:-}
|
||||
@ -1,53 +0,0 @@
|
||||
# Using Docker
|
||||
|
||||
To quickly get started, run the command `make docker`.
|
||||
|
||||
If docker is installed the Makefile will export extra targets in the fomrat `docker.*` to build and run the docker image. Type `make` for a list of available tasks.
|
||||
|
||||
There is a basic `docker-compose.yml` in the docker directory.
|
||||
|
||||
## Building the development image
|
||||
|
||||
Using `make docker` will build the dev image if it does not exist, then drops
|
||||
you inside the container with the langchain environment available in the shell.
|
||||
|
||||
### Customizing the image and installed dependencies
|
||||
|
||||
The image is built with a default python version and all extras and dev
|
||||
dependencies. It can be customized by changing the variables in the [.env](/docker/.env)
|
||||
file.
|
||||
|
||||
If you don't need all the `extra` dependencies a slimmer image can be obtained by
|
||||
commenting out `POETRY_EXTRA_PACKAGES` in the [.env](docker/.env) file.
|
||||
|
||||
### Image caching
|
||||
|
||||
The Dockerfile is optimized to cache the poetry install step. A rebuild is triggered when there a change to the source code.
|
||||
|
||||
## Example Usage
|
||||
|
||||
All commands from langchain's python environment are available by default in the container.
|
||||
|
||||
A few examples:
|
||||
```bash
|
||||
# run jupyter notebook
|
||||
docker run --rm -it IMG jupyter notebook
|
||||
|
||||
# run ipython
|
||||
docker run --rm -it IMG ipython
|
||||
|
||||
# start web server
|
||||
docker run --rm -p 8888:8888 IMG python -m http.server 8888
|
||||
```
|
||||
|
||||
## Testing / Linting
|
||||
|
||||
Tests and lints are run using your local source directory that is mounted on the volume /src.
|
||||
|
||||
Run unit tests in the container with `make docker.test`.
|
||||
|
||||
Run the linting and formatting checks with `make docker.lint`.
|
||||
|
||||
Note: this task can run in parallel using `make -j4 docker.lint`.
|
||||
|
||||
|
||||
@ -1,104 +0,0 @@
|
||||
# vim: ft=dockerfile
|
||||
#
|
||||
# see also: https://github.com/python-poetry/poetry/discussions/1879
|
||||
# - with https://github.com/bneijt/poetry-lock-docker
|
||||
# see https://github.com/thehale/docker-python-poetry
|
||||
# see https://github.com/max-pfeiffer/uvicorn-poetry
|
||||
|
||||
# use by default the slim version of python
|
||||
ARG PYTHON_IMAGE_TAG=slim
|
||||
ARG PYTHON_VERSION=${PYTHON_VERSION:-3.11.2}
|
||||
|
||||
####################
|
||||
# Base Environment
|
||||
####################
|
||||
FROM python:$PYTHON_VERSION-$PYTHON_IMAGE_TAG AS lchain-base
|
||||
|
||||
ARG UID=1000
|
||||
ARG USERNAME=lchain
|
||||
|
||||
ENV USERNAME=$USERNAME
|
||||
|
||||
RUN groupadd -g ${UID} $USERNAME
|
||||
RUN useradd -l -m -u ${UID} -g ${UID} $USERNAME
|
||||
|
||||
# used for mounting source code
|
||||
RUN mkdir /src
|
||||
VOLUME /src
|
||||
|
||||
|
||||
#######################
|
||||
## Poetry Builder Image
|
||||
#######################
|
||||
FROM lchain-base AS lchain-base-builder
|
||||
|
||||
ARG POETRY_EXTRA_PACKAGES=$POETRY_EXTRA_PACKAGES
|
||||
ARG POETRY_DEPENDENCIES=$POETRY_DEPENDENCIES
|
||||
|
||||
ENV HOME=/root
|
||||
ENV POETRY_HOME=/root/.poetry
|
||||
ENV POETRY_VIRTUALENVS_IN_PROJECT=false
|
||||
ENV POETRY_NO_INTERACTION=1
|
||||
ENV CACHE_DIR=$HOME/.cache
|
||||
ENV POETRY_CACHE_DIR=$CACHE_DIR/pypoetry
|
||||
ENV PATH="$POETRY_HOME/bin:$PATH"
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
build-essential \
|
||||
git \
|
||||
curl
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
RUN mkdir -p $CACHE_DIR
|
||||
|
||||
## setup poetry
|
||||
RUN curl -sSL -o $CACHE_DIR/pypoetry-installer.py https://install.python-poetry.org/
|
||||
RUN python3 $CACHE_DIR/pypoetry-installer.py
|
||||
|
||||
|
||||
# # Copy poetry files
|
||||
COPY poetry.* pyproject.toml ./
|
||||
|
||||
RUN mkdir /pip-prefix
|
||||
|
||||
RUN poetry export $POETRY_EXTRA_PACKAGES --with $POETRY_DEPENDENCIES -f requirements.txt --output requirements.txt --without-hashes && \
|
||||
pip install --no-cache-dir --disable-pip-version-check --prefix /pip-prefix -r requirements.txt
|
||||
|
||||
|
||||
# add custom motd message
|
||||
COPY docker/assets/etc/motd /tmp/motd
|
||||
RUN cat /tmp/motd > /etc/motd
|
||||
|
||||
RUN printf "\n%s\n%s\n" "$(poetry version)" "$(python --version)" >> /etc/motd
|
||||
|
||||
###################
|
||||
## Runtime Image
|
||||
###################
|
||||
FROM lchain-base AS lchain
|
||||
|
||||
#jupyter port
|
||||
EXPOSE 8888
|
||||
|
||||
COPY docker/assets/entry.sh /entry
|
||||
RUN chmod +x /entry
|
||||
|
||||
COPY --from=lchain-base-builder /etc/motd /etc/motd
|
||||
COPY --from=lchain-base-builder /usr/bin/git /usr/bin/git
|
||||
|
||||
USER ${USERNAME:-lchain}
|
||||
ENV HOME /home/$USERNAME
|
||||
WORKDIR /home/$USERNAME
|
||||
|
||||
COPY --chown=lchain:lchain --from=lchain-base-builder /pip-prefix $HOME/.local/
|
||||
|
||||
COPY . .
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
RUN pip install --no-deps --disable-pip-version-check --no-cache-dir -e .
|
||||
|
||||
|
||||
entrypoint ["/entry"]
|
||||
@ -1,84 +0,0 @@
|
||||
#do not call this makefile it is included in the main Makefile
|
||||
.PHONY: docker docker.jupyter docker.run docker.force_build docker.clean \
|
||||
docker.test docker.lint docker.lint.mypy docker.lint.black \
|
||||
docker.lint.isort docker.lint.flake
|
||||
|
||||
# read python version from .env file ignoring comments
|
||||
PYTHON_VERSION := $(shell grep PYTHON_VERSION docker/.env | cut -d '=' -f2)
|
||||
POETRY_EXTRA_PACKAGES := $(shell grep '^[^#]*POETRY_EXTRA_PACKAGES' docker/.env | cut -d '=' -f2)
|
||||
POETRY_DEPENDENCIES := $(shell grep 'POETRY_DEPENDENCIES' docker/.env | cut -d '=' -f2)
|
||||
|
||||
|
||||
DOCKER_SRC := $(shell find docker -type f)
|
||||
DOCKER_IMAGE_NAME = langchain/dev
|
||||
|
||||
# SRC is all files matched by the git ls-files command
|
||||
SRC := $(shell git ls-files -- '*' ':!:docker/*')
|
||||
|
||||
# set DOCKER_BUILD_PROGRESS=plain to see detailed build progress
|
||||
DOCKER_BUILD_PROGRESS ?= auto
|
||||
|
||||
# extra message to show when entering the docker container
|
||||
DOCKER_MOTD := docker/assets/etc/motd
|
||||
|
||||
ROOTDIR := $(shell git rev-parse --show-toplevel)
|
||||
|
||||
DOCKER_LINT_CMD = docker run --rm -i -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH)
|
||||
|
||||
docker: docker.run
|
||||
|
||||
docker.run: docker.build
|
||||
@echo "Docker image: $(DOCKER_IMAGE_NAME):$(GIT_HASH)"
|
||||
docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH)
|
||||
|
||||
docker.jupyter: docker.build
|
||||
docker run --rm -it -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) jupyter notebook
|
||||
|
||||
docker.build: $(SRC) $(DOCKER_SRC) $(DOCKER_MOTD)
|
||||
ifdef $(DOCKER_BUILDKIT)
|
||||
docker buildx build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
|
||||
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
|
||||
--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
|
||||
--progress=$(DOCKER_BUILD_PROGRESS) \
|
||||
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
|
||||
else
|
||||
docker build --build-arg PYTHON_VERSION=$(PYTHON_VERSION) \
|
||||
--build-arg POETRY_EXTRA_PACKAGES=$(POETRY_EXTRA_PACKAGES) \
|
||||
--build-arg POETRY_DEPENDENCIES=$(POETRY_DEPENDENCIES) \
|
||||
$(BUILD_FLAGS) -f docker/Dockerfile -t $(DOCKER_IMAGE_NAME):$(GIT_HASH) .
|
||||
endif
|
||||
docker tag $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
|
||||
@touch $@ # this prevents docker from rebuilding dependencies that have not
|
||||
@ # changed. Remove the file `docker/docker.build` to force a rebuild.
|
||||
|
||||
docker.force_build: $(DOCKER_SRC)
|
||||
@rm -f docker.build
|
||||
@$(MAKE) docker.build BUILD_FLAGS=--no-cache
|
||||
|
||||
docker.clean:
|
||||
docker rmi $(DOCKER_IMAGE_NAME):$(GIT_HASH) $(DOCKER_IMAGE_NAME):latest
|
||||
|
||||
docker.test: docker.build
|
||||
docker run --rm -it -u lchain -v $(ROOTDIR):/src $(DOCKER_IMAGE_NAME):$(GIT_HASH) \
|
||||
pytest /src/tests/unit_tests
|
||||
|
||||
# this assumes that the docker image has been built
|
||||
docker.lint: docker.lint.mypy docker.lint.black docker.lint.isort \
|
||||
docker.lint.flake
|
||||
|
||||
# these can run in parallel with -j[njobs]
|
||||
docker.lint.mypy:
|
||||
@$(DOCKER_LINT_CMD) mypy /src
|
||||
@printf "\t%s\n" "mypy ... "
|
||||
|
||||
docker.lint.black:
|
||||
@$(DOCKER_LINT_CMD) black /src --check
|
||||
@printf "\t%s\n" "black ... "
|
||||
|
||||
docker.lint.isort:
|
||||
@$(DOCKER_LINT_CMD) isort /src --check
|
||||
@printf "\t%s\n" "isort ... "
|
||||
|
||||
docker.lint.flake:
|
||||
@$(DOCKER_LINT_CMD) flake8 /src
|
||||
@printf "\t%s\n" "flake8 ... "
|
||||
@ -1,10 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export PATH=$HOME/.local/bin:$PATH
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
cat /etc/motd
|
||||
exec /bin/bash
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
@ -1,8 +0,0 @@
|
||||
All dependencies have been installed in the current shell. There is no
|
||||
virtualenv or a need for `poetry` inside the container.
|
||||
|
||||
Running the command `make docker.run` at the root directory of the project will
|
||||
build the container the first time. On the next runs it will use the cached
|
||||
image. A rebuild will happen when changes are made to the source code.
|
||||
|
||||
You local source directory has been mounted to the /src directory.
|
||||
@ -1,17 +0,0 @@
|
||||
version: "3.7"
|
||||
|
||||
services:
|
||||
langchain:
|
||||
hostname: langchain
|
||||
image: langchain/dev:latest
|
||||
build:
|
||||
context: ../
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
PYTHON_VERSION: ${PYTHON_VERSION}
|
||||
POETRY_EXTRA_PACKAGES: ${POETRY_EXTRA_PACKAGES}
|
||||
POETRY_DEPENDENCIES: ${POETRY_DEPENDENCIES}
|
||||
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- 127.0.0.1:8888:8888
|
||||
File diff suppressed because one or more lines are too long
@ -1,184 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9355a547",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Partial Prompt Templates\n",
|
||||
"\n",
|
||||
"A prompt template is a class with a `.format` method which takes in a key-value map and returns a string (a prompt) to pass to the language model. Like other methods, it can make sense to \"partial\" a prompt template - eg pass in a subset of the required values, as to create a new prompt template which expects only the remaining subset of values.\n",
|
||||
"\n",
|
||||
"LangChain supports this in two ways: we allow for partially formatted prompts (1) with string values, (2) with functions that return string values. These two different ways support different use cases. In the documentation below we go over the motivations for both use cases as well as how to do it in LangChain.\n",
|
||||
"\n",
|
||||
"## Partial With Strings\n",
|
||||
"\n",
|
||||
"One common use case for wanting to partial a prompt template is if you get some of the variables before others. For example, suppose you have a prompt template that requires two variables, `foo` and `baz`. If you get the `foo` value early on in the chain, but the `baz` value later, it can be annoying to wait until you have both variables in the same place to pass them to the prompt template. Instead, you can partial the prompt template with the `foo` value, and then pass the partialed prompt template along and just use that. Below is an example of doing this:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "643af5da",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.prompts import PromptTemplate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "4080d8d7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"foobaz\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = PromptTemplate(template=\"{foo}{bar}\", input_variables=[\"foo\", \"bar\"])\n",
|
||||
"partial_prompt = prompt.partial(foo=\"foo\");\n",
|
||||
"print(partial_prompt.format(bar=\"baz\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9986766e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"You can also just initialize the prompt with the partialed variables."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "e2ce95b3",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"foobaz\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = PromptTemplate(template=\"{foo}{bar}\", input_variables=[\"bar\"], partial_variables={\"foo\": \"foo\"})\n",
|
||||
"print(prompt.format(bar=\"baz\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a9c66f83",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Partial With Functions\n",
|
||||
"\n",
|
||||
"The other common use is to partial with a function. The use case for this is when you have a variable you know that you always want to fetch in a common way. A prime example of this is with date or time. Imagine you have a prompt which you always want to have the current date. You can't hard code it in the prompt, and passing it along with the other input variables is a bit annoying. In this case, it's very handy to be able to partial the prompt with a function that always returns the current date."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "d0712d8a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"def _get_datetime():\n",
|
||||
" now = datetime.now()\n",
|
||||
" return now.strftime(\"%m/%d/%Y, %H:%M:%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "4cbcb666",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tell me a funny joke about the day 02/27/2023, 22:15:16\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = PromptTemplate(\n",
|
||||
" template=\"Tell me a {adjective} joke about the day {date}\", \n",
|
||||
" input_variables=[\"adjective\", \"date\"]\n",
|
||||
");\n",
|
||||
"partial_prompt = prompt.partial(date=_get_datetime)\n",
|
||||
"print(partial_prompt.format(adjective=\"funny\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ffed6811",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"You can also just initialize the prompt with the partialed variables, which often makes more sense in this workflow."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "96285b25",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Tell me a funny joke about the day 02/27/2023, 22:15:16\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = PromptTemplate(\n",
|
||||
" template=\"Tell me a {adjective} joke about the day {date}\", \n",
|
||||
" input_variables=[\"adjective\"],\n",
|
||||
" partial_variables={\"date\": _get_datetime}\n",
|
||||
");\n",
|
||||
"print(prompt.format(adjective=\"funny\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4bff16f7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -1,180 +1,103 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "O4HPx3boF0"
|
||||
},
|
||||
"source": [],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "hqQkbPEwTJ"
|
||||
},
|
||||
"source": [
|
||||
"# Using the DockerWrapper utility"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "vCepuypaFH"
|
||||
},
|
||||
"source": [
|
||||
"from langchain.utilities.docker import DockerWrapper"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "BtYVqy2YtO"
|
||||
},
|
||||
"source": [
|
||||
"d = DockerWrapper(image='shell')"
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "ELWWm03ptQ"
|
||||
},
|
||||
"source": [
|
||||
"query = \"\"\"\n",
|
||||
"for i in $(seq 1 10)\n",
|
||||
"do\n",
|
||||
" echo $i\n",
|
||||
"done\n",
|
||||
"\"\"\"\n",
|
||||
"print(d.exec_run(query))"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "lGMqLz5sDo"
|
||||
},
|
||||
"source": [
|
||||
"p = DockerWrapper(image='python')\n",
|
||||
"\n",
|
||||
"py_payload = \"\"\"\n",
|
||||
"def hello_world():\n",
|
||||
" return 'hello world'\n",
|
||||
"\n",
|
||||
"hello_world()\n",
|
||||
"\"\"\""
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "X04Wd6zbrk"
|
||||
},
|
||||
"source": [
|
||||
"print(p.exec_run(py_payload))"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "'hello world'\n"
|
||||
}
|
||||
],
|
||||
"execution_count": 2
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "lKOfuDoJGk"
|
||||
},
|
||||
"source": [],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "eSzXtDrpqU"
|
||||
},
|
||||
"source": [
|
||||
"## Passing custom parameters\n",
|
||||
"\n",
|
||||
"By default containers are run with a safe set of parameters. You can pass any parameters\n",
|
||||
"that are accepted by the docker python sdk to the run and exec commands.\n",
|
||||
"\n",
|
||||
"### Using networking"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "eWFGCxD9pv"
|
||||
},
|
||||
"source": [
|
||||
"# by default containers don't have access to the network\n",
|
||||
"print(d.run('ping -c 1 google.com'))"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "STDERR: Command '/bin/sh -c 'ping -c 1 google.com'' in image 'alpine:latest' returned non-zero exit status 1: b\"ping: bad address 'google.com'\\n\"\n"
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "Z0YkpuXVyL"
|
||||
},
|
||||
"source": [
|
||||
"# using the network parameter\n",
|
||||
"print(d.run('ping -c 1 google.com', network='bridge'))"
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": "PING google.com (142.250.200.110): 56 data bytes\n64 bytes from 142.250.200.110: seq=0 ttl=42 time=13.695 ms\n\n--- google.com ping statistics ---\n1 packets transmitted, 1 packets received, 0% packet loss\nround-trip min/avg/max = 13.695/13.695/13.695 ms\n"
|
||||
}
|
||||
],
|
||||
"execution_count": 4
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"jukit_cell_id": "O4HPx3boF0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "hqQkbPEwTJ"
|
||||
},
|
||||
"source": [
|
||||
"# Using the DockerWrapper utility"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"jukit_cell_id": "vCepuypaFH"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.utilities.docker import DockerWrapper, DockerSocket"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"jukit_cell_id": "BtYVqy2YtO"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"d = DockerWrapper()\n",
|
||||
"query = \"\"\"\n",
|
||||
"for i in $(seq 1 10)\n",
|
||||
"do\n",
|
||||
" echo $i\n",
|
||||
"done\n",
|
||||
"\"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"jukit_cell_id": "ELWWm03ptQ"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"metadata": {
|
||||
"jukit_cell_id": "3rMWzzuLHq"
|
||||
},
|
||||
"source": [],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "python",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[header] blocking IO\n",
|
||||
"{'stdout': '1\\n2\\n3\\n4\\n5\\n6\\n7\\n8\\n9\\n10\\n'}\n",
|
||||
"1\n",
|
||||
"2\n",
|
||||
"3\n",
|
||||
"4\n",
|
||||
"5\n",
|
||||
"6\n",
|
||||
"7\n",
|
||||
"8\n",
|
||||
"9\n",
|
||||
"10\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(d.exec_run(query, \"alpine\"))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.11"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
|
||||
@ -1,202 +0,0 @@
|
||||
"""Loader that loads iFixit data."""
|
||||
from typing import List, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from langchain.docstore.document import Document
|
||||
from langchain.document_loaders.base import BaseLoader
|
||||
from langchain.document_loaders.web_base import WebBaseLoader
|
||||
|
||||
IFIXIT_BASE_URL = "https://www.ifixit.com/api/2.0"
|
||||
|
||||
|
||||
class IFixitLoader(BaseLoader):
|
||||
"""Load iFixit repair guides, device wikis and answers.
|
||||
|
||||
iFixit is the largest, open repair community on the web. The site contains nearly
|
||||
100k repair manuals, 200k Questions & Answers on 42k devices, and all the data is
|
||||
licensed under CC-BY.
|
||||
|
||||
This loader will allow you to download the text of a repair guide, text of Q&A's
|
||||
and wikis from devices on iFixit using their open APIs and web scraping.
|
||||
"""
|
||||
|
||||
def __init__(self, web_path: str):
|
||||
"""Initialize with web path."""
|
||||
if not web_path.startswith("https://www.ifixit.com"):
|
||||
raise ValueError("web path must start with 'https://www.ifixit.com'")
|
||||
|
||||
path = web_path.replace("https://www.ifixit.com", "")
|
||||
|
||||
allowed_paths = ["/Device", "/Guide", "/Answers", "/Teardown"]
|
||||
|
||||
""" TODO: Add /Wiki """
|
||||
if not any(path.startswith(allowed_path) for allowed_path in allowed_paths):
|
||||
raise ValueError(
|
||||
"web path must start with /Device, /Guide, /Teardown or /Answers"
|
||||
)
|
||||
|
||||
pieces = [x for x in path.split("/") if x]
|
||||
|
||||
"""Teardowns are just guides by a different name"""
|
||||
self.page_type = pieces[0] if pieces[0] != "Teardown" else "Guide"
|
||||
|
||||
if self.page_type == "Guide" or self.page_type == "Answers":
|
||||
self.id = pieces[2]
|
||||
else:
|
||||
self.id = pieces[1]
|
||||
|
||||
self.web_path = web_path
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
if self.page_type == "Device":
|
||||
return self.load_device()
|
||||
elif self.page_type == "Guide" or self.page_type == "Teardown":
|
||||
return self.load_guide()
|
||||
elif self.page_type == "Answers":
|
||||
return self.load_questions_and_answers()
|
||||
else:
|
||||
raise ValueError("Unknown page type: " + self.page_type)
|
||||
|
||||
@staticmethod
|
||||
def load_suggestions(query: str = "", doc_type: str = "all") -> List[Document]:
|
||||
res = requests.get(
|
||||
IFIXIT_BASE_URL + "/suggest/" + query + "?doctypes=" + doc_type
|
||||
)
|
||||
|
||||
if res.status_code != 200:
|
||||
raise ValueError(
|
||||
'Could not load suggestions for "' + query + '"\n' + res.json()
|
||||
)
|
||||
|
||||
data = res.json()
|
||||
|
||||
results = data["results"]
|
||||
output = []
|
||||
|
||||
for result in results:
|
||||
try:
|
||||
loader = IFixitLoader(result["url"])
|
||||
if loader.page_type == "Device":
|
||||
output += loader.load_device(include_guides=False)
|
||||
else:
|
||||
output += loader.load()
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return output
|
||||
|
||||
def load_questions_and_answers(
|
||||
self, url_override: Optional[str] = None
|
||||
) -> List[Document]:
|
||||
loader = WebBaseLoader(self.web_path if url_override is None else url_override)
|
||||
soup = loader.scrape()
|
||||
|
||||
output = []
|
||||
|
||||
title = soup.find("h1", "post-title").text
|
||||
|
||||
output.append("# " + title)
|
||||
output.append(soup.select_one(".post-content .post-text").text.strip())
|
||||
|
||||
output.append("\n## " + soup.find("div", "post-answers-header").text.strip())
|
||||
for answer in soup.select(".js-answers-list .post.post-answer"):
|
||||
if answer.has_attr("itemprop") and "acceptedAnswer" in answer["itemprop"]:
|
||||
output.append("\n### Accepted Answer")
|
||||
elif "post-helpful" in answer["class"]:
|
||||
output.append("\n### Most Helpful Answer")
|
||||
else:
|
||||
output.append("\n### Other Answer")
|
||||
|
||||
output += [
|
||||
a.text.strip() for a in answer.select(".post-content .post-text")
|
||||
]
|
||||
output.append("\n")
|
||||
|
||||
text = "\n".join(output).strip()
|
||||
|
||||
metadata = {"source": self.web_path, "title": title}
|
||||
|
||||
return [Document(page_content=text, metadata=metadata)]
|
||||
|
||||
def load_device(
|
||||
self, url_override: Optional[str] = None, include_guides: bool = True
|
||||
) -> List[Document]:
|
||||
documents = []
|
||||
if url_override is None:
|
||||
url = IFIXIT_BASE_URL + "/wikis/CATEGORY/" + self.id
|
||||
else:
|
||||
url = url_override
|
||||
|
||||
res = requests.get(url)
|
||||
data = res.json()
|
||||
text = "\n".join(
|
||||
[
|
||||
data[key]
|
||||
for key in ["title", "description", "contents_raw"]
|
||||
if key in data
|
||||
]
|
||||
).strip()
|
||||
|
||||
metadata = {"source": self.web_path, "title": data["title"]}
|
||||
documents.append(Document(page_content=text, metadata=metadata))
|
||||
|
||||
if include_guides:
|
||||
"""Load and return documents for each guide linked to from the device"""
|
||||
guide_urls = [guide["url"] for guide in data["guides"]]
|
||||
for guide_url in guide_urls:
|
||||
documents.append(IFixitLoader(guide_url).load()[0])
|
||||
|
||||
return documents
|
||||
|
||||
def load_guide(self, url_override: Optional[str] = None) -> List[Document]:
|
||||
if url_override is None:
|
||||
url = IFIXIT_BASE_URL + "/guides/" + self.id
|
||||
else:
|
||||
url = url_override
|
||||
|
||||
res = requests.get(url)
|
||||
|
||||
if res.status_code != 200:
|
||||
raise ValueError(
|
||||
"Could not load guide: " + self.web_path + "\n" + res.json()
|
||||
)
|
||||
|
||||
data = res.json()
|
||||
|
||||
doc_parts = ["# " + data["title"], data["introduction_raw"]]
|
||||
|
||||
doc_parts.append("\n\n###Tools Required:")
|
||||
if len(data["tools"]) == 0:
|
||||
doc_parts.append("\n - None")
|
||||
else:
|
||||
for tool in data["tools"]:
|
||||
doc_parts.append("\n - " + tool["text"])
|
||||
|
||||
doc_parts.append("\n\n###Parts Required:")
|
||||
if len(data["parts"]) == 0:
|
||||
doc_parts.append("\n - None")
|
||||
else:
|
||||
for part in data["parts"]:
|
||||
doc_parts.append("\n - " + part["text"])
|
||||
|
||||
for row in data["steps"]:
|
||||
doc_parts.append(
|
||||
"\n\n## "
|
||||
+ (
|
||||
row["title"]
|
||||
if row["title"] != ""
|
||||
else "Step {}".format(row["orderby"])
|
||||
)
|
||||
)
|
||||
|
||||
for line in row["lines"]:
|
||||
doc_parts.append(line["text_raw"])
|
||||
|
||||
doc_parts.append(data["conclusion_raw"])
|
||||
|
||||
text = "\n".join(doc_parts)
|
||||
|
||||
metadata = {"source": self.web_path, "title": data["title"]}
|
||||
|
||||
return [Document(page_content=text, metadata=metadata)]
|
||||
@ -1,13 +0,0 @@
|
||||
"""Loader that loads image files."""
|
||||
from typing import List
|
||||
|
||||
from langchain.document_loaders.unstructured import UnstructuredFileLoader
|
||||
|
||||
|
||||
class UnstructuredImageLoader(UnstructuredFileLoader):
|
||||
"""Loader that uses unstructured to load image files, such as PNGs and JPGs."""
|
||||
|
||||
def _get_elements(self) -> List:
|
||||
from unstructured.partition.image import partition_image
|
||||
|
||||
return partition_image(filename=self.file_path)
|
||||
@ -0,0 +1,32 @@
|
||||
import pexpect
|
||||
|
||||
def interact_with_python_shell(child: pexpect.spawn, **kwargs) -> str:
|
||||
"""Interact with a Python shell running in a pexpect spawn instance."""
|
||||
try:
|
||||
child.expect(">>>", **kwargs)
|
||||
except pexpect.exceptions.TIMEOUT:
|
||||
print("no data")
|
||||
return child.before.strip()
|
||||
buf = ""
|
||||
max_reads=5
|
||||
read_cnt=0
|
||||
while True:
|
||||
if read_cnt > max_reads:
|
||||
print("reached max depth")
|
||||
break
|
||||
try:
|
||||
child.expect("\n>>>", **kwargs)
|
||||
result = child.before.strip()
|
||||
if len(result) > 0:
|
||||
buf += result + "\n"
|
||||
except pexpect.exceptions.TIMEOUT:
|
||||
# The child process is still running, but it's not outputting anything.
|
||||
# Wait a bit and try again.
|
||||
#TODO: add trial times before exit
|
||||
read_cnt += 1
|
||||
continue
|
||||
except pexpect.exceptions.EOF:
|
||||
# The child process has exited.
|
||||
print("EOF")
|
||||
break
|
||||
return buf
|
||||
@ -1,110 +0,0 @@
|
||||
"""Low level socket IO for docker API."""
|
||||
import struct
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SOCK_BUF_SIZE = 1024
|
||||
|
||||
class DockerSocket:
|
||||
"""Wrapper around docker API's socket object. Can be used as a context manager."""
|
||||
|
||||
_timeout: int = 5
|
||||
|
||||
|
||||
def __init__(self, socket, timeout: int = _timeout):
|
||||
self.socket = socket
|
||||
self.socket._sock.settimeout(timeout)
|
||||
# self.socket._sock.setblocking(False)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
logger.debug("closing socket...")
|
||||
self.socket._sock.shutdown(2) # 2 = SHUT_RDWR
|
||||
self.socket._sock.close()
|
||||
self.socket.close()
|
||||
|
||||
def sendall(self, data: bytes) -> None:
|
||||
self.socket._sock.sendall(data)
|
||||
|
||||
def setblocking(self, flag: bool) -> None:
|
||||
self.socket._sock.setblocking(flag)
|
||||
|
||||
def recv(self) -> Any:
|
||||
"""Wrapper for socket.recv that does buffured read."""
|
||||
|
||||
# NOTE: this is optional as a bonus
|
||||
# TODO: Recv with TTY enabled
|
||||
#
|
||||
# When the TTY setting is enabled in POST /containers/create, the stream
|
||||
# is not multiplexed. The data exchanged over the hijacked connection is
|
||||
# simply the raw data from the process PTY and client's stdin.
|
||||
|
||||
# header := [8]byte{STREAM_TYPE, 0, 0, 0, SIZE1, SIZE2, SIZE3, SIZE4}
|
||||
# STREAM_TYPE can be:
|
||||
#
|
||||
# 0: stdin (is written on stdout)
|
||||
# 1: stdout
|
||||
# 2: stderr
|
||||
# SIZE1, SIZE2, SIZE3, SIZE4 are the four bytes of the uint32 size encoded as
|
||||
# big endian.
|
||||
#
|
||||
# Following the header is the payload, which is the specified number of bytes of
|
||||
# STREAM_TYPE.
|
||||
#
|
||||
# The simplest way to implement this protocol is the following:
|
||||
#
|
||||
# - Read 8 bytes.
|
||||
# - Choose stdout or stderr depending on the first byte.
|
||||
# - Extract the frame size from the last four bytes.
|
||||
# - Read the extracted size and output it on the correct output.
|
||||
# - Goto 1.
|
||||
|
||||
chunks = []
|
||||
# try:
|
||||
# self.socket._sock.recv(8)
|
||||
# except BlockingIOError as e:
|
||||
# raise ValueError("incomplete read from container output")
|
||||
|
||||
while True:
|
||||
header = b''
|
||||
try:
|
||||
# strip the header
|
||||
# the first recv is blocking to wait for the container to start
|
||||
header = self.socket._sock.recv(8)
|
||||
except BlockingIOError:
|
||||
# logger.debug("[header] blocking IO")
|
||||
break
|
||||
|
||||
self.socket._sock.setblocking(False)
|
||||
|
||||
if header == b'':
|
||||
break
|
||||
stream_type, size = struct.unpack("!BxxxI", header)
|
||||
|
||||
payload = b''
|
||||
while size:
|
||||
chunk = b''
|
||||
try:
|
||||
chunk = self.socket._sock.recv(min(size, SOCK_BUF_SIZE))
|
||||
except BlockingIOError:
|
||||
# logger.debug("[body] blocking IO")
|
||||
break
|
||||
if chunk == b'':
|
||||
raise ValueError("incomplete read from container output")
|
||||
payload += chunk
|
||||
size -= len(chunk)
|
||||
chunks.append((stream_type, payload))
|
||||
# try:
|
||||
# msg = self.socket._sock.recv(SOCK_BUF_SIZE)
|
||||
# chunk += msg
|
||||
# except BlockingIOError as e:
|
||||
# break
|
||||
|
||||
return chunks
|
||||
@ -1,449 +0,0 @@
|
||||
# TODO!: using pexpect to with containers
|
||||
# TODO: add default expect pattern to image template
|
||||
# TODO: pass max reads parameters for read trials
|
||||
# NOTE: spawning with tty true or not gives slightly different stdout format
|
||||
# NOTE: echo=False works when tty is disabled and only stdin is connected
|
||||
|
||||
import shlex
|
||||
import os
|
||||
import io
|
||||
import tarfile
|
||||
import time
|
||||
import pandas as pd # type: ignore
|
||||
import docker
|
||||
import socket
|
||||
|
||||
from typing import Any, Dict, Optional, Union, Type
|
||||
from pydantic import BaseModel, Extra, root_validator, Field
|
||||
from docker.errors import APIError, ContainerError # type: ignore
|
||||
|
||||
from .images import Shell, BaseImage, get_image_template
|
||||
from . import gvisor_runtime_available
|
||||
from .socket_io import DockerSocket
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_default_params = {
|
||||
# the only required parameter to be able to attach.
|
||||
'stdin_open': True,
|
||||
}
|
||||
|
||||
|
||||
def _get_command(query: str, **kwargs: Dict) -> str:
|
||||
"""Build an escaped command from a query string and keyword arguments."""
|
||||
cmd = query
|
||||
if 'default_command' in kwargs:
|
||||
cmd = shlex.join([*kwargs.get('default_command'), query]) # type: ignore
|
||||
|
||||
return cmd
|
||||
|
||||
|
||||
class DockerWrapper(BaseModel, extra=Extra.allow):
|
||||
"""Executes arbitrary commands or payloads on containers and returns the output.
|
||||
|
||||
Args:
|
||||
image (str | Type[BaseImage]): Docker image to use for execution. The
|
||||
image can be a string or a subclass of images.BaseImage.
|
||||
default_command (List[str]): Default command to use when creating the container.
|
||||
"""
|
||||
|
||||
_docker_client: docker.DockerClient = None # type: ignore
|
||||
_params: Dict = Field(default_factory=Shell().dict(), skip=True)
|
||||
image: Union[str, Type[BaseImage]] = Field(default_factory=Shell, skip=True)
|
||||
from_env: Optional[bool] = Field(default=True, skip=True)
|
||||
|
||||
# @property
|
||||
# def image_name(self) -> str:
|
||||
# """The image name that will be used when creating a container."""
|
||||
# return self._params.image
|
||||
#
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize docker client."""
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if self.from_env:
|
||||
self._docker_client = docker.from_env()
|
||||
if gvisor_runtime_available(docker.from_env()):
|
||||
self._params['runtime'] = 'runsc'
|
||||
|
||||
# if not isinstance(self.image, str) and issubclass(self.image, BaseImage):
|
||||
# self._params = {**self._params, **self.image().dict()}
|
||||
#
|
||||
# # if the user defined a custom image not pre registerd already we should
|
||||
# # not use the custom command
|
||||
# elif isinstance(self.image, str):
|
||||
# self._params = {**_default_params(), **{'image': self.image}}
|
||||
|
||||
@property
|
||||
def client(self) -> docker.DockerClient: # type: ignore
|
||||
"""Docker client."""
|
||||
return self._docker_client
|
||||
|
||||
@property
|
||||
def info(self) -> Any:
|
||||
"""Prints docker `info`."""
|
||||
return self._docker_client.info()
|
||||
|
||||
# @validator("image", pre=True, always=True)
|
||||
# def validate_image(cls, value):
|
||||
# if value is None:
|
||||
# raise ValueError("image is required")
|
||||
# if isinstance(value, str) :
|
||||
# image = get_image(value)
|
||||
# if isinstance(image, BaseImage):
|
||||
# return image
|
||||
# else:
|
||||
# #set default params to base ones
|
||||
# if issubclass(value, BaseImage):
|
||||
# return value
|
||||
# else:
|
||||
# raise ValueError("image must be a string or a subclass of images.BaseImage")
|
||||
|
||||
@root_validator()
|
||||
def validate_all(cls, values: Dict) -> Dict:
|
||||
"""Validate environment."""
|
||||
image = values.get("image")
|
||||
if image is None:
|
||||
raise ValueError("image is required")
|
||||
if isinstance(image, str):
|
||||
# try to get image
|
||||
_image = get_image_template(image)
|
||||
if isinstance(_image, str):
|
||||
# user wants a custom image, we should use default params
|
||||
values["_params"] = {**_default_params, **{'image': image}}
|
||||
else:
|
||||
# user wants a pre registered image, we should use the image params
|
||||
values["_params"] = _image().dict()
|
||||
# image is a BaseImage class
|
||||
elif issubclass(image.__class__, BaseImage):
|
||||
values["_params"] = image.dict()
|
||||
|
||||
|
||||
def field_filter(x):
|
||||
fields = cls.__fields__
|
||||
if x[0] == '_params':
|
||||
return False
|
||||
field = fields.get(x[0], None)
|
||||
if not field:
|
||||
return True
|
||||
return not field.field_info.extra.get('skip', False)
|
||||
filtered_fields: Dict[Any, Any] = dict(filter(field_filter, values.items())) # type: ignore
|
||||
values["_params"] = {**values["_params"],
|
||||
**filtered_fields}
|
||||
|
||||
return values
|
||||
|
||||
def _clean_kwargs(self, kwargs: dict) -> dict:
|
||||
kwargs.pop('default_command', None)
|
||||
kwargs.pop('stdin_command', None)
|
||||
return kwargs
|
||||
|
||||
|
||||
|
||||
#FIX: default shell command should be different in run vs exec mode
|
||||
def run(self, query: str, **kwargs: Any) -> str:
|
||||
"""Run arbitrary shell command inside a container.
|
||||
|
||||
This method will concatenate the registered default command with the provided
|
||||
query.
|
||||
|
||||
Args:
|
||||
query (str): The command to run.
|
||||
**kwargs: Pass extra parameters to DockerClient.container.run.
|
||||
|
||||
"""
|
||||
kwargs = {**self._params, **kwargs}
|
||||
args = {
|
||||
'image': self._params.get('image'),
|
||||
'command': query,
|
||||
}
|
||||
|
||||
del kwargs['image']
|
||||
cmd = _get_command(query, **kwargs)
|
||||
self._clean_kwargs(kwargs)
|
||||
|
||||
args['command'] = cmd
|
||||
# print(f"args: {args}")
|
||||
# print(f"kwargs: {kwargs}")
|
||||
# return
|
||||
logger.debug(f"running command {args['command']}")
|
||||
logger.debug(f"with params {kwargs}")
|
||||
try:
|
||||
result = self._docker_client.containers.run(*(args.values()),
|
||||
remove=True,
|
||||
**kwargs)
|
||||
return result.decode('utf-8').strip()
|
||||
except ContainerError as e:
|
||||
return f"STDERR: {e}"
|
||||
|
||||
# TODO: handle docker APIError ?
|
||||
except APIError as e:
|
||||
logger.debug(f"APIError: {e}")
|
||||
return "ERROR"
|
||||
|
||||
def _flush_prompt(self, _socket):
|
||||
flush = _socket.recv()
|
||||
_socket.setblocking(True)
|
||||
logger.debug(f"flushed output: {flush}")
|
||||
|
||||
def _massage_output_streams(self, output):
|
||||
df = pd.DataFrame(output, columns=['stream_type', 'payload'])
|
||||
df['payload'] = df['payload'].apply(lambda x: x.decode('utf-8'))
|
||||
df['stream_type'] = df['stream_type'].apply(
|
||||
lambda x: 'stdout' if x == 1 else 'stderr')
|
||||
payload = df.groupby('stream_type')['payload'].apply(''.join).to_dict()
|
||||
logger.debug(f"payload: {payload}")
|
||||
return payload
|
||||
|
||||
|
||||
# TODO: document dif between run and exec_run
|
||||
def exec_run(self, query: str, timeout: int = 5,
|
||||
delay: float = 0.5,
|
||||
with_stderr: bool = False,
|
||||
flush_prompt: bool = False,
|
||||
**kwargs: Any) -> str:
|
||||
"""Run a shell command inside an ephemeral container.
|
||||
|
||||
This will create a container, run the command, and then remove the
|
||||
container. the input is sent to the container's stdin through a socket
|
||||
using Docker API. It effectively simulates a tty session.
|
||||
|
||||
Args:
|
||||
query (str): The command to execute.
|
||||
timeout (int): The timeout for receiving from the attached stdin.
|
||||
delay (float): The delay in seconds before running the command.
|
||||
with_stderr (bool): If True, the stderr will be included in the output
|
||||
flush_prompt (bool): If True, the prompt will be flushed before running the command.
|
||||
**kwargs: Pass extra parameters to DockerClient.container.exec_run.
|
||||
"""
|
||||
# it is necessary to open stdin to keep the container running after it's started
|
||||
# the attach_socket will hold the connection open until the container is stopped or
|
||||
# the socket is closed.
|
||||
|
||||
# NOTE: using tty=True to be able to simulate a tty session.
|
||||
|
||||
# NOTE: some images like python need to be launched with custom
|
||||
# parameters to keep stdin open. For example python image needs to be
|
||||
# started with the command `python3 -i`
|
||||
|
||||
# remove local variables from kwargs
|
||||
for arg in kwargs.keys():
|
||||
if arg in locals():
|
||||
del kwargs[arg]
|
||||
|
||||
|
||||
kwargs = {**self._params, **kwargs}
|
||||
kwargs = self._clean_kwargs(kwargs)
|
||||
|
||||
# exec_run requires flags for stdin so we use `stdin_command` as
|
||||
# a default command for creating the container
|
||||
if 'stdin_command' in kwargs:
|
||||
assert isinstance(kwargs['stdin_command'], list)
|
||||
kwargs['command'] = shlex.join(kwargs['stdin_command'])
|
||||
del kwargs['stdin_command']
|
||||
|
||||
# kwargs.pop('default_command', None)
|
||||
# kwargs['command'] = cmd
|
||||
|
||||
# print(f"kwargs: {kwargs}")
|
||||
# return
|
||||
|
||||
# TODO: handle both output mode for tty=True/False
|
||||
logger.debug(f"creating container with params {kwargs}")
|
||||
|
||||
container = self._docker_client.containers.create(**kwargs)
|
||||
container.start()
|
||||
|
||||
# get underlying socket
|
||||
# important to set 'stream' or attach API does not work
|
||||
_socket = container.attach_socket(params={'stdout': 1, 'stderr': 1,
|
||||
'stdin': 1, 'stream': 1})
|
||||
|
||||
|
||||
# input()
|
||||
with DockerSocket(_socket, timeout=timeout) as _socket:
|
||||
# flush the output buffer (if any prompt)
|
||||
if flush_prompt:
|
||||
self._flush_prompt(_socket)
|
||||
|
||||
# TEST: make sure the container is ready ? use a blocking first call
|
||||
raw_input = f"{query}\n".encode('utf-8')
|
||||
_socket.sendall(raw_input)
|
||||
|
||||
#NOTE: delay ensures that the command is executed after the input is sent
|
||||
time.sleep(delay) #this should be available as a parameter
|
||||
|
||||
try:
|
||||
output = _socket.recv()
|
||||
except socket.timeout:
|
||||
return "ERROR: timeout"
|
||||
|
||||
|
||||
try:
|
||||
container.kill()
|
||||
except APIError:
|
||||
pass
|
||||
container.remove(force=True)
|
||||
|
||||
if output is None:
|
||||
logger.warning("no output")
|
||||
return "ERROR"
|
||||
|
||||
# output is stored in a list of tuples (stream_type, payload)
|
||||
payload = self._massage_output_streams(output)
|
||||
|
||||
|
||||
#NOTE: stderr might contain only the prompt
|
||||
if 'stdout' in payload and 'stderr' in payload and with_stderr:
|
||||
return f"STDOUT:\n {payload['stdout'].strip()}\nSTDERR:\n {payload['stderr']}"
|
||||
elif 'stderr' in payload and not 'stdout' in payload:
|
||||
return f"STDERR: {payload['stderr']}"
|
||||
else:
|
||||
return payload['stdout'].strip()
|
||||
|
||||
|
||||
def exec_attached(self, query: str, container: str,
|
||||
delay: float = 0.5,
|
||||
timeout: int = 5,
|
||||
with_stderr: bool = False,
|
||||
flush_prompt: bool = False,
|
||||
**kwargs: Any) -> str:
|
||||
"""Attach to container and exec query on it.
|
||||
|
||||
This method is very similary to exec_run. It only differs in that it attaches to
|
||||
an already specifed container instead of creating a new one for each query.
|
||||
|
||||
Args:
|
||||
query (str): The command to execute.
|
||||
container (str): The container to attach to.
|
||||
timeout (int): The timeout for receiving from the attached stdin.
|
||||
delay (float): The delay in seconds before running the command.
|
||||
with_stderr (bool): If True, the stderr will be included in the output
|
||||
flush_prompt (bool): If True, the prompt will be flushed before running the command.
|
||||
**kwargs: Pass extra parameters to DockerClient.container.exec_run.
|
||||
|
||||
"""
|
||||
|
||||
# remove local variables from kwargs
|
||||
for arg in kwargs.keys():
|
||||
if arg in locals():
|
||||
del kwargs[arg]
|
||||
|
||||
|
||||
kwargs = {**self._params, **kwargs}
|
||||
kwargs = self._clean_kwargs(kwargs)
|
||||
|
||||
logger.debug(f"attaching to container {container} with params {kwargs}")
|
||||
|
||||
try:
|
||||
_container = self._docker_client.containers.get(container)
|
||||
except Exception as e:
|
||||
logger.error(f"container {container}: {e}")
|
||||
return "ERROR"
|
||||
|
||||
_socket = _container.attach_socket(params={'stdout': 1, 'stderr': 1,
|
||||
'stdin': 1, 'stream': 1})
|
||||
|
||||
|
||||
with DockerSocket(_socket, timeout=timeout) as _socket:
|
||||
# flush the output buffer (if any prompt)
|
||||
if flush_prompt:
|
||||
self._flush_prompt(_socket)
|
||||
|
||||
raw_input = f"{query}\n".encode('utf-8')
|
||||
_socket.sendall(raw_input)
|
||||
|
||||
#NOTE: delay ensures that the command is executed after the input is sent
|
||||
time.sleep(delay) #this should be available as a parameter
|
||||
|
||||
try:
|
||||
output = _socket.recv()
|
||||
except socket.timeout:
|
||||
return "ERROR: timeout"
|
||||
|
||||
if output is None:
|
||||
logger.warning("no output")
|
||||
return "ERROR"
|
||||
|
||||
payload = self._massage_output_streams(output)
|
||||
print(payload)
|
||||
|
||||
#NOTE: stderr might contain only the prompt
|
||||
if 'stdout' in payload and 'stderr' in payload and with_stderr:
|
||||
return f"STDOUT:\n {payload['stdout'].strip()}\nSTDERR:\n {payload['stderr']}"
|
||||
elif 'stderr' in payload and not 'stdout' in payload:
|
||||
return f"STDERR: {payload['stderr']}"
|
||||
else:
|
||||
return payload['stdout'].strip()
|
||||
|
||||
|
||||
|
||||
#WIP method that will copy the given payload to the container filesystem then
|
||||
# invoke the command on the file and return the output
|
||||
def run_file(self, payload: bytes, filename: Optional[str] = None,
|
||||
**kwargs: Any) -> str:
|
||||
"""Run arbitrary shell command inside an ephemeral container on the
|
||||
specified input payload."""
|
||||
|
||||
|
||||
for arg in kwargs.keys():
|
||||
if arg in locals():
|
||||
del kwargs[arg]
|
||||
|
||||
kwargs = {**self._params, **kwargs}
|
||||
self._clean_kwargs(kwargs)
|
||||
|
||||
kwargs['command'] = '/bin/sh'
|
||||
|
||||
k_file_location = '/tmp/payload'
|
||||
if filename is not None:
|
||||
# store at /tmp/file_name
|
||||
# strip all leading path components
|
||||
file_loc = os.path.basename(filename)
|
||||
k_file_location = f'/tmp/{file_loc}'
|
||||
|
||||
# print(kwargs)
|
||||
# return
|
||||
|
||||
# create a container with the given payload
|
||||
# container = self._docker_client.containers.create(**kwargs)
|
||||
# container.start()
|
||||
container = self._docker_client.containers.list()[0]
|
||||
print(container.short_id)
|
||||
|
||||
|
||||
# copy the payload to the container
|
||||
try:
|
||||
# put the data in tar archive at the path specified by k_file_location
|
||||
archive = io.BytesIO()
|
||||
with tarfile.TarFile(fileobj=archive, mode='w') as tar:
|
||||
tarinfo = tarfile.TarInfo(name='test-archive')
|
||||
tarinfo.size = len(payload)
|
||||
tarinfo.mtime = int(time.time())
|
||||
tar.addfile(tarinfo, io.BytesIO(payload))
|
||||
archive.seek(0)
|
||||
|
||||
# store archive on local host at /tmp/test
|
||||
# with open('/tmp/test', 'wb') as f:
|
||||
# f.write(archive.read())
|
||||
|
||||
|
||||
container.put_archive(path='/', data=archive)
|
||||
except APIError as e:
|
||||
logger.error(f"Error: {e}")
|
||||
return "ERROR"
|
||||
|
||||
#execute the command
|
||||
exit_code, out = container.exec_run(['sh', k_file_location])
|
||||
print(f"exit_code: {exit_code}")
|
||||
print(f"out: {out}")
|
||||
|
||||
|
||||
# try:
|
||||
# container.kill()
|
||||
# except APIError:
|
||||
# pass
|
||||
# container.remove(force=True)
|
||||
|
||||
return ""
|
||||
@ -1 +0,0 @@
|
||||
"""Test document loader integrations."""
|
||||
@ -1,37 +0,0 @@
|
||||
from langchain.document_loaders.ifixit import IFixitLoader
|
||||
|
||||
|
||||
def test_ifixit_loader() -> None:
|
||||
"""Test iFixit loader."""
|
||||
web_path = "https://www.ifixit.com/Guide/iPad+9+Battery+Replacement/151279"
|
||||
loader = IFixitLoader(web_path)
|
||||
assert loader.page_type == "Guide"
|
||||
assert loader.id == "151279"
|
||||
assert loader.web_path == web_path
|
||||
|
||||
|
||||
def test_ifixit_loader_teardown() -> None:
|
||||
web_path = "https://www.ifixit.com/Teardown/Banana+Teardown/811"
|
||||
loader = IFixitLoader(web_path)
|
||||
""" Teardowns are just guides by a different name """
|
||||
assert loader.page_type == "Guide"
|
||||
assert loader.id == "811"
|
||||
|
||||
|
||||
def test_ifixit_loader_device() -> None:
|
||||
web_path = "https://www.ifixit.com/Device/Standard_iPad"
|
||||
loader = IFixitLoader(web_path)
|
||||
""" Teardowns are just guides by a different name """
|
||||
assert loader.page_type == "Device"
|
||||
assert loader.id == "Standard_iPad"
|
||||
|
||||
|
||||
def test_ifixit_loader_answers() -> None:
|
||||
web_path = (
|
||||
"https://www.ifixit.com/Answers/View/318583/My+iPhone+6+is+typing+and+"
|
||||
"opening+apps+by+itself"
|
||||
)
|
||||
loader = IFixitLoader(web_path)
|
||||
|
||||
assert loader.page_type == "Answers"
|
||||
assert loader.id == "318583"
|
||||
Loading…
Reference in New Issue