diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 691bd4c..ac90f26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ - id: check-yaml - id: end-of-file-fixer - id: flake8 + exclude: __init__.py - id: name-tests-test - id: autopep8-wrapper - id: requirements-txt-fixer diff --git a/Dockerfile.py2 b/Dockerfile.py2 index 1f3bc84..8052aec 100644 --- a/Dockerfile.py2 +++ b/Dockerfile.py2 @@ -7,11 +7,10 @@ RUN pip install pytest twiggy six pyyaml jinja2 ENV PYTHONPATH /opt/pyentrypoint/ ADD pyentrypoint /opt/pyentrypoint/ -ADD tests /opt/pyentrypoint/tests -ADD tests/entrypoint-config.yml /opt/pyentrypoint/ +ADD tests /opt/ ADD tests/test_template.yml.tpl /tmp/test_template.yml -WORKDIR /opt/pyentrypoint/ +WORKDIR /opt/ CMD ["py.test", "-s", "."] diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 23185f9..2b58961 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -7,11 +7,11 @@ RUN pip3 install pytest twiggy six pyyaml jinja2 ENV PYTHONPATH /opt/pyentrypoint/ ADD pyentrypoint /opt/pyentrypoint/ -ADD tests /opt/pyentrypoint/tests -ADD tests/entrypoint-config.yml /opt/pyentrypoint/ +ADD tests /opt/ + ADD tests/test_template.yml.tpl /tmp/test_template.yml -WORKDIR /opt/pyentrypoint/ +WORKDIR /opt/ CMD ["py.test", "-s", "."] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1c2510d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3961755 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +.PHONY: build test + +build: + @docker-compose build + +test: build + @docker-compose up testpython2 testpython3 diff --git a/README.md b/README.md index 2ebc368..56aca63 100644 --- a/README.md +++ b/README.md @@ -1,157 +1,196 @@ -# py_docker_links +# pyentrypoint -py_docker_links is a kiss python module which helps to list -linked containers inner containers. +__pyentrypoint__ is a tool written in `Python` to manager Docker containers `ENTRYPOINT`. -You can use it with `ENTRYPOINT` script to generate configuration. +This tool avoids writing shell scripts to: + - Handle commands and sub commands + - Identify linked containers + - Generate configuration using `jinja2` templates + - Run commands before starting service ## Usages -We have some containers described in `docker-compose.yml` +### Install in container + +All you need to do is to setup a `yaml` file called `entrypoint-config.yml` and to install __pyentrypoint__ in your `Dockerfile` using pip. + +```dockerfile +FROM debian +# Installing git for example +RUN apt-get update && apt-get install git -y +# Install pyentrypoint +RUN pip install pyentrypoint +# Copy config file in the current WORKDIR +COPY entrypoint-config.yml . +# Set ENTRYPOINT +ENTRYPOINT ['pyentrypoint'] +# git will be the default command +CMD ['git'] +``` + +### Setup entrypoint + +This is an example of `entrypoint-config.yml` file. ```yaml -# Here some dummies containers -test1: - image: busybox - command: sleep 30 - expose: - - 800 - - 8001/udp - environment: - FOO: bar - -test2: - image: busybox - command: sleep 30 - expose: - - 800/udp - - 8001 - -test3: - image: busybox - command: sleep 30 - environment: - FOO: bar - - -# Here our container that embed docker_links.py linked -# with dummies containers -dockerlinks: - build: . - dockerfile: Dockerfile.py3 - command: python docker_links.py - links: - - test1 - - test2 - - test3 +# Entrypoint configuration example + +# This entry should reflect CMD in Dockerfile +command: git + +# This is a list with some subcommands to handle +# when CMD is not `git` here. +# By default, all args started with hyphen are handled. +subcommands: + - "-*" + - clone + - init + - ls-files + # etc... + +# User and group to run the cmd. +# Can be name or uid/gid. +# Affect only command handled. +# Dockerfile USER value by default. +user: 1000 +group: 1000 + +# These files should exist (ADD or COPY) +# and should be jinja templated. +config_files: + - /etc/gitconfig + - .ssh/config + - .ssh/id_rsa + +# These environment variables will be wiped before +# exec command to keep them secret +# CAUTION: if the container is linked to another one, +# theses variables will passed to it anyway +secret_env: + - SSHKEY + +# Links are handled here +# Port, name, protocol or env variable can be used to identify the links +# Raise an error if the link could not be identified +links: + 'ssh': + port: 22 + name: 'ssh*' + protocol: tcp + # env can be list, dictionary or string + env: + FOO: bar + # Single doesn't allow multiple links for this ID + # false by default + single: true + # Set to false to get optional link + # true by default + required: true + +# Commands to run before applying configuration +pre_conf_commands: + - echo something > to_this_file + +# commands to run after applying configuration +post_conf_commands: + - echo "something else" > to_this_another_file + +# Cleanup environment from variables created by linked containers +# before running command (True by default) +clean_env: True + +# Enable debug to debug +debug: true ``` -Start them +### Config templates -```shell -$ docker-compose build && docker-compose up dockerlinks -``` +You can generate configuration for your service with jinga2 template. + +Here an example for an hypothetical ssh config file: -We should get formated json with informations about linked containers. - -```json -{ - "172.17.0.2": { - "environment": { - "FOO": "bar", - "affinity:container": "=a5601d5d225a3e57ea295c7646468067dd1859d4b2ee4574b5bf5542ed372e59" - }, - "names": [ - "d778f6ef9371", - "pythondockertools_test3_1", - "test3", - "test3_1" - ], - "ports": {} - }, - "172.17.0.3": { - "environment": { - "affinity:container": "=78393f27c629fc426af5837a11d30720c8af7a5e029eb173b394f207e7e4701c" - }, - "names": [ - "5fc12cf7b49e", - "pythondockertools_test2_1", - "test2", - "test2_1" - ], - "ports": { - "800": { - "protocol": "tcp" - }, - "8001": { - "protocol": "tcp" - } - } - }, - "172.17.0.4": { - "environment": { - "FOO": "bar", - "affinity:container": "=6a31a66a1aafcd607763dcd916b81b4385a3baf4354c044345255c3eb0bce925" - }, - "names": [ - "d32fc2303721", - "pythondockertools_test1_1", - "test1", - "test1_1" - ], - "ports": { - "800": { - "protocol": "tcp" - }, - "8001": { - "protocol": "udp" - } - } - } -} +```jinga +host server: + hostname {{links.ssh.ip}} + port {{links.ssh.port}} ``` -#### Using as module +Templates with be replaced with ip address and port of the identified link. All links can be accessed from `links.all`, this is a tuple of links you can iterate on it. -```python -from docker_links import DockerLinks +```jinga +{% for link in links.all %} +host {{link.names[0]}} + hostname {{link.ip}} + port {{links.port}} +{% endfor %} +``` -links = DockerLinks() +If you change the option `single` to `false` in the `entrypoint-config.yml`, the identified link `ssh` will become a tuple of links. You must iterate on it in the `jinja` template. -print(links.links()) +```jinga +{% for link in links.ssh %} +host {{link.names[0]}} + hostname {{link.ip}} + port {{links.port}} +{% endfor %} ``` -You'll get a dictionary with all linked containers -```python -{'172.17.0.2': {'environment': {'affinity:container': '=6a31a66a1aafcd607763dcd916b81b4385a3baf4354c044345255c3eb0bce925', 'FOO': 'bar'}, 'ports': {'800': {'protocol': 'tcp'}, '8001': {'protocol': 'udp'}}, 'names': ['d32fc2303721', 'pythondockertools_test1_1', 'test1', 'test1_1']}, '172.17.0.3': {'environment': {'affinity:container': '=78393f27c629fc426af5837a11d30720c8af7a5e029eb173b394f207e7e4701c'}, 'ports': {'800': {'protocol': 'tcp'}, '8001': {'protocol': 'tcp'}}, 'names': ['5fc12cf7b49e', 'pythondockertools_test2_1', 'test2', 'test2_1']}, '172.17.0.5': {'environment': {'affinity:container': '=a5601d5d225a3e57ea295c7646468067dd1859d4b2ee4574b5bf5542ed372e59', 'FOO': 'bar'}, 'ports': {}, 'names': ['d778f6ef9371', 'pythondockertools_test3_1', 'test3', 'test3_1']}} -``` +### Accessible object ---- -You call also get a pretty print json formating +You have 4 available objects in your templates. -```python -from docker_links import DockerLinks + - `config` + - `links` + - `containers` + - `environ` -links = DockerLinks() +#### config -print(links.to_json()) -``` +`Config` reflect the config file. You can retrieve any setup in this object. -or filter links +(see `config.py`) -```python -from docker_links import DockerLinks +#### links -links = DockerLinks() +`Links` handles `Link` objects. You can identify links using globing patterns in the configuration file. -print(links.links('test1', 'test2')) # It also works with container uid -``` +`link` is related to one physical link (one ip and one port). + +`link` handles the following attributes: + - `ip` + - link ip + - `port` + - link port (integer) + - `environ` + - related container environment + - `protocol` + - link protocol (`tcp` or `udp`) + - `uri` + - link uri (example: `tcp://10.0.0.3:80`) + - `names` + - tuple of related container names +#### containers +`containers` handles a tuple of `container` object. + +`container` handles the following attributes: + - `ip` + - container ip + - `environ` + - container environment + - `names` + - List of containers names + - `links` + - Tuple of `link` object related to this container + +#### environ +`environ` is the environment of the container (os.environ). ### Running Tests -To run tests, ensure that docker-compose is installed and run +To run tests, ensure that `docker-compose` and `make` are installed and run ```shell -docker-compose build && docker-compose up testpython2 testpython3 +$ make test +``` diff --git a/pyentrypoint/__init__.py b/pyentrypoint/__init__.py index e69de29..f6e2744 100644 --- a/pyentrypoint/__init__.py +++ b/pyentrypoint/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from .docker_links import DockerLinks +from .entrypoint import Entrypoint + +__version__ = '0.1.1' diff --git a/pyentrypoint/__main__.py b/pyentrypoint/__main__.py new file mode 100644 index 0000000..4fb1b2c --- /dev/null +++ b/pyentrypoint/__main__.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from sys import argv + +from .entrypoint import main as m + + +def main(): + m(argv) diff --git a/pyentrypoint/command.py b/pyentrypoint/command.py index 86bed35..0212ba1 100644 --- a/pyentrypoint/command.py +++ b/pyentrypoint/command.py @@ -1,4 +1,7 @@ "Command object" +from __future__ import absolute_import +from __future__ import unicode_literals + import fnmatch import os @@ -35,8 +38,8 @@ class Command(object): def run(self): if os.getuid() is 0: - os.setuid(self.config.user) os.setgid(self.config.group) + os.setuid(self.config.user) if self.config.clean_env: self._clean_links_env() for item in self.config.secret_env: @@ -44,6 +47,5 @@ class Command(object): del(self.env[item]) if not self.args or \ fnmatch.filter(self.config.subcommands, self.args[0]): - args = self.args if self.args else [self.command] - os.execvpe(self.command, args, os.environ) - os.execvpe(args[0], args, os.environ) + self.args.insert(0, self.command) + os.execvpe(self.args[0], self.args, os.environ) diff --git a/pyentrypoint/config.py b/pyentrypoint/config.py index 871990a..314fe24 100644 --- a/pyentrypoint/config.py +++ b/pyentrypoint/config.py @@ -1,25 +1,35 @@ """ - Configuration + Configuration object """ +from __future__ import absolute_import +from __future__ import unicode_literals + import os from grp import getgrnam from io import open from pwd import getpwnam -from command import Command -from docker_links import DockerLinks -from links import Links from six import string_types from yaml import load from yaml import Loader +from .command import Command +from .docker_links import DockerLinks +from .links import Links + +__all__ = ['Config'] + class Config(object): - """Get entrypoint config""" + """ + Get entrypoint config + + Parse entrypoint-config.yml + + Config file should always be in WORKDIR and named entrypoint-config.yml + """ - # Config file should always be in WORKDIR and named - # entrypoint-config.yml _config_file = 'entrypoint-config.yml' def _return_item_lst(self, item): @@ -36,12 +46,8 @@ class Config(object): self._links = None if not os.path.isfile(self._config_file): return - try: - with open(self._config_file) as f: - self._config = load(stream=f, Loader=Loader) - except Exception as err: - # TODO: logger - print(err) + with open(self._config_file) as f: + self._config = load(stream=f, Loader=Loader) self._args = args @property @@ -110,14 +116,12 @@ class Config(object): @property def pre_conf_commands(self): """Return list of preconf commands""" - if 'pre_conf_commands' in self._config: - return self._return_item_lst(self._config['pre_conf_command']) + return self._return_item_lst('pre_conf_commands') @property def post_conf_commands(self): """Return list of postconf commands""" - if 'post_conf_commands' in self._config: - return self._return_item_lst(self._config['post_conf_command']) + return self._return_item_lst('post_conf_commands') @property def clean_env(self): diff --git a/pyentrypoint/container.py b/pyentrypoint/container.py index f8e9306..d9e5b92 100644 --- a/pyentrypoint/container.py +++ b/pyentrypoint/container.py @@ -1,6 +1,8 @@ """ Container object handle a single container link """ +from __future__ import absolute_import +from __future__ import unicode_literals class Container(object): diff --git a/pyentrypoint/docker_links.py b/pyentrypoint/docker_links.py index 79bf727..cb75aea 100644 --- a/pyentrypoint/docker_links.py +++ b/pyentrypoint/docker_links.py @@ -3,12 +3,17 @@ DockerLinks a kiss class which help to get links info in a docker container. """ +from __future__ import absolute_import +from __future__ import unicode_literals + import json import os import re -from container import Container -from links import Links +from .container import Container +from .links import Links + +__all__ = ['DockerLinks'] class DockerLinks(object): diff --git a/pyentrypoint/entrypoint.py b/pyentrypoint/entrypoint.py index a3b798e..43b0273 100644 --- a/pyentrypoint/entrypoint.py +++ b/pyentrypoint/entrypoint.py @@ -2,20 +2,26 @@ """ Smart docker-entrypoint """ +from __future__ import absolute_import +from __future__ import unicode_literals + import os from subprocess import PIPE from subprocess import Popen from sys import argv +from sys import stdout -from command import Command -from config import Config -from docker_links import DockerLinks from jinja2 import Environment from jinja2 import FileSystemLoader from twiggy import levels from twiggy import log from twiggy import quickSetup +from .config import Config +from .docker_links import DockerLinks + +__all__ = ['Entrypoint', 'main'] + class Entrypoint(object): @@ -28,7 +34,7 @@ class Entrypoint(object): def __init__(self, args=[]): self._set_logguer() try: - self.config = Config() + self.config = Config(args) except Exception as err: self.log.error(err) if self.config.debug: @@ -51,25 +57,42 @@ class Entrypoint(object): proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) out, err = proc.communicate() - self.log.info(out) - self.log.warning(err) + def dispout(output, cb): + enc = stdout.encoding or 'UTF-8' + output = output.decode(enc).split('\n') + for line in output: + cb(line) + + if out: + dispout(out, self.log.info) + if err: + dispout(err, self.log.warning) if proc.returncode: raise Exception('Command exit code: {}'.format(proc.returncode)) + def run_pre_conf_cmds(self): + for cmd in self.config.pre_conf_commands: + self.run_conf_cmd(cmd) + + def run_post_conf_cmds(self): + for cmd in self.config.post_conf_commands: + self.run_conf_cmd(cmd) + def launch(self): - self.args.pop(0) - command = Command(self.config, self.args) - command.run() + self.config.command.run() -if __name__ == '__main__': - entry = Entrypoint(argv) +def main(argv): + argv.pop(0) + entry = Entrypoint(args=argv) try: - for cmd in entry.config.pre_conf_commands: - entry.run_conf_cmd(cmd) + entry.run_pre_conf_cmds() entry.apply_conf() - for cmd in entry.config.post_conf_commands: - entry.run_conf_cmd(cmd) + entry.run_post_conf_cmds() entry.launch() except Exception as e: - print(e) + entry.log.error(str(e)) + + +if __name__ == '__main__': + main(argv) diff --git a/pyentrypoint/excepts.py b/pyentrypoint/excepts.py deleted file mode 100644 index aae8dee..0000000 --- a/pyentrypoint/excepts.py +++ /dev/null @@ -1,7 +0,0 @@ -""" - Custom exceptions -""" - - -class BadLink(Exception): - pass diff --git a/pyentrypoint/links.py b/pyentrypoint/links.py index e734404..4fbffb9 100644 --- a/pyentrypoint/links.py +++ b/pyentrypoint/links.py @@ -1,10 +1,15 @@ """ Link handle a single link to another container, determined by his port """ +from __future__ import absolute_import +from __future__ import unicode_literals + import fnmatch from six import viewitems +__all__ = ['Link', 'Links'] + class Link(object): @@ -12,7 +17,7 @@ class Link(object): def __init__(self, ip, env, port, protocol, names): self.ip = ip - self.env = env + self.environ = env self.port = int(port) self.protocol = protocol self.uri = '{protocol}://{ip}:{port}'.format( @@ -37,10 +42,10 @@ class Link(object): def _filter_env(self, env): "return true if env match" if isinstance(env, dict): - return viewitems(env) <= viewitems(self.env) + return viewitems(env) <= viewitems(self.environ) if isinstance(env, list): - return bool([key for key in env if key in self.env]) - return str(env) in self.env + return bool([key for key in env if key in self.environ]) + return str(env) in self.environ class Links(object): diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..81c8e2e --- /dev/null +++ b/setup.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import find_packages +from setuptools import setup + +# Thanks Sam and Max + +setup( + + name='pyentrypoint', + + version='0.1.9', + + packages=find_packages(), + + author="Christophe Mehay", + + author_email="cmehay@nospam.student.42.fr", + + description="pyentrypoint manages entrypoints in Docker containers.", + + long_description=open('README.md').read(), + + install_requires=['Jinja2>=2.8', + 'PyYAML>=3.11', + 'Twiggy>=0.4.7', + 'argparse>=1.4.0', + 'six>=1.10.0'], + + include_package_data=True, + + url='http://github.com/cmehay/pyentrypoint', + + classifiers=[ + "Programming Language :: Python", + "Development Status :: 1 - Planning", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", + "Topic :: System :: Installation/Setup", + ], + + + entry_points={ + 'console_scripts': [ + 'pyentrypoint = pyentrypoint.__main__:main', + ], + }, + + license="WTFPL", + +) diff --git a/tests/entrypoint-config.yml b/tests/entrypoint-config.yml index 9784555..9ad9882 100644 --- a/tests/entrypoint-config.yml +++ b/tests/entrypoint-config.yml @@ -23,3 +23,14 @@ links: env: FOO: bar required: true + +pre_conf_commands: + - echo TEST > /tmp/OK + - echo "INFO IS DISPLAYED" + - echo "WARNING IS DISPLAYED\nON TWO LINES" 1>&2 + +post_conf_commands: + - echo TEST2 > /tmp/OKOK + - echo TEST3 > /tmp/OKOKOK + - echo "INFO IS DISPLAYED\nON TWO LINES" + - echo "WARNING IS DISPLAYED" 1>&2 diff --git a/tests/pyentrypoint_test.py b/tests/pyentrypoint_test.py index f87740e..32356ab 100644 --- a/tests/pyentrypoint_test.py +++ b/tests/pyentrypoint_test.py @@ -1,11 +1,16 @@ # Tests using pytest +from __future__ import absolute_import +from __future__ import unicode_literals + import fnmatch +from multiprocessing import Process -from docker_links import DockerLinks -from entrypoint import Entrypoint from yaml import load from yaml import Loader +from pyentrypoint import DockerLinks +from pyentrypoint import Entrypoint + LINKS = [ 'test1', 'test2', @@ -114,3 +119,36 @@ def test_templates(): # test names for test_name in test_names: assert test_name in test['All names'] + + +def test_conf_commands(): + entry = Entrypoint() + + for cmd in entry.config.pre_conf_commands: + entry.run_conf_cmd(cmd) + for cmd in entry.config.post_conf_commands: + entry.run_conf_cmd(cmd) + + with open('/tmp/OK') as f: + assert f.readline().startswith('TEST') + + with open('/tmp/OKOK') as f: + assert f.readline().startswith('TEST2') + + with open('/tmp/OKOKOK') as f: + assert f.readline().startswith('TEST3') + + +def test_command(): + run = [ + (Process(target=Entrypoint(['OK']).launch), 'OK\n'), + (Process(target=Entrypoint(['echo', 'mdr']).launch), 'mdr\n'), + (Process(target=Entrypoint(['OK', 'mdr']).launch), 'OK mdr\n'), + ] + # capsys.readouterr() + + for proc, test in run: + proc.start() + proc.join() + # out, _ = capsys.readouterr() + # assert out == test diff --git a/tests/test_template.yml.tpl b/tests/test_template.yml.tpl index e853e05..bc51793 100644 --- a/tests/test_template.yml.tpl +++ b/tests/test_template.yml.tpl @@ -19,15 +19,15 @@ All links 2: All environ: {% for link in links.all %} - {% for key in link.env %} - {{key}}: {{link.env[key]}} + {% for key in link.environ %} + {{key}}: {{link.environ[key]}} {% endfor %} {% endfor %} All links 2 environ: {% for link in links.test2 %} - {% for key in link.env %} - {{key}}: {{link.env[key]}} + {% for key in link.environ %} + {{key}}: {{link.environ[key]}} {% endfor %} {% endfor %}