From 6a457f239831413712fdbfdcfc0d4627207cf821 Mon Sep 17 00:00:00 2001 From: Christophe Mehay Date: Sun, 7 Jun 2020 10:06:31 +0200 Subject: [PATCH] Add `run_post_commands_in_parallele` to run each post commands in separate process and stream output --- .dockerignore | 1 - Dockerfile.py3 | 7 ++-- Dockerfile.py3-alpine | 7 ++-- Dockerfile.py3.6 | 7 ++-- Dockerfile.py3.6-alpine | 7 ++-- Dockerfile.py3.7 | 7 ++-- Dockerfile.py3.7-alpine | 7 ++-- Dockerfile.py3.8 | 7 ++-- Dockerfile.py3.8-alpine | 7 ++-- README.md | 6 +++- docs/config.rst | 14 +++++++- entrypoint-config.yml | 9 ++++- pyentrypoint/config.py | 30 ++++++++++++---- pyentrypoint/runner.py | 56 +++++++++++++++++++----------- pyproject.toml | 4 +-- tests/configs/runner.yml | 5 +-- tests/configs/runner_parallele.yml | 8 +++++ tests/pyentrypoint_test.py | 2 +- tests/reloader_test.py | 17 ++------- tests/runner_test.py | 35 +++++++++++++++---- 20 files changed, 153 insertions(+), 90 deletions(-) create mode 100644 tests/configs/runner_parallele.yml diff --git a/.dockerignore b/.dockerignore index 7218461..214b4f8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,5 @@ * !pyentrypoint -!setup.py !README.md !tests !poetry.lock diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 229a5db..6c4931d 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -2,12 +2,11 @@ FROM python:3 ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3-alpine b/Dockerfile.py3-alpine index 9569b45..a21c259 100644 --- a/Dockerfile.py3-alpine +++ b/Dockerfile.py3-alpine @@ -2,13 +2,12 @@ FROM python:3-alpine ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ apk add gcc && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3.6 b/Dockerfile.py3.6 index c89fc37..ca0ec5c 100644 --- a/Dockerfile.py3.6 +++ b/Dockerfile.py3.6 @@ -2,12 +2,11 @@ FROM python:3.6 ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3.6-alpine b/Dockerfile.py3.6-alpine index 38629d3..a4e3fcd 100644 --- a/Dockerfile.py3.6-alpine +++ b/Dockerfile.py3.6-alpine @@ -2,13 +2,12 @@ FROM python:3.6-alpine ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ apk add gcc && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3.7 b/Dockerfile.py3.7 index 48a34b4..8bac180 100644 --- a/Dockerfile.py3.7 +++ b/Dockerfile.py3.7 @@ -2,12 +2,11 @@ FROM python:3.7 ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3.7-alpine b/Dockerfile.py3.7-alpine index 9481711..e1caf69 100644 --- a/Dockerfile.py3.7-alpine +++ b/Dockerfile.py3.7-alpine @@ -2,13 +2,12 @@ FROM python:3.7-alpine ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ apk add gcc && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3.8 b/Dockerfile.py3.8 index 816eab6..a83ba64 100644 --- a/Dockerfile.py3.8 +++ b/Dockerfile.py3.8 @@ -2,12 +2,11 @@ FROM python:3.8 ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/Dockerfile.py3.8-alpine b/Dockerfile.py3.8-alpine index c54e790..68e1048 100644 --- a/Dockerfile.py3.8-alpine +++ b/Dockerfile.py3.8-alpine @@ -2,13 +2,12 @@ FROM python:3.8-alpine ENV POETRY_VIRTUALENVS_CREATE=false -ADD . /tmp/ +ADD . /usr/local/src/ -RUN cd /tmp && \ +RUN cd /usr/local/src/ && \ apk add gcc && \ pip install --upgrade pip poetry && \ - poetry install --no-dev && \ - rm -rf * + poetry install --no-dev ONBUILD ADD entrypoint-config.yml . diff --git a/README.md b/README.md index 3cb910c..a6f71ad 100644 --- a/README.md +++ b/README.md @@ -181,8 +181,12 @@ pre_conf_commands: post_conf_commands: - echo "something else" > to_this_another_file +# commands to run in parallele with the main command post_run_commands: - - echo run commands after started service + - echo do something in parallele with the main command + +# run post_run_commands in parallele or sequentially (default is sequential) +run_post_commands_in_parallele: true # default false # Reload service when configuration change by sending a signal to process reload: diff --git a/docs/config.rst b/docs/config.rst index a27f7aa..8327c13 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -93,8 +93,12 @@ This is an example of ``entrypoint-config.yml`` file. post_conf_commands: - echo "something else" > to_this_another_file + # commands to run in parallele with the main command post_run_commands: - - echo run commands after started service + - echo do something in parallele with the main command + + # run post_run_commands in parallele or sequentially (default is sequential) + run_post_commands_in_parallele: true # default false # Reload service when configuration change by sending a signal to process reload: @@ -281,6 +285,14 @@ List of shell commands to run after service is started - sleep 5 - echo "something else" > to_this_another_file +run_post_commands_in_parallele +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +run post_run_commands in paralle or sequentially (default is sequential) + +.. code:: yaml + + run_post_commands_in_parallele: true reload ^^^^^^ diff --git a/entrypoint-config.yml b/entrypoint-config.yml index cd10572..6301c48 100644 --- a/entrypoint-config.yml +++ b/entrypoint-config.yml @@ -76,6 +76,13 @@ pre_conf_commands: post_conf_commands: - echo something even useless +# commands to run in parallele with the main command +post_run_commands: + - echo do something in parallele with the main command is ran + +# run porst_run_commands in parallele or sequentially (default is sequential) +run_post_commands_in_parallele: true # default false + # Reload service when configuration change by sending a signal to process reload: signal: SIGHUP # Optional, signal to send, default is SIGHUP @@ -84,7 +91,7 @@ reload: files: # Optional, list of files to watch - /etc/conf/to/watch # can also be enabled with a boolean: -reload: true +# reload: true # Cleanup environment from variables created by linked containers # before running command (True by default) diff --git a/pyentrypoint/config.py b/pyentrypoint/config.py index 8c90263..c224448 100755 --- a/pyentrypoint/config.py +++ b/pyentrypoint/config.py @@ -335,8 +335,8 @@ class Config(ConfigMeta): @property def debug(self): """Enable debug logs.""" - if envtobool('ENTRYPOINT_DEBUG', False): - return True + if 'ENTRYPOINT_DEBUG' in os.environ: + return envtobool('ENTRYPOINT_DEBUG', False) if 'debug' in self._config: return bool(self._get_by_command(item='debug', value_types=[bool])) @@ -347,8 +347,8 @@ class Config(ConfigMeta): """Disable logging""" if self.debug: return False - if envtobool('ENTRYPOINT_QUIET', False): - return True + if 'ENTRYPOINT_QUIET' in os.environ: + return envtobool('ENTRYPOINT_QUIET', False) return bool(self._config.get('quiet', False)) @property @@ -364,6 +364,22 @@ class Config(ConfigMeta): @property def raw_output(self): """Check if command output should be displayed using logging or not""" - if envtobool('ENTRYPOINT_RAW', False): - return True - return bool(self._config.get('raw_output', False)) + if 'ENTRYPOINT_RAW' in os.environ: + return envtobool('ENTRYPOINT_RAW', False) + if 'raw_output' in self._config: + return bool(self._get_by_command(item='raw_output', + value_types=[bool])) + return False + + @property + def run_post_commands_in_parallele(self): + """Run all post post run commands in parallele using process""" + if 'ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE' in os.environ: + return envtobool('ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE', + False) + if 'run_post_commands_in_parallele' in self._config: + return bool(self._get_by_command( + item='run_post_commands_in_parallele', + value_types=[bool] + )) + return False diff --git a/pyentrypoint/runner.py b/pyentrypoint/runner.py index 2b8e233..37239ad 100644 --- a/pyentrypoint/runner.py +++ b/pyentrypoint/runner.py @@ -2,7 +2,6 @@ from multiprocessing import Process from subprocess import PIPE from subprocess import Popen -from sys import stdout as stdoutput from .logs import Logs @@ -14,37 +13,52 @@ class Runner(object): self.log = Logs().log self.cmds = cmds self.raw_output = config.raw_output + self.parallele_run = config.run_post_commands_in_parallele + self.stdout_cb = self.log.info if not self.raw_output else print + self.stderr_cb = self.log.warning if not self.raw_output else print + + def stdout(self, output): + for line in output: + self.stdout_cb(line.rstrip()) + + def stderr(self, output): + for line in output: + self.stderr_cb(line.rstrip()) + + def stream_output(self, proc): + display_stdout = Process(target=self.stdout, args=(proc.stdout,)) + display_stdout.start() + display_stderr = Process(target=self.stderr, args=(proc.stderr,)) + display_stderr.start() def run_cmd(self, cmd, stdout=False): self.log.debug('run command: {}'.format(cmd)) - proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) - out, err = proc.communicate() - - def dispout(output, cb): - enc = stdoutput.encoding or 'UTF-8' - output = output.decode(enc).split('\n') - lenght = len(output) - for c, line in enumerate(output): - if c + 1 == lenght and not len(line): - # Do not display last empty line - break - cb(line) - - if err: - display_cb = self.log.warning if not self.raw_output else print - dispout(err, display_cb) - if out: - display_cb = self.log.info if not self.raw_output else print + with Popen(cmd, + shell=True, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True) as proc: if stdout: - return out.decode() - dispout(out, display_cb) + output, _ = proc.communicate() + else: + self.stream_output(proc) if proc.returncode: raise Exception('Command exit code: {}'.format(proc.returncode)) + if stdout and output: + return output def run(self): for cmd in self.cmds: self.run_cmd(cmd) def run_in_process(self): + if self.parallele_run: + return self.run_in_parallele() self.proc = Process(target=self.run) self.proc.start() + + def run_in_parallele(self): + for cmd in self.cmds: + self.proc = Process(target=self.run_cmd, + args=[cmd]) + self.proc.start() diff --git a/pyproject.toml b/pyproject.toml index 90c2133..87be715 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool] [tool.poetry] name = "pyentrypoint" -version = "0.7.2" +version = "0.7.3" description = "pyentrypoint manages entrypoints in Docker containers." license = "WTFPL" classifiers = ["Programming Language :: Python", "Development Status :: 1 - Planning", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Topic :: System :: Installation/Setup"] @@ -25,7 +25,7 @@ pytest-mock = "^3.1.0" pre-commit = "^2.3.0" [build-system] -requires = ["poetry>=0.12"] +requires = ["poetry>=1.0.8"] build-backend = "poetry.masonry.api" [tool.poetry.scripts] diff --git a/tests/configs/runner.yml b/tests/configs/runner.yml index 55ab63d..3ebb844 100644 --- a/tests/configs/runner.yml +++ b/tests/configs/runner.yml @@ -1,5 +1,6 @@ post_run_commands: - - sleep 2 - - echo 'OK' > /tmp/runner_test + - date '+%s' > /tmp/timestamp1 + - sleep 4 + - date '+%s' > /tmp/timestamp2 command: sleep diff --git a/tests/configs/runner_parallele.yml b/tests/configs/runner_parallele.yml new file mode 100644 index 0000000..0038930 --- /dev/null +++ b/tests/configs/runner_parallele.yml @@ -0,0 +1,8 @@ +post_run_commands: + - date '+%s' > /tmp/timestamp1 + - sleep 4 + - date '+%s' > /tmp/timestamp2 + +command: sleep + +run_post_commands_in_parallele: true diff --git a/tests/pyentrypoint_test.py b/tests/pyentrypoint_test.py index 5dad6df..7b020e5 100644 --- a/tests/pyentrypoint_test.py +++ b/tests/pyentrypoint_test.py @@ -108,7 +108,7 @@ def test_containers(): def test_templates(): test_confs = ['configs/base.yml'] for test_conf in test_confs: - entry = Entrypoint(conf='configs/base.yml') + entry = Entrypoint(conf=test_conf) conf = entry.config diff --git a/tests/reloader_test.py b/tests/reloader_test.py index 97a98ac..15899c3 100644 --- a/tests/reloader_test.py +++ b/tests/reloader_test.py @@ -1,25 +1,14 @@ "Tests for reloader" - - -try: - # Python2 - import mock -except ImportError: - # Python3 - from unittest import mock - import os - -from pyentrypoint import Entrypoint - import subprocess - from signal import SIGHUP - from time import sleep +from unittest import mock from commons import clean_env +from pyentrypoint import Entrypoint + def teardown_function(function): clean_env() diff --git a/tests/runner_test.py b/tests/runner_test.py index 6a4eb4f..47d2353 100644 --- a/tests/runner_test.py +++ b/tests/runner_test.py @@ -5,18 +5,39 @@ from multiprocessing import Process from pyentrypoint import Entrypoint +def compare_timestamps(): + with open('/tmp/timestamp1', 'r') as f: + first = int(f.readline()) + with open('/tmp/timestamp2', 'r') as f: + second = int(f.readline()) + return second - first + + def test_runner(): run = [ (Process(target=Entrypoint( conf='configs/runner.yml', - args=['sleep', '5']).launch), - '/tmp/runner_test', 0, 0), + args=['sleep', '5']).launch), 0, 0), + ] + + for proc, uid, gid in run: + proc.start() + proc.join() + assert compare_timestamps() > 3 + assert os.stat('/tmp/timestamp1').st_uid == uid + assert os.stat('/tmp/timestamp1').st_gid == gid + + +def test_runner_parallele(): + run = [ + (Process(target=Entrypoint( + conf='configs/runner_parallele.yml', + args=['sleep', '5']).launch), 0, 0), ] - for proc, test, uid, gid in run: + for proc, uid, gid in run: proc.start() proc.join() - with open(test, 'r') as f: - assert f.readline().startswith('OK') - assert os.stat(test).st_uid == uid - assert os.stat(test).st_gid == gid + assert compare_timestamps() < 1 + assert os.stat('/tmp/timestamp1').st_uid == uid + assert os.stat('/tmp/timestamp1').st_gid == gid