Add `run_post_commands_in_parallele` to run each post commands in separate process and stream output

.dockerignore

@@ -1,6 +1,5 @@
 *
 !pyentrypoint
-!setup.py
 !README.md
 !tests
 !poetry.lock

Dockerfile.py3

@@ -2,12 +2,11 @@ FROM python:3
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3-alpine

@@ -2,13 +2,12 @@ FROM python:3-alpine
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     apk add gcc && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3.6

@@ -2,12 +2,11 @@ FROM python:3.6
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3.6-alpine

@@ -2,13 +2,12 @@ FROM python:3.6-alpine
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     apk add gcc && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3.7

@@ -2,12 +2,11 @@ FROM python:3.7
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3.7-alpine

@@ -2,13 +2,12 @@ FROM python:3.7-alpine
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     apk add gcc && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3.8

@@ -2,12 +2,11 @@ FROM python:3.8
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

Dockerfile.py3.8-alpine

@@ -2,13 +2,12 @@ FROM python:3.8-alpine
 
 ENV POETRY_VIRTUALENVS_CREATE=false
 
-ADD . /tmp/
+ADD . /usr/local/src/
 
-RUN cd /tmp && \
+RUN cd /usr/local/src/ && \
     apk add gcc && \
     pip install --upgrade pip poetry && \
-    poetry install --no-dev && \
+    poetry install --no-dev
-    rm -rf *
 
 ONBUILD ADD entrypoint-config.yml .
 

README.md

@@ -181,8 +181,12 @@ pre_conf_commands:
 post_conf_commands:
     - echo "something else" > to_this_another_file
 
+# commands to run in parallele with the main command
 post_run_commands:
-    - echo run commands after started service
+    - echo do something in parallele with the main command
+
+# run post_run_commands in parallele or sequentially (default is sequential)
+run_post_commands_in_parallele: true # default false
 
 # Reload service when configuration change by sending a signal to process
 reload:

docs/config.rst

@@ -93,8 +93,12 @@ This is an example of ``entrypoint-config.yml`` file.
     post_conf_commands:
         - echo "something else" > to_this_another_file
 
+    # commands to run in parallele with the main command
     post_run_commands:
-        - echo run commands after started service
+        - echo do something in parallele with the main command
+
+    # run post_run_commands in parallele or sequentially (default is sequential)
+    run_post_commands_in_parallele: true # default false
 
     # Reload service when configuration change by sending a signal to process
     reload:
@@ -281,6 +285,14 @@ List of shell commands to run after service is started
         - sleep 5
         - echo "something else" > to_this_another_file
 
+run_post_commands_in_parallele
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+run post_run_commands in paralle or sequentially (default is sequential)
+
+.. code:: yaml
+
+    run_post_commands_in_parallele: true
 
 reload
 ^^^^^^

entrypoint-config.yml

@@ -76,6 +76,13 @@ pre_conf_commands:
 post_conf_commands:
     - echo something even useless
 
+# commands to run in parallele with the main command
+post_run_commands:
+    - echo do something in parallele with the main command is ran
+
+# run porst_run_commands in parallele or sequentially (default is sequential)
+run_post_commands_in_parallele: true # default false
+
 # Reload service when configuration change by sending a signal to process
 reload:
     signal: SIGHUP # Optional, signal to send, default is SIGHUP
@@ -84,7 +91,7 @@ reload:
     files: # Optional, list of files to watch
         - /etc/conf/to/watch
 # can also be enabled with a boolean:
-reload: true
+# reload: true
 
 # Cleanup environment from variables created by linked containers
 # before running command (True by default)

pyentrypoint/config.py

@@ -335,8 +335,8 @@ class Config(ConfigMeta):
     @property
     def debug(self):
         """Enable debug logs."""
-        if envtobool('ENTRYPOINT_DEBUG', False):
+        if 'ENTRYPOINT_DEBUG' in os.environ:
-            return True
+            return envtobool('ENTRYPOINT_DEBUG', False)
         if 'debug' in self._config:
             return bool(self._get_by_command(item='debug',
                                              value_types=[bool]))
@@ -347,8 +347,8 @@ class Config(ConfigMeta):
         """Disable logging"""
         if self.debug:
             return False
-        if envtobool('ENTRYPOINT_QUIET', False):
+        if 'ENTRYPOINT_QUIET' in os.environ:
-            return True
+            return envtobool('ENTRYPOINT_QUIET', False)
         return bool(self._config.get('quiet', False))
 
     @property
@@ -364,6 +364,22 @@ class Config(ConfigMeta):
     @property
     def raw_output(self):
         """Check if command output should be displayed using logging or not"""
-        if envtobool('ENTRYPOINT_RAW', False):
+        if 'ENTRYPOINT_RAW' in os.environ:
-            return True
+            return envtobool('ENTRYPOINT_RAW', False)
-        return bool(self._config.get('raw_output', False))
+        if 'raw_output' in self._config:
+            return bool(self._get_by_command(item='raw_output',
+                                             value_types=[bool]))
+        return False
+
+    @property
+    def run_post_commands_in_parallele(self):
+        """Run all post post run commands in parallele using process"""
+        if 'ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE' in os.environ:
+            return envtobool('ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE',
+                             False)
+        if 'run_post_commands_in_parallele' in self._config:
+            return bool(self._get_by_command(
+                item='run_post_commands_in_parallele',
+                value_types=[bool]
+            ))
+        return False

pyentrypoint/runner.py

@@ -2,7 +2,6 @@
 from multiprocessing import Process
 from subprocess import PIPE
 from subprocess import Popen
-from sys import stdout as stdoutput
 
 from .logs import Logs
 
@@ -14,37 +13,52 @@ class Runner(object):
         self.log = Logs().log
         self.cmds = cmds
         self.raw_output = config.raw_output
+        self.parallele_run = config.run_post_commands_in_parallele
+        self.stdout_cb = self.log.info if not self.raw_output else print
+        self.stderr_cb = self.log.warning if not self.raw_output else print
+
+    def stdout(self, output):
+        for line in output:
+            self.stdout_cb(line.rstrip())
+
+    def stderr(self, output):
+        for line in output:
+            self.stderr_cb(line.rstrip())
+
+    def stream_output(self, proc):
+        display_stdout = Process(target=self.stdout, args=(proc.stdout,))
+        display_stdout.start()
+        display_stderr = Process(target=self.stderr, args=(proc.stderr,))
+        display_stderr.start()
 
     def run_cmd(self, cmd, stdout=False):
         self.log.debug('run command: {}'.format(cmd))
-        proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
+        with Popen(cmd,
-        out, err = proc.communicate()
+                   shell=True,
-
+                   stdout=PIPE,
-        def dispout(output, cb):
+                   stderr=PIPE,
-            enc = stdoutput.encoding or 'UTF-8'
+                   universal_newlines=True) as proc:
-            output = output.decode(enc).split('\n')
-            lenght = len(output)
-            for c, line in enumerate(output):
-                if c + 1 == lenght and not len(line):
-                    # Do not display last empty line
-                    break
-                cb(line)
-
-        if err:
-            display_cb = self.log.warning if not self.raw_output else print
-            dispout(err, display_cb)
-        if out:
-            display_cb = self.log.info if not self.raw_output else print
             if stdout:
-                return out.decode()
+                output, _ = proc.communicate()
-            dispout(out, display_cb)
+            else:
+                self.stream_output(proc)
         if proc.returncode:
             raise Exception('Command exit code: {}'.format(proc.returncode))
+        if stdout and output:
+            return output
 
     def run(self):
         for cmd in self.cmds:
             self.run_cmd(cmd)
 
     def run_in_process(self):
+        if self.parallele_run:
+            return self.run_in_parallele()
         self.proc = Process(target=self.run)
         self.proc.start()
+
+    def run_in_parallele(self):
+        for cmd in self.cmds:
+            self.proc = Process(target=self.run_cmd,
+                                args=[cmd])
+            self.proc.start()

pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "pyentrypoint"
-version = "0.7.2"
+version = "0.7.3"
 description = "pyentrypoint manages entrypoints in Docker containers."
 license = "WTFPL"
 classifiers = ["Programming Language :: Python", "Development Status :: 1 - Planning", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Topic :: System :: Installation/Setup"]
@@ -25,7 +25,7 @@ pytest-mock = "^3.1.0"
 pre-commit = "^2.3.0"
 
 [build-system]
-requires = ["poetry>=0.12"]
+requires = ["poetry>=1.0.8"]
 build-backend = "poetry.masonry.api"
 
 [tool.poetry.scripts]

tests/configs/runner.yml

@@ -1,5 +1,6 @@
 post_run_commands:
-    - sleep 2
+    - date '+%s' > /tmp/timestamp1
-    - echo 'OK' > /tmp/runner_test
+    - sleep 4
+    - date '+%s' > /tmp/timestamp2
 
 command: sleep

tests/configs/runner_parallele.yml

@@ -0,0 +1,8 @@
+post_run_commands:
+    - date '+%s' > /tmp/timestamp1
+    - sleep 4
+    - date '+%s' > /tmp/timestamp2
+
+command: sleep
+
+run_post_commands_in_parallele: true

tests/pyentrypoint_test.py

@@ -108,7 +108,7 @@ def test_containers():
 def test_templates():
     test_confs = ['configs/base.yml']
     for test_conf in test_confs:
-        entry = Entrypoint(conf='configs/base.yml')
+        entry = Entrypoint(conf=test_conf)
 
         conf = entry.config
 

tests/reloader_test.py

@@ -1,25 +1,14 @@
 "Tests for reloader"
-
-
-try:
-    # Python2
-    import mock
-except ImportError:
-    # Python3
-    from unittest import mock
-
 import os
-
-from pyentrypoint import Entrypoint
-
 import subprocess
-
 from signal import SIGHUP
-
 from time import sleep
+from unittest import mock
 
 from commons import clean_env
 
+from pyentrypoint import Entrypoint
+
 
 def teardown_function(function):
     clean_env()

tests/runner_test.py

@@ -5,18 +5,39 @@ from multiprocessing import Process
 from pyentrypoint import Entrypoint
 
 
+def compare_timestamps():
+    with open('/tmp/timestamp1', 'r') as f:
+        first = int(f.readline())
+    with open('/tmp/timestamp2', 'r') as f:
+        second = int(f.readline())
+    return second - first
+
+
 def test_runner():
     run = [
         (Process(target=Entrypoint(
             conf='configs/runner.yml',
-            args=['sleep', '5']).launch),
+            args=['sleep', '5']).launch), 0, 0),
-         '/tmp/runner_test', 0, 0),
+    ]
+
+    for proc, uid, gid in run:
+        proc.start()
+        proc.join()
+        assert compare_timestamps() > 3
+        assert os.stat('/tmp/timestamp1').st_uid == uid
+        assert os.stat('/tmp/timestamp1').st_gid == gid
+
+
+def test_runner_parallele():
+    run = [
+        (Process(target=Entrypoint(
+            conf='configs/runner_parallele.yml',
+            args=['sleep', '5']).launch), 0, 0),
     ]
 
-    for proc, test, uid, gid in run:
+    for proc, uid, gid in run:
         proc.start()
         proc.join()
-        with open(test, 'r') as f:
+        assert compare_timestamps() < 1
-            assert f.readline().startswith('OK')
+        assert os.stat('/tmp/timestamp1').st_uid == uid
-        assert os.stat(test).st_uid == uid
+        assert os.stat('/tmp/timestamp1').st_gid == gid
-        assert os.stat(test).st_gid == gid