2
0
mirror of https://github.com/cmehay/pyentrypoint synced 2024-10-30 15:21:11 +00:00

Add run_post_commands_in_parallele to run each post commands in separate process and stream output

This commit is contained in:
Christophe Mehay 2020-06-07 10:06:31 +02:00
parent 3908415898
commit 6a457f2398
20 changed files with 153 additions and 90 deletions

View File

@ -1,6 +1,5 @@
* *
!pyentrypoint !pyentrypoint
!setup.py
!README.md !README.md
!tests !tests
!poetry.lock !poetry.lock

View File

@ -2,12 +2,11 @@ FROM python:3
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,13 +2,12 @@ FROM python:3-alpine
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
apk add gcc && \ apk add gcc && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,12 +2,11 @@ FROM python:3.6
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,13 +2,12 @@ FROM python:3.6-alpine
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
apk add gcc && \ apk add gcc && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,12 +2,11 @@ FROM python:3.7
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,13 +2,12 @@ FROM python:3.7-alpine
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
apk add gcc && \ apk add gcc && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,12 +2,11 @@ FROM python:3.8
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -2,13 +2,12 @@ FROM python:3.8-alpine
ENV POETRY_VIRTUALENVS_CREATE=false ENV POETRY_VIRTUALENVS_CREATE=false
ADD . /tmp/ ADD . /usr/local/src/
RUN cd /tmp && \ RUN cd /usr/local/src/ && \
apk add gcc && \ apk add gcc && \
pip install --upgrade pip poetry && \ pip install --upgrade pip poetry && \
poetry install --no-dev && \ poetry install --no-dev
rm -rf *
ONBUILD ADD entrypoint-config.yml . ONBUILD ADD entrypoint-config.yml .

View File

@ -181,8 +181,12 @@ pre_conf_commands:
post_conf_commands: post_conf_commands:
- echo "something else" > to_this_another_file - echo "something else" > to_this_another_file
# commands to run in parallele with the main command
post_run_commands: post_run_commands:
- echo run commands after started service - echo do something in parallele with the main command
# run post_run_commands in parallele or sequentially (default is sequential)
run_post_commands_in_parallele: true # default false
# Reload service when configuration change by sending a signal to process # Reload service when configuration change by sending a signal to process
reload: reload:

View File

@ -93,8 +93,12 @@ This is an example of ``entrypoint-config.yml`` file.
post_conf_commands: post_conf_commands:
- echo "something else" > to_this_another_file - echo "something else" > to_this_another_file
# commands to run in parallele with the main command
post_run_commands: post_run_commands:
- echo run commands after started service - echo do something in parallele with the main command
# run post_run_commands in parallele or sequentially (default is sequential)
run_post_commands_in_parallele: true # default false
# Reload service when configuration change by sending a signal to process # Reload service when configuration change by sending a signal to process
reload: reload:
@ -281,6 +285,14 @@ List of shell commands to run after service is started
- sleep 5 - sleep 5
- echo "something else" > to_this_another_file - echo "something else" > to_this_another_file
run_post_commands_in_parallele
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
run post_run_commands in paralle or sequentially (default is sequential)
.. code:: yaml
run_post_commands_in_parallele: true
reload reload
^^^^^^ ^^^^^^

View File

@ -76,6 +76,13 @@ pre_conf_commands:
post_conf_commands: post_conf_commands:
- echo something even useless - echo something even useless
# commands to run in parallele with the main command
post_run_commands:
- echo do something in parallele with the main command is ran
# run porst_run_commands in parallele or sequentially (default is sequential)
run_post_commands_in_parallele: true # default false
# Reload service when configuration change by sending a signal to process # Reload service when configuration change by sending a signal to process
reload: reload:
signal: SIGHUP # Optional, signal to send, default is SIGHUP signal: SIGHUP # Optional, signal to send, default is SIGHUP
@ -84,7 +91,7 @@ reload:
files: # Optional, list of files to watch files: # Optional, list of files to watch
- /etc/conf/to/watch - /etc/conf/to/watch
# can also be enabled with a boolean: # can also be enabled with a boolean:
reload: true # reload: true
# Cleanup environment from variables created by linked containers # Cleanup environment from variables created by linked containers
# before running command (True by default) # before running command (True by default)

View File

@ -335,8 +335,8 @@ class Config(ConfigMeta):
@property @property
def debug(self): def debug(self):
"""Enable debug logs.""" """Enable debug logs."""
if envtobool('ENTRYPOINT_DEBUG', False): if 'ENTRYPOINT_DEBUG' in os.environ:
return True return envtobool('ENTRYPOINT_DEBUG', False)
if 'debug' in self._config: if 'debug' in self._config:
return bool(self._get_by_command(item='debug', return bool(self._get_by_command(item='debug',
value_types=[bool])) value_types=[bool]))
@ -347,8 +347,8 @@ class Config(ConfigMeta):
"""Disable logging""" """Disable logging"""
if self.debug: if self.debug:
return False return False
if envtobool('ENTRYPOINT_QUIET', False): if 'ENTRYPOINT_QUIET' in os.environ:
return True return envtobool('ENTRYPOINT_QUIET', False)
return bool(self._config.get('quiet', False)) return bool(self._config.get('quiet', False))
@property @property
@ -364,6 +364,22 @@ class Config(ConfigMeta):
@property @property
def raw_output(self): def raw_output(self):
"""Check if command output should be displayed using logging or not""" """Check if command output should be displayed using logging or not"""
if envtobool('ENTRYPOINT_RAW', False): if 'ENTRYPOINT_RAW' in os.environ:
return True return envtobool('ENTRYPOINT_RAW', False)
return bool(self._config.get('raw_output', False)) if 'raw_output' in self._config:
return bool(self._get_by_command(item='raw_output',
value_types=[bool]))
return False
@property
def run_post_commands_in_parallele(self):
"""Run all post post run commands in parallele using process"""
if 'ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE' in os.environ:
return envtobool('ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE',
False)
if 'run_post_commands_in_parallele' in self._config:
return bool(self._get_by_command(
item='run_post_commands_in_parallele',
value_types=[bool]
))
return False

View File

@ -2,7 +2,6 @@
from multiprocessing import Process from multiprocessing import Process
from subprocess import PIPE from subprocess import PIPE
from subprocess import Popen from subprocess import Popen
from sys import stdout as stdoutput
from .logs import Logs from .logs import Logs
@ -14,37 +13,52 @@ class Runner(object):
self.log = Logs().log self.log = Logs().log
self.cmds = cmds self.cmds = cmds
self.raw_output = config.raw_output self.raw_output = config.raw_output
self.parallele_run = config.run_post_commands_in_parallele
self.stdout_cb = self.log.info if not self.raw_output else print
self.stderr_cb = self.log.warning if not self.raw_output else print
def stdout(self, output):
for line in output:
self.stdout_cb(line.rstrip())
def stderr(self, output):
for line in output:
self.stderr_cb(line.rstrip())
def stream_output(self, proc):
display_stdout = Process(target=self.stdout, args=(proc.stdout,))
display_stdout.start()
display_stderr = Process(target=self.stderr, args=(proc.stderr,))
display_stderr.start()
def run_cmd(self, cmd, stdout=False): def run_cmd(self, cmd, stdout=False):
self.log.debug('run command: {}'.format(cmd)) self.log.debug('run command: {}'.format(cmd))
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) with Popen(cmd,
out, err = proc.communicate() shell=True,
stdout=PIPE,
def dispout(output, cb): stderr=PIPE,
enc = stdoutput.encoding or 'UTF-8' universal_newlines=True) as proc:
output = output.decode(enc).split('\n')
lenght = len(output)
for c, line in enumerate(output):
if c + 1 == lenght and not len(line):
# Do not display last empty line
break
cb(line)
if err:
display_cb = self.log.warning if not self.raw_output else print
dispout(err, display_cb)
if out:
display_cb = self.log.info if not self.raw_output else print
if stdout: if stdout:
return out.decode() output, _ = proc.communicate()
dispout(out, display_cb) else:
self.stream_output(proc)
if proc.returncode: if proc.returncode:
raise Exception('Command exit code: {}'.format(proc.returncode)) raise Exception('Command exit code: {}'.format(proc.returncode))
if stdout and output:
return output
def run(self): def run(self):
for cmd in self.cmds: for cmd in self.cmds:
self.run_cmd(cmd) self.run_cmd(cmd)
def run_in_process(self): def run_in_process(self):
if self.parallele_run:
return self.run_in_parallele()
self.proc = Process(target=self.run) self.proc = Process(target=self.run)
self.proc.start() self.proc.start()
def run_in_parallele(self):
for cmd in self.cmds:
self.proc = Process(target=self.run_cmd,
args=[cmd])
self.proc.start()

View File

@ -1,7 +1,7 @@
[tool] [tool]
[tool.poetry] [tool.poetry]
name = "pyentrypoint" name = "pyentrypoint"
version = "0.7.2" version = "0.7.3"
description = "pyentrypoint manages entrypoints in Docker containers." description = "pyentrypoint manages entrypoints in Docker containers."
license = "WTFPL" license = "WTFPL"
classifiers = ["Programming Language :: Python", "Development Status :: 1 - Planning", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Topic :: System :: Installation/Setup"] classifiers = ["Programming Language :: Python", "Development Status :: 1 - Planning", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Topic :: System :: Installation/Setup"]
@ -25,7 +25,7 @@ pytest-mock = "^3.1.0"
pre-commit = "^2.3.0" pre-commit = "^2.3.0"
[build-system] [build-system]
requires = ["poetry>=0.12"] requires = ["poetry>=1.0.8"]
build-backend = "poetry.masonry.api" build-backend = "poetry.masonry.api"
[tool.poetry.scripts] [tool.poetry.scripts]

View File

@ -1,5 +1,6 @@
post_run_commands: post_run_commands:
- sleep 2 - date '+%s' > /tmp/timestamp1
- echo 'OK' > /tmp/runner_test - sleep 4
- date '+%s' > /tmp/timestamp2
command: sleep command: sleep

View File

@ -0,0 +1,8 @@
post_run_commands:
- date '+%s' > /tmp/timestamp1
- sleep 4
- date '+%s' > /tmp/timestamp2
command: sleep
run_post_commands_in_parallele: true

View File

@ -108,7 +108,7 @@ def test_containers():
def test_templates(): def test_templates():
test_confs = ['configs/base.yml'] test_confs = ['configs/base.yml']
for test_conf in test_confs: for test_conf in test_confs:
entry = Entrypoint(conf='configs/base.yml') entry = Entrypoint(conf=test_conf)
conf = entry.config conf = entry.config

View File

@ -1,25 +1,14 @@
"Tests for reloader" "Tests for reloader"
import os
import subprocess
try: from signal import SIGHUP
# Python2 from time import sleep
import mock
except ImportError:
# Python3
from unittest import mock from unittest import mock
import os from commons import clean_env
from pyentrypoint import Entrypoint from pyentrypoint import Entrypoint
import subprocess
from signal import SIGHUP
from time import sleep
from commons import clean_env
def teardown_function(function): def teardown_function(function):
clean_env() clean_env()

View File

@ -5,18 +5,39 @@ from multiprocessing import Process
from pyentrypoint import Entrypoint from pyentrypoint import Entrypoint
def compare_timestamps():
with open('/tmp/timestamp1', 'r') as f:
first = int(f.readline())
with open('/tmp/timestamp2', 'r') as f:
second = int(f.readline())
return second - first
def test_runner(): def test_runner():
run = [ run = [
(Process(target=Entrypoint( (Process(target=Entrypoint(
conf='configs/runner.yml', conf='configs/runner.yml',
args=['sleep', '5']).launch), args=['sleep', '5']).launch), 0, 0),
'/tmp/runner_test', 0, 0),
] ]
for proc, test, uid, gid in run: for proc, uid, gid in run:
proc.start() proc.start()
proc.join() proc.join()
with open(test, 'r') as f: assert compare_timestamps() > 3
assert f.readline().startswith('OK') assert os.stat('/tmp/timestamp1').st_uid == uid
assert os.stat(test).st_uid == uid assert os.stat('/tmp/timestamp1').st_gid == gid
assert os.stat(test).st_gid == gid
def test_runner_parallele():
run = [
(Process(target=Entrypoint(
conf='configs/runner_parallele.yml',
args=['sleep', '5']).launch), 0, 0),
]
for proc, uid, gid in run:
proc.start()
proc.join()
assert compare_timestamps() < 1
assert os.stat('/tmp/timestamp1').st_uid == uid
assert os.stat('/tmp/timestamp1').st_gid == gid