mirror of
https://github.com/cmehay/pyentrypoint
synced 2024-10-30 15:21:11 +00:00
Add run_post_commands_in_parallele
to run each post commands in separate process and stream output
This commit is contained in:
parent
3908415898
commit
6a457f2398
@ -1,6 +1,5 @@
|
|||||||
*
|
*
|
||||||
!pyentrypoint
|
!pyentrypoint
|
||||||
!setup.py
|
|
||||||
!README.md
|
!README.md
|
||||||
!tests
|
!tests
|
||||||
!poetry.lock
|
!poetry.lock
|
||||||
|
@ -2,12 +2,11 @@ FROM python:3
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,13 +2,12 @@ FROM python:3-alpine
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
apk add gcc && \
|
apk add gcc && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,12 +2,11 @@ FROM python:3.6
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,13 +2,12 @@ FROM python:3.6-alpine
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
apk add gcc && \
|
apk add gcc && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,12 +2,11 @@ FROM python:3.7
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,13 +2,12 @@ FROM python:3.7-alpine
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
apk add gcc && \
|
apk add gcc && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,12 +2,11 @@ FROM python:3.8
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -2,13 +2,12 @@ FROM python:3.8-alpine
|
|||||||
|
|
||||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||||
|
|
||||||
ADD . /tmp/
|
ADD . /usr/local/src/
|
||||||
|
|
||||||
RUN cd /tmp && \
|
RUN cd /usr/local/src/ && \
|
||||||
apk add gcc && \
|
apk add gcc && \
|
||||||
pip install --upgrade pip poetry && \
|
pip install --upgrade pip poetry && \
|
||||||
poetry install --no-dev && \
|
poetry install --no-dev
|
||||||
rm -rf *
|
|
||||||
|
|
||||||
ONBUILD ADD entrypoint-config.yml .
|
ONBUILD ADD entrypoint-config.yml .
|
||||||
|
|
||||||
|
@ -181,8 +181,12 @@ pre_conf_commands:
|
|||||||
post_conf_commands:
|
post_conf_commands:
|
||||||
- echo "something else" > to_this_another_file
|
- echo "something else" > to_this_another_file
|
||||||
|
|
||||||
|
# commands to run in parallele with the main command
|
||||||
post_run_commands:
|
post_run_commands:
|
||||||
- echo run commands after started service
|
- echo do something in parallele with the main command
|
||||||
|
|
||||||
|
# run post_run_commands in parallele or sequentially (default is sequential)
|
||||||
|
run_post_commands_in_parallele: true # default false
|
||||||
|
|
||||||
# Reload service when configuration change by sending a signal to process
|
# Reload service when configuration change by sending a signal to process
|
||||||
reload:
|
reload:
|
||||||
|
@ -93,8 +93,12 @@ This is an example of ``entrypoint-config.yml`` file.
|
|||||||
post_conf_commands:
|
post_conf_commands:
|
||||||
- echo "something else" > to_this_another_file
|
- echo "something else" > to_this_another_file
|
||||||
|
|
||||||
|
# commands to run in parallele with the main command
|
||||||
post_run_commands:
|
post_run_commands:
|
||||||
- echo run commands after started service
|
- echo do something in parallele with the main command
|
||||||
|
|
||||||
|
# run post_run_commands in parallele or sequentially (default is sequential)
|
||||||
|
run_post_commands_in_parallele: true # default false
|
||||||
|
|
||||||
# Reload service when configuration change by sending a signal to process
|
# Reload service when configuration change by sending a signal to process
|
||||||
reload:
|
reload:
|
||||||
@ -281,6 +285,14 @@ List of shell commands to run after service is started
|
|||||||
- sleep 5
|
- sleep 5
|
||||||
- echo "something else" > to_this_another_file
|
- echo "something else" > to_this_another_file
|
||||||
|
|
||||||
|
run_post_commands_in_parallele
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
run post_run_commands in paralle or sequentially (default is sequential)
|
||||||
|
|
||||||
|
.. code:: yaml
|
||||||
|
|
||||||
|
run_post_commands_in_parallele: true
|
||||||
|
|
||||||
reload
|
reload
|
||||||
^^^^^^
|
^^^^^^
|
||||||
|
@ -76,6 +76,13 @@ pre_conf_commands:
|
|||||||
post_conf_commands:
|
post_conf_commands:
|
||||||
- echo something even useless
|
- echo something even useless
|
||||||
|
|
||||||
|
# commands to run in parallele with the main command
|
||||||
|
post_run_commands:
|
||||||
|
- echo do something in parallele with the main command is ran
|
||||||
|
|
||||||
|
# run porst_run_commands in parallele or sequentially (default is sequential)
|
||||||
|
run_post_commands_in_parallele: true # default false
|
||||||
|
|
||||||
# Reload service when configuration change by sending a signal to process
|
# Reload service when configuration change by sending a signal to process
|
||||||
reload:
|
reload:
|
||||||
signal: SIGHUP # Optional, signal to send, default is SIGHUP
|
signal: SIGHUP # Optional, signal to send, default is SIGHUP
|
||||||
@ -84,7 +91,7 @@ reload:
|
|||||||
files: # Optional, list of files to watch
|
files: # Optional, list of files to watch
|
||||||
- /etc/conf/to/watch
|
- /etc/conf/to/watch
|
||||||
# can also be enabled with a boolean:
|
# can also be enabled with a boolean:
|
||||||
reload: true
|
# reload: true
|
||||||
|
|
||||||
# Cleanup environment from variables created by linked containers
|
# Cleanup environment from variables created by linked containers
|
||||||
# before running command (True by default)
|
# before running command (True by default)
|
||||||
|
@ -335,8 +335,8 @@ class Config(ConfigMeta):
|
|||||||
@property
|
@property
|
||||||
def debug(self):
|
def debug(self):
|
||||||
"""Enable debug logs."""
|
"""Enable debug logs."""
|
||||||
if envtobool('ENTRYPOINT_DEBUG', False):
|
if 'ENTRYPOINT_DEBUG' in os.environ:
|
||||||
return True
|
return envtobool('ENTRYPOINT_DEBUG', False)
|
||||||
if 'debug' in self._config:
|
if 'debug' in self._config:
|
||||||
return bool(self._get_by_command(item='debug',
|
return bool(self._get_by_command(item='debug',
|
||||||
value_types=[bool]))
|
value_types=[bool]))
|
||||||
@ -347,8 +347,8 @@ class Config(ConfigMeta):
|
|||||||
"""Disable logging"""
|
"""Disable logging"""
|
||||||
if self.debug:
|
if self.debug:
|
||||||
return False
|
return False
|
||||||
if envtobool('ENTRYPOINT_QUIET', False):
|
if 'ENTRYPOINT_QUIET' in os.environ:
|
||||||
return True
|
return envtobool('ENTRYPOINT_QUIET', False)
|
||||||
return bool(self._config.get('quiet', False))
|
return bool(self._config.get('quiet', False))
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@ -364,6 +364,22 @@ class Config(ConfigMeta):
|
|||||||
@property
|
@property
|
||||||
def raw_output(self):
|
def raw_output(self):
|
||||||
"""Check if command output should be displayed using logging or not"""
|
"""Check if command output should be displayed using logging or not"""
|
||||||
if envtobool('ENTRYPOINT_RAW', False):
|
if 'ENTRYPOINT_RAW' in os.environ:
|
||||||
return True
|
return envtobool('ENTRYPOINT_RAW', False)
|
||||||
return bool(self._config.get('raw_output', False))
|
if 'raw_output' in self._config:
|
||||||
|
return bool(self._get_by_command(item='raw_output',
|
||||||
|
value_types=[bool]))
|
||||||
|
return False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def run_post_commands_in_parallele(self):
|
||||||
|
"""Run all post post run commands in parallele using process"""
|
||||||
|
if 'ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE' in os.environ:
|
||||||
|
return envtobool('ENTRYPOINT_RUN_POST_COMMANDS_IN_PARALLELE',
|
||||||
|
False)
|
||||||
|
if 'run_post_commands_in_parallele' in self._config:
|
||||||
|
return bool(self._get_by_command(
|
||||||
|
item='run_post_commands_in_parallele',
|
||||||
|
value_types=[bool]
|
||||||
|
))
|
||||||
|
return False
|
||||||
|
@ -2,7 +2,6 @@
|
|||||||
from multiprocessing import Process
|
from multiprocessing import Process
|
||||||
from subprocess import PIPE
|
from subprocess import PIPE
|
||||||
from subprocess import Popen
|
from subprocess import Popen
|
||||||
from sys import stdout as stdoutput
|
|
||||||
|
|
||||||
from .logs import Logs
|
from .logs import Logs
|
||||||
|
|
||||||
@ -14,37 +13,52 @@ class Runner(object):
|
|||||||
self.log = Logs().log
|
self.log = Logs().log
|
||||||
self.cmds = cmds
|
self.cmds = cmds
|
||||||
self.raw_output = config.raw_output
|
self.raw_output = config.raw_output
|
||||||
|
self.parallele_run = config.run_post_commands_in_parallele
|
||||||
|
self.stdout_cb = self.log.info if not self.raw_output else print
|
||||||
|
self.stderr_cb = self.log.warning if not self.raw_output else print
|
||||||
|
|
||||||
|
def stdout(self, output):
|
||||||
|
for line in output:
|
||||||
|
self.stdout_cb(line.rstrip())
|
||||||
|
|
||||||
|
def stderr(self, output):
|
||||||
|
for line in output:
|
||||||
|
self.stderr_cb(line.rstrip())
|
||||||
|
|
||||||
|
def stream_output(self, proc):
|
||||||
|
display_stdout = Process(target=self.stdout, args=(proc.stdout,))
|
||||||
|
display_stdout.start()
|
||||||
|
display_stderr = Process(target=self.stderr, args=(proc.stderr,))
|
||||||
|
display_stderr.start()
|
||||||
|
|
||||||
def run_cmd(self, cmd, stdout=False):
|
def run_cmd(self, cmd, stdout=False):
|
||||||
self.log.debug('run command: {}'.format(cmd))
|
self.log.debug('run command: {}'.format(cmd))
|
||||||
proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
|
with Popen(cmd,
|
||||||
out, err = proc.communicate()
|
shell=True,
|
||||||
|
stdout=PIPE,
|
||||||
def dispout(output, cb):
|
stderr=PIPE,
|
||||||
enc = stdoutput.encoding or 'UTF-8'
|
universal_newlines=True) as proc:
|
||||||
output = output.decode(enc).split('\n')
|
|
||||||
lenght = len(output)
|
|
||||||
for c, line in enumerate(output):
|
|
||||||
if c + 1 == lenght and not len(line):
|
|
||||||
# Do not display last empty line
|
|
||||||
break
|
|
||||||
cb(line)
|
|
||||||
|
|
||||||
if err:
|
|
||||||
display_cb = self.log.warning if not self.raw_output else print
|
|
||||||
dispout(err, display_cb)
|
|
||||||
if out:
|
|
||||||
display_cb = self.log.info if not self.raw_output else print
|
|
||||||
if stdout:
|
if stdout:
|
||||||
return out.decode()
|
output, _ = proc.communicate()
|
||||||
dispout(out, display_cb)
|
else:
|
||||||
|
self.stream_output(proc)
|
||||||
if proc.returncode:
|
if proc.returncode:
|
||||||
raise Exception('Command exit code: {}'.format(proc.returncode))
|
raise Exception('Command exit code: {}'.format(proc.returncode))
|
||||||
|
if stdout and output:
|
||||||
|
return output
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
for cmd in self.cmds:
|
for cmd in self.cmds:
|
||||||
self.run_cmd(cmd)
|
self.run_cmd(cmd)
|
||||||
|
|
||||||
def run_in_process(self):
|
def run_in_process(self):
|
||||||
|
if self.parallele_run:
|
||||||
|
return self.run_in_parallele()
|
||||||
self.proc = Process(target=self.run)
|
self.proc = Process(target=self.run)
|
||||||
self.proc.start()
|
self.proc.start()
|
||||||
|
|
||||||
|
def run_in_parallele(self):
|
||||||
|
for cmd in self.cmds:
|
||||||
|
self.proc = Process(target=self.run_cmd,
|
||||||
|
args=[cmd])
|
||||||
|
self.proc.start()
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
[tool]
|
[tool]
|
||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "pyentrypoint"
|
name = "pyentrypoint"
|
||||||
version = "0.7.2"
|
version = "0.7.3"
|
||||||
description = "pyentrypoint manages entrypoints in Docker containers."
|
description = "pyentrypoint manages entrypoints in Docker containers."
|
||||||
license = "WTFPL"
|
license = "WTFPL"
|
||||||
classifiers = ["Programming Language :: Python", "Development Status :: 1 - Planning", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Topic :: System :: Installation/Setup"]
|
classifiers = ["Programming Language :: Python", "Development Status :: 1 - Planning", "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Topic :: System :: Installation/Setup"]
|
||||||
@ -25,7 +25,7 @@ pytest-mock = "^3.1.0"
|
|||||||
pre-commit = "^2.3.0"
|
pre-commit = "^2.3.0"
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
requires = ["poetry>=0.12"]
|
requires = ["poetry>=1.0.8"]
|
||||||
build-backend = "poetry.masonry.api"
|
build-backend = "poetry.masonry.api"
|
||||||
|
|
||||||
[tool.poetry.scripts]
|
[tool.poetry.scripts]
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
post_run_commands:
|
post_run_commands:
|
||||||
- sleep 2
|
- date '+%s' > /tmp/timestamp1
|
||||||
- echo 'OK' > /tmp/runner_test
|
- sleep 4
|
||||||
|
- date '+%s' > /tmp/timestamp2
|
||||||
|
|
||||||
command: sleep
|
command: sleep
|
||||||
|
8
tests/configs/runner_parallele.yml
Normal file
8
tests/configs/runner_parallele.yml
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
post_run_commands:
|
||||||
|
- date '+%s' > /tmp/timestamp1
|
||||||
|
- sleep 4
|
||||||
|
- date '+%s' > /tmp/timestamp2
|
||||||
|
|
||||||
|
command: sleep
|
||||||
|
|
||||||
|
run_post_commands_in_parallele: true
|
@ -108,7 +108,7 @@ def test_containers():
|
|||||||
def test_templates():
|
def test_templates():
|
||||||
test_confs = ['configs/base.yml']
|
test_confs = ['configs/base.yml']
|
||||||
for test_conf in test_confs:
|
for test_conf in test_confs:
|
||||||
entry = Entrypoint(conf='configs/base.yml')
|
entry = Entrypoint(conf=test_conf)
|
||||||
|
|
||||||
conf = entry.config
|
conf = entry.config
|
||||||
|
|
||||||
|
@ -1,25 +1,14 @@
|
|||||||
"Tests for reloader"
|
"Tests for reloader"
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
try:
|
from signal import SIGHUP
|
||||||
# Python2
|
from time import sleep
|
||||||
import mock
|
|
||||||
except ImportError:
|
|
||||||
# Python3
|
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
import os
|
from commons import clean_env
|
||||||
|
|
||||||
from pyentrypoint import Entrypoint
|
from pyentrypoint import Entrypoint
|
||||||
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
from signal import SIGHUP
|
|
||||||
|
|
||||||
from time import sleep
|
|
||||||
|
|
||||||
from commons import clean_env
|
|
||||||
|
|
||||||
|
|
||||||
def teardown_function(function):
|
def teardown_function(function):
|
||||||
clean_env()
|
clean_env()
|
||||||
|
@ -5,18 +5,39 @@ from multiprocessing import Process
|
|||||||
from pyentrypoint import Entrypoint
|
from pyentrypoint import Entrypoint
|
||||||
|
|
||||||
|
|
||||||
|
def compare_timestamps():
|
||||||
|
with open('/tmp/timestamp1', 'r') as f:
|
||||||
|
first = int(f.readline())
|
||||||
|
with open('/tmp/timestamp2', 'r') as f:
|
||||||
|
second = int(f.readline())
|
||||||
|
return second - first
|
||||||
|
|
||||||
|
|
||||||
def test_runner():
|
def test_runner():
|
||||||
run = [
|
run = [
|
||||||
(Process(target=Entrypoint(
|
(Process(target=Entrypoint(
|
||||||
conf='configs/runner.yml',
|
conf='configs/runner.yml',
|
||||||
args=['sleep', '5']).launch),
|
args=['sleep', '5']).launch), 0, 0),
|
||||||
'/tmp/runner_test', 0, 0),
|
|
||||||
]
|
]
|
||||||
|
|
||||||
for proc, test, uid, gid in run:
|
for proc, uid, gid in run:
|
||||||
proc.start()
|
proc.start()
|
||||||
proc.join()
|
proc.join()
|
||||||
with open(test, 'r') as f:
|
assert compare_timestamps() > 3
|
||||||
assert f.readline().startswith('OK')
|
assert os.stat('/tmp/timestamp1').st_uid == uid
|
||||||
assert os.stat(test).st_uid == uid
|
assert os.stat('/tmp/timestamp1').st_gid == gid
|
||||||
assert os.stat(test).st_gid == gid
|
|
||||||
|
|
||||||
|
def test_runner_parallele():
|
||||||
|
run = [
|
||||||
|
(Process(target=Entrypoint(
|
||||||
|
conf='configs/runner_parallele.yml',
|
||||||
|
args=['sleep', '5']).launch), 0, 0),
|
||||||
|
]
|
||||||
|
|
||||||
|
for proc, uid, gid in run:
|
||||||
|
proc.start()
|
||||||
|
proc.join()
|
||||||
|
assert compare_timestamps() < 1
|
||||||
|
assert os.stat('/tmp/timestamp1').st_uid == uid
|
||||||
|
assert os.stat('/tmp/timestamp1').st_gid == gid
|
||||||
|
Loading…
Reference in New Issue
Block a user