From 18026f1aa37f139a166ef912b0fdb31f793e320c Mon Sep 17 00:00:00 2001 From: Christophe Mehay Date: Sat, 1 Oct 2016 03:29:04 +0200 Subject: [PATCH] Add reloader --- Dockerfile.py2 | 2 +- Dockerfile.py3 | 2 +- README.md | 16 +++- docs/config.rst | 41 +++++++++- docs/index.rst | 1 + entrypoint-config.yml | 19 ++++- pyentrypoint/command.py | 6 ++ pyentrypoint/config.py | 39 +++++++++- pyentrypoint/entrypoint.py | 2 + pyentrypoint/links.py | 4 +- pyentrypoint/logs.py | 4 +- pyentrypoint/reloader.py | 91 ++++++++++++++++++++++ setup.py | 5 +- tests/configs/reloader/reloader.yml | 4 + tests/configs/reloader/reloader_config.yml | 10 +++ tests/reloader_test.py | 44 +++++++++++ 16 files changed, 274 insertions(+), 16 deletions(-) create mode 100644 pyentrypoint/reloader.py create mode 100644 tests/configs/reloader/reloader.yml create mode 100644 tests/configs/reloader/reloader_config.yml create mode 100644 tests/reloader_test.py diff --git a/Dockerfile.py2 b/Dockerfile.py2 index 17b397d..d0e5529 100644 --- a/Dockerfile.py2 +++ b/Dockerfile.py2 @@ -2,7 +2,7 @@ FROM python:2 -RUN pip install pytest six pyyaml jinja2 colorlog +RUN pip install pytest six pyyaml jinja2 colorlog watchdog pytest-mock ENV PYTHONPATH /opt/ diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 5ea86c2..62eec5d 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -2,7 +2,7 @@ FROM python:3 -RUN pip3 install pytest six pyyaml jinja2 colorlog +RUN pip3 install pytest six pyyaml jinja2 colorlog watchdog pytest-mock ENV PYTHONPATH /opt/ diff --git a/README.md b/README.md index 141adf0..539e76b 100644 --- a/README.md +++ b/README.md @@ -117,12 +117,26 @@ pre_conf_commands: post_conf_commands: - echo "something else" > to_this_another_file +# Reload service when configuration change by sending a signal to process +reload: + signal: SIGHUP # Optional, signal to send, default is SIGHUP + pid: 1 # Optional, pid to send signal, default is 1 + watch_config_files: true # Optional, watch defined config files, default True + files: # Optional, list of files to watch + - /etc/conf/to/watch +# can also be enabled like this: +reload: true + + # Cleanup environment from variables created by linked containers # before running command (True by default) -clean_env: True +clean_env: true # Enable debug to debug debug: true + +# Do not output anything except error +quiet: false ``` ### Config templates diff --git a/docs/config.rst b/docs/config.rst index 83878d6..93fdb15 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -35,13 +35,13 @@ This is an example of ``entrypoint-config.yml`` file. - .ssh/config.tpl # Will apply to ".ssh/config" - /tmp/id_rsa: .ssh/id_rsa # Will apply "/tmp/id_rsa" template to ".ssh/id_rsa" - # These environment variables will be wiped before # exec command to keep them secret # CAUTION: if the container is linked to another one, # theses variables will passed to it anyway secret_env: - SSHKEY + - '*' # Support globbing, all environment will be wiped # Links are handled here # Port, name, protocol or env variable can be used to identify the links @@ -51,7 +51,7 @@ This is an example of ``entrypoint-config.yml`` file. port: 22 name: 'ssh*' protocol: tcp - # env can be list, dictionary or string + # env can be list, dict or string env: FOO: bar # Single doesn't allow multiple links for this ID @@ -69,13 +69,26 @@ This is an example of ``entrypoint-config.yml`` file. post_conf_commands: - echo "something else" > to_this_another_file + # Reload service when configuration change by sending a signal to process + reload: + signal: SIGHUP # Optional, signal to send, default is SIGHUP + watch_config_files: true # Optional, watch defined config files, default True + files: # Optional, list of files to watch + - /etc/conf/to/watch + # can also be enabled with a boolean: + reload: true + # Cleanup environment from variables created by linked containers # before running command (True by default) - clean_env: True + clean_env: true # Enable debug to debug debug: true + # Do not output anything except error + quiet: false + + yaml references ~~~~~~~~~~~~~~~ @@ -209,6 +222,23 @@ List of shell commands to run after applying configuration post_conf_commands: - echo "something else" > to_this_another_file +reload +^^^^^^ + +Send SIGHUP to PID 1 to reload service when configuration change + +Accept boolean or dictionary + +.. code:: yaml + + reload: + signal: SIGHUP # Optional, signal to send, default is SIGHUP + watch_config_files: true # Optional, watch defined config files, default True + files: # Optional, list of files to watch + - /etc/conf/to/watch + # can also be enabled with a boolean: + reload: true + clean_env ^^^^^^^^^ @@ -219,3 +249,8 @@ debug ^^^^^ Print some debug. + +quiet +^^^^^ + +Do not output anything except error diff --git a/docs/index.rst b/docs/index.rst index 45b91d1..b6130e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ This tool avoids writing shell scripts to: - Auto configure container using `jinja2` templates - Run commands before starting service - Clean environment before running service +- Reload service when configuration has changed - Increase security by setuid/setgid service Contents: diff --git a/entrypoint-config.yml b/entrypoint-config.yml index dfc8af9..df9d13e 100644 --- a/entrypoint-config.yml +++ b/entrypoint-config.yml @@ -24,8 +24,8 @@ group: 1000 # and should be jinja templated. config_files: - /etc/gitconfig - - .ssh/config - - .ssh/id_rsa + - .ssh/config.tpl # Will apply to ".ssh/config" + - /tmp/id_rsa: .ssh/id_rsa # Will apply "/tmp/id_rsa" template to ".ssh/id_rsa" # These environment variables will be wiped before # exec command to keep them secret @@ -59,9 +59,22 @@ pre_conf_commands: post_conf_commands: - echo something even useless +# Reload service when configuration change by sending a signal to process +reload: + signal: SIGHUP # Optional, signal to send, default is SIGHUP + pid: 1 # Optional, pid to send signal, default is 1 + watch_config_files: true # Optional, watch defined config files, default True + files: # Optional, list of files to watch + - /etc/conf/to/watch +# can also be enabled with a boolean: +reload: true + # Cleanup environment from variables created by linked containers # before running command (True by default) -clean_env: True +clean_env: true # Enable debug to debug debug: true + +# Do not output anything except error +quiet: false diff --git a/pyentrypoint/command.py b/pyentrypoint/command.py index f3ba0c6..146cdd6 100644 --- a/pyentrypoint/command.py +++ b/pyentrypoint/command.py @@ -81,6 +81,11 @@ class Command(object): )) os.execvpe(self.args[0], self.args, self.env) + def _run_reloader(self): + if self.config.reload: + self.log.debug('Launching reloader process') + self.config.reload.run_in_process() + def run(self): if not self.is_handled: self._exec() @@ -99,4 +104,5 @@ class Command(object): if not self.args or \ [p for p in subcom if fnmatch(self.args[0], p)]: self.args.insert(0, self.command) + self._run_reloader() self._exec() diff --git a/pyentrypoint/config.py b/pyentrypoint/config.py index a01a12f..5356957 100644 --- a/pyentrypoint/config.py +++ b/pyentrypoint/config.py @@ -18,6 +18,7 @@ from .constants import ENTRYPOINT_FILE from .docker_links import DockerLinks from .links import Links from .logs import Logs +from .reloader import Reloader __all__ = ['Config'] @@ -44,9 +45,25 @@ class ConfigMeta(object): template = list(item.keys())[0] outfile = item[template] yield (template, outfile) - if isinstance(config_files, dict): + else: raise Exception("config_files setup missformated.") + def get_reloader(self): + """Setup and get reloader""" + config_files = [file[1] for file in self.get_templates()] + reload = self._config['reload'] + if isinstance(reload, bool): + return Reloader(files=config_files) + if isinstance(reload, dict): + signal = reload.get('signal', 'SIGHUP') + watch_config_files = bool(reload.get('watch_config_files')) + files = reload.get('files', []) + if not isinstance(files, list): + raise Exception('Reload files is not a list') + if watch_config_files: + files.extend(config_files) + return Reloader(files=files, sig=signal) + def _check_config(self): for key in self._config: if not hasattr(Config, key) or key == '__init__': @@ -71,6 +88,7 @@ class Config(ConfigMeta): self._args = args self._links = None self._command = None + self._reload = None self._config_file = conf if not os.path.isfile(self._config_file): self.log.critical('Entrypoint config file does not provided') @@ -155,6 +173,16 @@ class Config(ConfigMeta): """Return list of postconf commands""" return self._return_item_lst('post_conf_commands') + @property + def reload(self): + """Return Reloader object if reload is set""" + if self._reload: + return self._reload + if not self._config.get('reload'): + return None + self._reload = self.get_reloader() + return self._reload + @property def clean_env(self): """Clean env from linked containers before running command""" @@ -170,3 +198,12 @@ class Config(ConfigMeta): if 'debug' in self._config: return bool(self._config['debug']) return False + + @property + def quiet(self): + """Disable logging""" + if self.debug: + return False + if 'ENTRYPOINT_QUIET' in os.environ: + return True + return bool(self._config.get('quiet', False)) diff --git a/pyentrypoint/entrypoint.py b/pyentrypoint/entrypoint.py index 377f953..226e412 100644 --- a/pyentrypoint/entrypoint.py +++ b/pyentrypoint/entrypoint.py @@ -44,6 +44,8 @@ class Entrypoint(object): else: if self.config.debug: Logs.set_debug() + if self.config.quiet: + Logs.set_critical() self.args = args @property diff --git a/pyentrypoint/links.py b/pyentrypoint/links.py index 4fbffb9..7fcdd5f 100644 --- a/pyentrypoint/links.py +++ b/pyentrypoint/links.py @@ -56,7 +56,7 @@ class Links(object): _def_options = {'single': False, 'required': True} - def __init__(self, config=None, links=None): + def __init__(self, config={}, links=None): if not links or len(links.links()) is 0: pass @@ -72,7 +72,7 @@ class Links(object): self._conf = config def _get_link(self, name): - config = self._conf[name] + config = self._conf.get(name, {}) links = self._links options = dict(self._def_options) for key, val in config.items(): diff --git a/pyentrypoint/logs.py b/pyentrypoint/logs.py index f460f4c..20ce84a 100644 --- a/pyentrypoint/logs.py +++ b/pyentrypoint/logs.py @@ -82,12 +82,12 @@ class Logs(object): @classmethod def set_warning(cls): - """Set log level to info""" + """Set log level to warning""" cls.lvl = logging.WARNING cls.log.setLevel(cls.lvl) @classmethod def set_critical(cls): - """Set log level to info""" + """Set log level to critical""" cls.lvl = logging.CRITICAL cls.log.setLevel(cls.lvl) diff --git a/pyentrypoint/reloader.py b/pyentrypoint/reloader.py new file mode 100644 index 0000000..a64d34c --- /dev/null +++ b/pyentrypoint/reloader.py @@ -0,0 +1,91 @@ +""" + Send signal to pid 1 to reload service +""" +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import signal +from multiprocessing import Process + +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer + +from .logs import Logs + + +class Reload(FileSystemEventHandler): + + """Reload object""" + + def __init__(self, sig, files, pid=1): + self.signal = sig + self.files = files + self.pid = pid + self.log = Logs().log + + def on_any_event(self, event): + if event.src_path in self.files: + self.log.info( + 'File {file} has changed, send sig {sig} to pid {pid}'.format( + file=event.src_path, + sig=self.signal, + pid=self.pid, + ) + ) + os.kill(self.pid, self.signal) + + +class Reloader(object): + + """Reload service when files change""" + + def __init__(self, sig='SIGHUP', files=[]): + if not files: + raise Exception('No file to watch for reload') + + self.proc = None + self._files = files + sig_attr = getattr(signal, sig) + try: + assert int(sig_attr) + except: + raise Exception('Wrong signal provided for reload') + self.observer = Observer() + rel = Reload(sig=sig_attr, files=self.files) + for dir in self.dirs: + self.observer.schedule(rel, dir, recursive=False) + + def _get_files(self): + """Return iterator of tuples (path, file)""" + for f in self._files: + if os.path.isdir(f): + yield (f, f) + yield (os.path.dirname(f), f) + + @property + def files(self): + """Return list of watched files""" + return list(set([files[1] for files in self._get_files()])) + + @property + def dirs(self): + """Return list of watched directories""" + return list(set([files[0] for files in self._get_files()])) + + def run(self, ret=False): + while True: + self.observer.start() + if ret: + return self.observer + self.observer.join() + + def run_in_process(self): + self.proc = Process(target=self.run) + self.proc.start() + + def stop(self): + if self.proc: + self.proc.stop() + else: + self.observer.stop() diff --git a/setup.py b/setup.py index f4b29f6..163b288 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup # Thanks Sam and Max -__version__ = '0.3.8' +__version__ = '0.4.0' if __name__ == '__main__': setup( @@ -28,7 +28,8 @@ if __name__ == '__main__': 'PyYAML>=3.11', 'colorlog>=2.6.1', 'argparse>=1.4.0', - 'six>=1.10.0'], + 'six>=1.10.0', + 'watchdog>=0.8.3'], include_package_data=True, diff --git a/tests/configs/reloader/reloader.yml b/tests/configs/reloader/reloader.yml new file mode 100644 index 0000000..ebe18c0 --- /dev/null +++ b/tests/configs/reloader/reloader.yml @@ -0,0 +1,4 @@ +config_files: + - /tmp/test_template2.yml.tpl: /tmp/reload + +reload: true diff --git a/tests/configs/reloader/reloader_config.yml b/tests/configs/reloader/reloader_config.yml new file mode 100644 index 0000000..348dffb --- /dev/null +++ b/tests/configs/reloader/reloader_config.yml @@ -0,0 +1,10 @@ +config_files: + - /tmp/test_template2.yml.tpl: /tmp/reload + +reload: + watch_config_files: false + files: + - /tmp/reload_custom + +pre_conf_commands: + - touch /tmp/reload_custom diff --git a/tests/reloader_test.py b/tests/reloader_test.py new file mode 100644 index 0000000..c89068d --- /dev/null +++ b/tests/reloader_test.py @@ -0,0 +1,44 @@ +"Tests for reloader" +from __future__ import absolute_import +from __future__ import unicode_literals + + +try: + # Python2 + import mock +except ImportError: + # Python3 + from unittest import mock + +from pyentrypoint import Entrypoint + +import subprocess + +from signal import SIGHUP + +from time import sleep + + +def test_reloader(): + + with mock.patch('os.kill') as os_kill: + entry = Entrypoint(conf='configs/reloader/reloader.yml') + entry.apply_conf() + entry.config.reload.run(ret=True) + subprocess.check_call(['touch', '/tmp/reload']) + sleep(1) + entry.config.reload.stop() + os_kill.assert_called_once_with(1, SIGHUP) + + +def test_reloader_custom(): + + with mock.patch('os.kill') as os_kill: + entry = Entrypoint(conf='configs/reloader/reloader_config.yml') + entry.apply_conf() + entry.run_pre_conf_cmds() + entry.config.reload.run(ret=True) + subprocess.check_call(['touch', '/tmp/reload', '/tmp/reload_custom']) + sleep(1) + entry.config.reload.stop() + os_kill.assert_called_once_with(1, SIGHUP)