#!/usr/bin/env python3 # # python wsgi application for managing many lokinet instances # __doc__ = """lokinet bootserv wsgi app also handles webhooks for CI run me with via gunicorn pylokinet.bootserv:app """ import os from pylokinet import rc import json import random import time from datetime import datetime from email.utils import parsedate, format_datetime from dateutil.parser import parse as date_parse import requests root = './lokinet' def _compare_dates(left, right): """ return true if left timestamp is bigger than right """ return date_parse(left) > date_parse(right) class TokenHolder: _dir = root _token = None def __init__(self, f="token"): if not os.path.exists(self._dir): os.mkdir(self._dir, 0o700) f = os.path.join(self._dir, f) if os.path.exists(f): with open(f) as fd: self._token = fd.read().replace("\n", "") def verify(self, token): """ return true if token matches """ if self._token is None: return False return self._token == token class BinHolder: """ serves a binary file in a dir """ _dir = os.path.join(root, 'bin') def __init__(self, f): if not os.path.exists(self._dir): os.mkdir(self._dir, 0o700) self._fpath = os.path.join(self._dir, f) def put(self, r): """ put a new file into the place that is held """ with open(self._fpath, "wb") as f: for chunk in r.iter_content(chunk_size=1024): f.write(chunk) def is_new(self, date): """ return true if last modified timestamp is fresher than current """ t = date_parse('{}'.format(date)) if not t: return False if os.path.exists(self._fpath): st = os.stat(self._fpath) return st.st_mtime < t.timestamp() return True def serve(self, last_modified, respond): """ serve file with caching """ t = parsedate(last_modified) if t: t = time.mktime(t) if t is None: t = 0 if not os.path.exists(self._fpath): respond("404 Not Found", []) return [] st = os.stat(self._fpath) if st.st_mtime < t: respond("304 Not Modified", [("Last-Modified", format_datetime(st.st_mtime)) ]) return [] with open(self._fpath, "rb") as f: data = f.read() respond("200 OK", [("Content-Type", "application/octect-stream"), ("Last-Modified", format_datetime(datetime.fromtimestamp(int(st.st_mtime)))),("Content-Length", "{}".format(st.st_size))]) return [data] class RCHolder: _dir = os.path.join(root, 'nodedb') _rc_files = list() def __init__(self): if os.path.exists(self._dir): for root, _, files in os.walk(self._dir): for f in files: self._add_rc(os.path.join(root, f)) else: os.mkdir(self._dir, 0o700) def prune(self): """ remove invalid entries """ delfiles = [] for p in self._rc_files: with open(p, 'rb') as f: if not rc.validate(f.read()): delfiles.append(p) for f in delfiles: os.remove(f) def validate_then_put(self, body): if not rc.validate(body): return False k = rc.get_pubkey(body) print(k) if k is None: return False with open(os.path.join(self._dir, k), "wb") as f: f.write(body) return True def _add_rc(self, fpath): self._rc_files.append(fpath) def serve_random(self): with open(random.choice(self._rc_files), 'rb') as f: return f.read() def empty(self): return len(self._rc_files) == 0 def handle_rc_upload(body, respond): holder = RCHolder() if holder.validate_then_put(body): respond("200 OK", [("Content-Type", "text/plain")]) return ["rc accepted".encode('ascii')] else: respond("400 Bad Request", [("Content-Type", "text/plain")]) return ["bad rc".encode('ascii')] def serve_random_rc(): holder = RCHolder() if holder.empty(): return None else: return holder.serve_random() def response(status, msg, respond): respond(status, [("Content-Type", "text/plain"), ("Content-Length", "{}".format(len(msg)))]) return [msg.encode("utf-8")] def handle_serve_lokinet(modified_since, respond): l = BinHolder('lokinet.zip') return l.serve(modified_since, respond) def fetch_lokinet(j, ref="staging", name="build:linux"): holder = BinHolder("lokinet.zip") if 'builds' not in j: return False selected = None attrs = dict() if 'object_attributes' in j: attrs = j['object_attributes'] if 'ref' not in attrs or attrs["ref"] != ref: return True for build in j['builds']: if 'name' not in build or build['name'] != name: continue if 'status' not in build or build['status'] != 'success': continue if 'finished_at' not in build or build['finished_at'] is None: continue if holder.is_new(build['finished_at']): if selected is None or _compare_dates(build["finished_at"], selected["finished_at"]): selected = build if selected and 'id' in selected: url = 'https://gitlab.com/lokiproject/loki-network/-/jobs/{}/artifacts/download'.format(selected['id']) r = requests.get(url) if r.status_code == 200: holder.put(r) return True #if 'artifacts_file' not in selected: # return False #f = selected["artifacts_file"] #return True def handle_webhook(j, token, event, respond): """ handle CI webhook """ t = TokenHolder() if not t.verify(token): respond("403 Forbidden", []) return [] event = event.lower() if event == 'pipeline hook': if fetch_lokinet(j): respond("200 OK", []) return [] else: respond("500 Internal Server Error", []) return [] else: respond("404 Not Found", []) return [] def app(environ, start_response): request_body_size = int(environ.get("CONTENT_LENGTH", 0)) method = environ.get("REQUEST_METHOD") if method.upper() == "PUT" and request_body_size > 0: rcbody = environ.get("wsgi.input").read(request_body_size) return handle_rc_upload(rcbody, start_response) elif method.upper() == "POST": if environ.get("PATH_INFO") == "/": j = json.loads(environ.get("wsgi.input").read(request_body_size)) token = environ.get("HTTP_X_GITLAB_TOKEN") return handle_webhook(j, token, environ.get("HTTP_X_GITLAB_EVENT"), start_response) else: return response("404 Not Found", 'bad url', start_response) elif method.upper() == "GET": if environ.get("PATH_INFO") == "/bootstrap.signed": resp = serve_random_rc() if resp is not None: start_response('200 OK', [("Content-Type", "application/octet-stream")]) return [resp] else: return response('404 Not Found', 'no RCs', start_response) elif environ.get("PATH_INFO") == "/ping": return response('200 OK', 'pong', start_response) elif environ.get("PATH_INFO") == "/lokinet.zip": return handle_serve_lokinet(environ.get("HTTP_IF_MODIFIED_SINCE"),start_response) elif environ.get("PATH_INFO") == "/": return response("200 OK", "lokinet bootserv", start_response) else: return response('404 Not Found', 'Not found', start_response) else: return response('405 Method Not Allowed', 'method not allowed', start_response) def main(): """ run as cron job """ h = RCHolder() h.prune() if __name__ == '__main__': main()