From cf447dc3a4b3e4e955e9d845600f8b11904c5043 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 23 May 2020 00:39:04 -0300 Subject: [PATCH] Build (and upload) debs by merging into the deb branches Similar to the static files, the debs also end up under builds.lokinet.dev/deb. --- .drone.jsonnet | 56 +++++++++++++++++++++++++++++++-- contrib/ci/drone-debs-upload.sh | 50 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100755 contrib/ci/drone-debs-upload.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index 8be9f0e38..77014c4ce 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -41,9 +41,56 @@ local debian_pipeline(name, image, './test/catchAll --use-colour yes', ] + extra_cmds, } + ], +}; + +// Builds a snapshot .deb on a debian-like system by merging into the debian/* or ubuntu/* branch +local deb_builder(image, distro, distro_branch, arch='amd64', imaginary_repo=false) = { + kind: 'pipeline', + type: 'docker', + name: 'DEB (' + distro + (if arch == 'amd64' then '' else '/' + arch) + ')', + platform: { arch: arch }, + environment: { distro_branch: distro_branch, distro: distro }, + steps: [ + { + name: 'build', + image: image, + failure: 'ignore', + environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, + commands: [ + 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', + 'apt-get update', + 'apt-get install -y eatmydata', + 'eatmydata apt-get install -y git devscripts equivs ccache git-buildpackage python3-dev' + (if imaginary_repo then ' gpg' else'') + ] + (if imaginary_repo then [ // Some distros need the imaginary.stream repo for backported sodium, etc. + 'echo deb https://deb.imaginary.stream $${distro} main >/etc/apt/sources.list.d/imaginary.stream.list', + 'curl -s https://deb.imaginary.stream/public.gpg | apt-key add -', + 'eatmydata apt-get update' + ] else []) + [ + ||| + # Look for the debian branch in this repo first, try upstream if that fails. + if ! git checkout $${distro_branch}; then + git remote add --fetch upstream https://github.com/loki-project/loki-network.git && + git checkout $${distro_branch} + fi + |||, + 'git merge ${DRONE_COMMIT}', + 'eatmydata git submodule update --init --recursive', + 'export DEBEMAIL="${DRONE_COMMIT_AUTHOR_EMAIL}" DEBFULLNAME="${DRONE_COMMIT_AUTHOR_NAME}"', + 'gbp dch -S -s "HEAD^" --spawn-editor=never -U low', + 'eatmydata mk-build-deps --install --remove --tool "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y"', + 'export DEB_BUILD_OPTIONS="parallel=$$(nproc)"', + 'grep -q lib debian/lokinet-bin.install || echo "/usr/lib/lib*.so*" >>debian/lokinet-bin.install', + 'debuild -e CCACHE_DIR -b', + 'pwd', + 'find ./contrib/ci', + './contrib/ci/drone-debs-upload.sh ' + distro, + ] + } ] }; + [ { name: 'lint check', @@ -65,11 +112,8 @@ local debian_pipeline(name, image, cmake_extra='-DCMAKE_C_COMPILER=clang-10 -DCMAKE_CXX_COMPILER=clang++-10 ', lto=true), debian_pipeline("Debian sid/gcc-10 (amd64)", "debian:sid", deps='g++-10 '+default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10'), - debian_pipeline("Debian buster (amd64)", "debian:buster", cmake_extra='-DDOWNLOAD_SODIUM=ON'), debian_pipeline("Debian buster (i386)", "i386/debian:buster", cmake_extra='-DDOWNLOAD_SODIUM=ON'), debian_pipeline("Ubuntu focal (amd64)", "ubuntu:focal"), - debian_pipeline("Ubuntu bionic (amd64)", "ubuntu:bionic", deps='g++-8 ' + default_deps_base, - cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DDOWNLOAD_SODIUM=ON'), // ARM builds (ARM64 and armhf) debian_pipeline("Ubuntu bionic (ARM64)", "ubuntu:bionic", arch="arm64", deps='g++-8 ' + default_deps_base, @@ -86,6 +130,12 @@ local debian_pipeline(name, image, '../contrib/ci/drone-static-upload.sh' ]), + // Deb builds: + deb_builder("debian:sid", "sid", "debian/sid"), + deb_builder("debian:buster", "buster", "debian/buster", imaginary_repo=true), + deb_builder("ubuntu:focal", "focal", "ubuntu/focal"), + deb_builder("debian:sid", "sid", "debian/sid", arch='arm64'), + // Macos build { kind: 'pipeline', diff --git a/contrib/ci/drone-debs-upload.sh b/contrib/ci/drone-debs-upload.sh new file mode 100755 index 000000000..57495e808 --- /dev/null +++ b/contrib/ci/drone-debs-upload.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Script used with Drone CI to upload debs from the deb building pipelines (because specifying all +# this in .drone.jsonnet is too painful). This is expected to run from the base project dir after +# having build with debuild (which will leave the debs in ..). + +set -o errexit + +distro="$1" + +if [ -z "$distro" ]; then + echo "Bad usage: need distro name as first argument" + exit 1 +fi + +if [ -z "$SSH_KEY" ]; then + echo -e "\n\n\n\e[31;1mUnable to upload debs: SSH_KEY not set\e[0m" + # Just warn but don't fail, so that this doesn't trigger a build failure for untrusted builds + exit 0 +fi + +echo "$SSH_KEY" >~/ssh_key + +set -o xtrace # Don't start tracing until *after* we write the ssh key + +chmod 600 ~/ssh_key + +upload_to="builds.lokinet.dev/debs/${DRONE_REPO// /_}@${DRONE_BRANCH// /_}/$(date --date=@$DRONE_BUILD_CREATED +%Y%m%dT%H%M%SZ)-${DRONE_COMMIT:0:9}/$distro/$DRONE_STAGE_ARCH" + +# sftp doesn't have any equivalent to mkdir -p, so we have to split the above up into a chain of +# -mkdir a/, -mkdir a/b/, -mkdir a/b/c/, ... commands. The leading `-` allows the command to fail +# without error. +upload_dirs=(${upload_to//\// }) +mkdirs= +dir_tmp="" +for p in "${upload_dirs[@]}"; do + dir_tmp="$dir_tmp$p/" + mkdirs="$mkdirs +-mkdir $dir_tmp" +done + +sftp -i ~/ssh_key -b - -o StrictHostKeyChecking=off drone@builds.lokinet.dev <