CI docker multiarch

Adds multiarch docker images and starts using them.

.drone.jsonnet

@@ -265,8 +265,10 @@ local mac_builder(name,
   debian_pipeline('Debian sid/Debug (amd64)', docker_base + 'debian-sid', build_type='Debug'),
   clang(13),
   full_llvm(13),
-  debian_pipeline('Debian bullseye (i386)', 'i386/debian:bullseye', cmake_extra='-DDOWNLOAD_SODIUM=ON'),
+  debian_pipeline('Debian stable (i386)', docker_base + 'debian-stable:i386'),
-  debian_pipeline('Ubuntu focal (amd64)', docker_base + 'ubuntu-focal'),
+  debian_pipeline('Debian buster (amd64)', docker_base + 'debian-buster', cmake_extra='-DDOWNLOAD_SODIUM=ON'),
+  debian_pipeline('Ubuntu latest (amd64)', docker_base + 'ubuntu-rolling'),
+  debian_pipeline('Ubuntu LTS (amd64)', docker_base + 'ubuntu-lts'),
   debian_pipeline('Ubuntu bionic (amd64)',
                   docker_base + 'ubuntu-bionic',
                   deps=['g++-8'] + default_deps_nocxx,
@@ -274,11 +276,11 @@ local mac_builder(name,
                   loki_repo=true),
 
   // ARM builds (ARM64 and armhf)
-  debian_pipeline('Debian sid (ARM64)', 'debian:sid', arch='arm64', jobs=4),
+  debian_pipeline('Debian sid (ARM64)', docker_base + 'debian-sid', arch='arm64', jobs=4),
-  debian_pipeline('Debian bullseye (armhf)', 'arm32v7/debian:bullseye', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4),
+  debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable:arm32v7', arch='arm64', jobs=4),
   // Static armhf build (gets uploaded)
   debian_pipeline('Static (buster armhf)',
-                  'arm32v7/debian:buster',
+                  docker_base + 'debian-buster:arm32v7',
                   arch='arm64',
                   deps=['g++', 'python3-dev', 'automake', 'libtool'],
                   cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' +
@@ -316,7 +318,7 @@ local mac_builder(name,
 
   // integration tests
   debian_pipeline('Router Hive',
-                  'ubuntu:focal',
+                  docker_base + 'ubuntu-lts',
                   deps=['python3-dev', 'python3-pytest', 'python3-pybind11'] + default_deps,
                   cmake_extra='-DWITH_HIVE=ON'),
 
@@ -325,7 +327,7 @@ local mac_builder(name,
   deb_builder(docker_base + 'debian-bullseye', 'bullseye', 'debian/bullseye'),
   deb_builder(docker_base + 'ubuntu-impish', 'impish', 'ubuntu/impish'),
   deb_builder(docker_base + 'ubuntu-focal', 'focal', 'ubuntu/focal'),
-  deb_builder('debian:sid', 'sid', 'debian/sid', arch='arm64'),
+  deb_builder(docker_base + 'debian-sid', 'sid', 'debian/sid', arch='arm64'),
 
   // Macos builds:
   mac_builder('macOS (Release)'),

contrib/ci/docker/00-debian-bullseye-base.dockerfile

@@ -1,3 +1,4 @@
-FROM debian:bullseye
+ARG ARCH=amd64
+FROM ${ARCH}/debian:bullseye
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

contrib/ci/docker/00-debian-buster-base.dockerfile

@@ -1,3 +1,4 @@
-FROM debian:buster
+ARG ARCH=amd64
+FROM ${ARCH}/debian:buster
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

contrib/ci/docker/00-debian-sid-base.dockerfile

@@ -1,3 +1,4 @@
-FROM debian:sid
+ARG ARCH=amd64
+FROM ${ARCH}/debian:sid
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

contrib/ci/docker/00-debian-stable-base.dockerfile

@@ -1 +0,0 @@
-00-debian-bullseye-base.dockerfile

contrib/ci/docker/00-debian-stable-base.dockerfile

@@ -0,0 +1,4 @@
+ARG ARCH=amd64
+FROM ${ARCH}/debian:stable
+RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
+RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

contrib/ci/docker/00-debian-testing-base.dockerfile

@@ -1,3 +1,4 @@
-FROM debian:testing
+ARG ARCH=amd64
+FROM ${ARCH}/debian:testing
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y'

contrib/ci/docker/10-debian-bullseye.dockerfile

@@ -1,2 +1,3 @@
-FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH}
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-debian-buster.dockerfile

@@ -1,2 +1,3 @@
-FROM registry.oxen.rocks/lokinet-ci-debian-buster-base
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH}
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-debian-sid.dockerfile

@@ -1,2 +1,3 @@
-FROM registry.oxen.rocks/lokinet-ci-debian-sid-base
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH}
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-debian-stable.dockerfile

@@ -1 +0,0 @@
-10-debian-bullseye.dockerfile

contrib/ci/docker/10-debian-stable.dockerfile

@@ -0,0 +1,3 @@
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH}
+RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-ubuntu-bionic.dockerfile

@@ -1,3 +1,4 @@
-FROM ubuntu:bionic
+ARG ARCH=amd64
+FROM ${ARCH}/ubuntu:bionic
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client lsb-release libzmq3-dev libpgm-dev libuv1-dev openssh-client && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include'

contrib/ci/docker/10-ubuntu-focal.dockerfile

@@ -1,3 +1,4 @@
-FROM ubuntu:focal
+ARG ARCH=amd64
+FROM ${ARCH}/ubuntu:focal
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-ubuntu-impish.dockerfile

@@ -0,0 +1,4 @@
+ARG ARCH=amd64
+FROM ${ARCH}/ubuntu:impish
+RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
+RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-ubuntu-lts.dockerfile

@@ -0,0 +1,4 @@
+ARG ARCH=amd64
+FROM ${ARCH}/ubuntu:latest
+RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
+RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/10-ubuntu-rolling.dockerfile

@@ -0,0 +1,4 @@
+ARG ARCH=amd64
+FROM ${ARCH}/ubuntu:rolling
+RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
+RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client'

contrib/ci/docker/50-android.dockerfile

@@ -1,3 +1,4 @@
-FROM registry.oxen.rocks/lokinet-ci-debian-testing-base
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH}
 RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses'

contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile

@@ -0,0 +1 @@
+../00-debian-buster-base.dockerfile

contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile

@@ -0,0 +1 @@
+../00-debian-stable-base.dockerfile

contrib/ci/docker/arm32v7/10-debian-buster.dockerfile

@@ -0,0 +1 @@
+../10-debian-buster.dockerfile

contrib/ci/docker/arm32v7/10-debian-stable.dockerfile

@@ -0,0 +1 @@
+../10-debian-stable.dockerfile

contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile

@@ -0,0 +1 @@
+../00-debian-sid-base.dockerfile

contrib/ci/docker/arm64v8/10-debian-sid.dockerfile

@@ -0,0 +1 @@
+../10-debian-sid.dockerfile

contrib/ci/docker/debian-sid-clang.dockerfile

@@ -1,2 +1,3 @@
-FROM registry.oxen.rocks/lokinet-ci-debian-sid
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-sid/${ARCH}
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y clang-13 lld-13 libc++-13-dev libc++abi-13-dev'

contrib/ci/docker/debian-win32-cross.dockerfile

@@ -1,4 +1,5 @@
-FROM debian:testing
+ARG ARCH=amd64
+FROM ${ARCH}/debian:testing
 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections'
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client'
 RUN /bin/bash -c 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix'

contrib/ci/docker/flutter.dockerfile

@@ -1,2 +1,3 @@
-FROM registry.oxen.rocks/lokinet-ci-android
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-android/${ARCH}
 RUN /bin/bash -c 'cd /opt && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz | tar xJv && ln -s /opt/flutter/bin/flutter /usr/local/bin/ && flutter precache'

contrib/ci/docker/i386/00-debian-stable-base.dockerfile

@@ -0,0 +1 @@
+../00-debian-stable-base.dockerfile

contrib/ci/docker/i386/10-debian-stable.dockerfile

@@ -0,0 +1 @@
+../10-debian-stable.dockerfile

contrib/ci/docker/lint.dockerfile

@@ -1,2 +1,3 @@
-FROM registry.oxen.rocks/lokinet-ci-debian-sid-base
+ARG ARCH=amd64
+FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH}
 RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata git clang-format-11 jsonnet'

contrib/ci/docker/readme.md

@@ -3,7 +3,7 @@
 To rebuild all ci images and push them to the oxen registry server do:
 
     $ docker login registry.oxen.rocks
-    $ ./rebuild-docker-images.sh *.dockerfile
+    $ ./rebuild-docker-images.sh
 
 If you aren't part of the Oxen team, you'll likely need to set up your own registry and change
 registry.oxen.rocks to your own domain name in order to do anything useful with this.

contrib/ci/docker/rebuild-docker-images.sh

@@ -2,18 +2,41 @@
 
 set -o errexit
 
+trap 'echo -e "\n\n\n\e[31;1mAn error occurred!\e[1m\n\n"' ERR
+
 registry=registry.oxen.rocks
 
 if [[ $# -eq 0 ]]; then
-    files=(*.dockerfile)
+    files=(*.dockerfile i386/*.dockerfile arm64v8/*.dockerfile arm32v7/*.dockerfile)
 else
     files=("$@")
 fi
 
+declare -A manifests
+
 for file in "${files[@]}"; do
-    name="${file#[0-9][0-9]-}" # s/^\d\d-//
+    if [[ "$file" == */* ]]; then
+        arch="${file%%/*}"
+        name="${file#*/}"
+    else
+        arch="amd64"
+        name="$file"
+    fi
+
+    name="${name#[0-9][0-9]-}" # s/^\d\d-//
     name="${name%.dockerfile}" # s/\.dockerfile$//
-    echo -e "\e[32;1mrebuilding $name\e[0m"
+    namearch=$registry/lokinet-ci-$name/$arch
-    docker build --pull -f $file -t $registry/lokinet-ci-$name .
+    latest=$registry/lokinet-ci-$name:latest
-    docker push $registry/lokinet-ci-$name
+    echo -e "\e[32;1mrebuilding \e[35;1m$namearch\e[0m"
+    docker build --pull -f $file -t $namearch --build-arg ARCH=$arch .
+    docker push $namearch
+
+    manifests[$latest]="${manifests[$latest]} $namearch"
+done
+
+for latest in "${!manifests[@]}"; do
+    echo -e "\e[32;1mpushing new manifest for \e[33;1m$latest[\e[35;1m${manifests[$latest]} \e[33;1m]\e[0m"
+    docker manifest rm $latest 2>/dev/null || true
+    docker manifest create $latest ${manifests[$latest]}
+    docker manifest push $latest
 done