From 07fb41cfd4d54de8215b87cbe6c190c05680f1af Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 13 Oct 2021 08:05:12 -0400 Subject: [PATCH 01/25] fix debian sid compiler error --- crypto/libntrup/src/avx/mult.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/libntrup/src/avx/mult.c b/crypto/libntrup/src/avx/mult.c index 237bb240b..687acb46a 100644 --- a/crypto/libntrup/src/avx/mult.c +++ b/crypto/libntrup/src/avx/mult.c @@ -333,7 +333,7 @@ mult96x8_float(__m256 h[192], const __m256 f[96], const __m256 g[96]) /* 96*(16*int8 stored in 32*int8) g inputs between -8 and 8 */ /* 192*16*int16 h outputs between -2400 and 2400 */ static void -mult96x16(__m256i h[192], const __m256i f[96], const __m256i g[96]) +mult96x16(__m256i * h, const __m256i * f, const __m256i * g) { __m256 hfloat[192]; __m256 gfloat[96]; From 50449038b48354f291608c7057039a23c51d1550 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 16 Oct 2021 15:02:06 -0300 Subject: [PATCH 02/25] Another gcc-11 fix GCC is wrongly warning that `h` is uninitialized here, but it clearly isn't. Work around it. --- llarp/iwp/session.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp index d9388325c..0b5b3bec5 100644 --- a/llarp/iwp/session.cpp +++ b/llarp/iwp/session.cpp @@ -748,11 +748,14 @@ namespace llarp LogError("short XMIT from ", m_RemoteAddr); return; } - uint16_t sz = bufbe16toh(data.data() + CommandOverhead + PacketOverhead); - uint64_t rxid = bufbe64toh(data.data() + CommandOverhead + sizeof(uint16_t) + PacketOverhead); - ShortHash h{ - data.data() + CommandOverhead + sizeof(uint16_t) + sizeof(uint64_t) + PacketOverhead}; - LogTrace("rxid=", rxid, " sz=", sz, " h=", h.ToHex(), " from ", m_RemoteAddr); + auto* pos = data.data() + CommandOverhead + PacketOverhead; + uint16_t sz = bufbe16toh(pos); + pos += sizeof(sz); + uint64_t rxid = bufbe64toh(pos); + pos += sizeof(rxid); + auto p2 = pos + ShortHash::SIZE; + assert(p2 == data.data() + XMITOverhead); + LogTrace("rxid=", rxid, " sz=", sz, " h=", oxenmq::to_hex(pos, p2), " from ", m_RemoteAddr); m_LastRX = m_Parent->Now(); { // check for replay @@ -769,8 +772,8 @@ namespace llarp auto itr = m_RXMsgs.find(rxid); if (itr == m_RXMsgs.end()) { - itr = - m_RXMsgs.emplace(rxid, InboundMessage{rxid, sz, std::move(h), m_Parent->Now()}).first; + itr = m_RXMsgs.emplace(rxid, InboundMessage{rxid, sz, ShortHash{pos}, m_Parent->Now()}) + .first; sz = std::min(sz, uint16_t{FragmentSize}); if ((data.size() - XMITOverhead) == sz) { From 497c62b586760e7cc5c79474043e24137351ac2f Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 14 Oct 2021 19:49:45 -0300 Subject: [PATCH 03/25] Update clang build to 13; add full llvm-13 build --- .drone.jsonnet | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index ece1882bd..48a38b93a 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -176,6 +176,26 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = ] }; +local clang(version) = debian_pipeline( + 'Debian sid/clang-' + version + ' (amd64)', + docker_base + 'debian-sid', + deps='clang-' + version + ' ' + default_deps_nocxx, + cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' ' +); + +local full_llvm(version) = debian_pipeline( + 'Debian sid/llvm-' + version + ' (amd64)', + 'debian:sid', + deps='clang-' + version + ' lld-' + version + ' libc++-' + version + '-dev libc++abi-' + version + '-dev ' + + default_deps_nocxx, + cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + + ' -DCMAKE_CXX_COMPILER=clang++-' + version + + ' -DCMAKE_CXX_FLAGS=-stdlib=libc++ ' + + std.join(' ', [ + '-DCMAKE_' + type + '_LINKER_FLAGS=-fuse-ld=lld-' + version + for type in ['EXE', 'MODULE', 'SHARED'] + ]) +); // Macos build local mac_builder(name, @@ -226,8 +246,8 @@ local mac_builder(name, // Various debian builds debian_pipeline("Debian sid (amd64)", "debian:sid"), debian_pipeline("Debian sid/Debug (amd64)", "debian:sid", build_type='Debug'), - debian_pipeline("Debian sid/clang-11 (amd64)", docker_base+'debian-sid', deps='clang-11 '+default_deps_nocxx, - cmake_extra='-DCMAKE_C_COMPILER=clang-11 -DCMAKE_CXX_COMPILER=clang++-11 '), + clang(13), + full_llvm(13), debian_pipeline("Debian buster (i386)", "i386/debian:buster", cmake_extra='-DDOWNLOAD_SODIUM=ON'), debian_pipeline("Ubuntu focal (amd64)", docker_base+'ubuntu-focal'), debian_pipeline("Ubuntu bionic (amd64)", "ubuntu:bionic", deps='g++-8 ' + default_deps_nocxx, From 1e22417adea7dc790d46d0a02cfd83f7e2690b87 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 14 Oct 2021 19:58:26 -0300 Subject: [PATCH 04/25] Add missing deprecated copy assignment operator Clang-13 warns (and -Werror dies) without it. --- llarp/router_contact.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 347c0e405..2a7aef20a 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -35,7 +35,8 @@ namespace llarp explicit NetID(const byte_t* val); - explicit NetID(const NetID& other) = default; + NetID(const NetID& other) = default; + NetID& operator=(const NetID& other) = default; bool operator==(const NetID& other) const; From 9b0b9fe67b8939f0183fbfbf1efc99d6904f6cb5 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 14 Oct 2021 21:01:33 -0300 Subject: [PATCH 05/25] Add jsonnet formatting (if jsonnet is installed) --- contrib/ci/drone-format-verify.sh | 1 + contrib/format.sh | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/contrib/ci/drone-format-verify.sh b/contrib/ci/drone-format-verify.sh index 68a7ac620..387340018 100755 --- a/contrib/ci/drone-format-verify.sh +++ b/contrib/ci/drone-format-verify.sh @@ -2,4 +2,5 @@ test "x$IGNORE" != "x" && exit 0 repo=$(readlink -e $(dirname $0)/../../) clang-format-11 -i $(find $repo/jni $repo/daemon $repo/llarp $repo/include $repo/pybind | grep -E '\.[hc](pp)?$') +jsonnetfmt -i $repo/.drone.jsonnet git --no-pager diff --exit-code --color || (echo -ne '\n\n\e[31;1mLint check failed; please run ./contrib/format.sh\e[0m\n\n' ; exit 1) diff --git a/contrib/format.sh b/contrib/format.sh index e514590bd..28ceb65ce 100755 --- a/contrib/format.sh +++ b/contrib/format.sh @@ -41,3 +41,14 @@ if [ $? -eq 0 ]; then fi fi + +jsonnet_format=$(command -v jsonnetfmt 2>/dev/null) +if [ $? -eq 0 ]; then + if [ "$1" = "verify" ]; then + if ! $jsonnet_format --test .drone.jsonnet; then + exit 4 + fi + else + $jsonnet_format --in-place .drone.jsonnet + fi +fi From 52492c62530a7563ad77054bb86b38cdbcdf30a0 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 14 Oct 2021 21:02:11 -0300 Subject: [PATCH 06/25] make format --- .drone.jsonnet | 535 ++++++++++++++++++++------------------- llarp/router_contact.hpp | 3 +- 2 files changed, 279 insertions(+), 259 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 48a38b93a..d8dae0803 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,301 +1,320 @@ -local default_deps_base='libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libcurl4-openssl-dev make'; -local default_deps_nocxx='libsodium-dev ' + default_deps_base; // libsodium-dev needs to be >= 1.0.18 -local default_deps='g++ ' + default_deps_nocxx; -local default_windows_deps='mingw-w64 zip nsis'; +local default_deps_base = 'libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libcurl4-openssl-dev make'; +local default_deps_nocxx = 'libsodium-dev ' + default_deps_base; // libsodium-dev needs to be >= 1.0.18 +local default_deps = 'g++ ' + default_deps_nocxx; +local default_windows_deps = 'mingw-w64 zip nsis'; local docker_base = 'registry.oxen.rocks/lokinet-ci-'; local submodules = { - name: 'submodules', - image: 'drone/git', - commands: ['git fetch --tags', 'git submodule update --init --recursive --depth=1'] + name: 'submodules', + image: 'drone/git', + commands: ['git fetch --tags', 'git submodule update --init --recursive --depth=1'], }; local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q'; // Regular build on a debian-like system: -local debian_pipeline(name, image, - arch='amd64', - deps=default_deps, - build_type='Release', - lto=false, - werror=true, - cmake_extra='', - extra_cmds=[], - jobs=6, - tests=true, - loki_repo=false, - allow_fail=false) = { - kind: 'pipeline', - type: 'docker', - name: name, - platform: { arch: arch }, - trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } }, - steps: [ - submodules, - { - name: 'build', - image: image, - [if allow_fail then "failure"]: "ignore", - environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, - commands: [ - 'echo "Building on ${DRONE_STAGE_MACHINE}"', - 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', - apt_get_quiet + ' update', - apt_get_quiet + ' install -y eatmydata', - ] + (if loki_repo then [ +local debian_pipeline(name, + image, + arch='amd64', + deps=default_deps, + build_type='Release', + lto=false, + werror=true, + cmake_extra='', + extra_cmds=[], + jobs=6, + tests=true, + loki_repo=false, + allow_fail=false) = { + kind: 'pipeline', + type: 'docker', + name: name, + platform: { arch: arch }, + trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } }, + steps: [ + submodules, + { + name: 'build', + image: image, + [if allow_fail then 'failure']: 'ignore', + environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, + commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + ] + ( + if loki_repo then [ 'eatmydata ' + apt_get_quiet + ' install -y lsb-release', 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', 'echo deb http://deb.loki.network $$(lsb_release -sc) main >/etc/apt/sources.list.d/loki.network.list', - 'eatmydata ' + apt_get_quiet + ' update' - ] else [] + 'eatmydata ' + apt_get_quiet + ' update', + ] else [] ) + [ - 'eatmydata ' + apt_get_quiet + ' dist-upgrade -y', - 'eatmydata ' + apt_get_quiet + ' install -y gdb cmake git pkg-config ccache ' + deps, - 'mkdir build', - 'cd build', - 'cmake .. -DWITH_SETCAP=OFF -DCMAKE_CXX_FLAGS=-fdiagnostics-color=always -DCMAKE_BUILD_TYPE='+build_type+' ' + - (if werror then '-DWARNINGS_AS_ERRORS=ON ' else '') + - '-DWITH_LTO=' + (if lto then 'ON ' else 'OFF ') + - (if tests then '' else '-DWITH_TESTS=OFF ') + - cmake_extra, - 'VERBOSE=1 make -j' + jobs, - ] - + (if tests then ['../contrib/ci/drone-gdb.sh ./test/testAll --use-colour yes'] else []) - + extra_cmds, - } - ], + 'eatmydata ' + apt_get_quiet + ' dist-upgrade -y', + 'eatmydata ' + apt_get_quiet + ' install -y gdb cmake git pkg-config ccache ' + deps, + 'mkdir build', + 'cd build', + 'cmake .. -DWITH_SETCAP=OFF -DCMAKE_CXX_FLAGS=-fdiagnostics-color=always -DCMAKE_BUILD_TYPE=' + build_type + ' ' + + (if werror then '-DWARNINGS_AS_ERRORS=ON ' else '') + + '-DWITH_LTO=' + (if lto then 'ON ' else 'OFF ') + + (if tests then '' else '-DWITH_TESTS=OFF ') + + cmake_extra, + 'VERBOSE=1 make -j' + jobs, + ] + + (if tests then ['../contrib/ci/drone-gdb.sh ./test/testAll --use-colour yes'] else []) + + extra_cmds, + }, + ], }; local apk_builder(name, image, extra_cmds=[], allow_fail=false, jobs=6) = { - kind: 'pipeline', - type: 'docker', - name: name, - platform: {arch: "amd64"}, - trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } }, - steps: [ - submodules, - { - name: 'build', - image: image, - [if allow_fail then "failure"]: "ignore", - environment: { SSH_KEY: { from_secret: "SSH_KEY" }, ANDROID: "android" }, - commands: [ - 'VERBOSE=1 JOBS='+jobs+' NDK=/usr/lib/android-ndk ./contrib/android.sh', - 'git clone https://github.com/oxen-io/lokinet-flutter-app lokinet-mobile', - 'cp -av lokinet-jni-*/* lokinet-mobile/lokinet_lib/android/src/main/jniLibs/', - 'cd lokinet-mobile', - 'flutter build apk --debug', - 'cd ..', - 'cp lokinet-mobile/build/app/outputs/apk/debug/app-debug.apk lokinet.apk' - ] + extra_cmds - } - ] + kind: 'pipeline', + type: 'docker', + name: name, + platform: { arch: 'amd64' }, + trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } }, + steps: [ + submodules, + { + name: 'build', + image: image, + [if allow_fail then 'failure']: 'ignore', + environment: { SSH_KEY: { from_secret: 'SSH_KEY' }, ANDROID: 'android' }, + commands: [ + 'VERBOSE=1 JOBS=' + jobs + ' NDK=/usr/lib/android-ndk ./contrib/android.sh', + 'git clone https://github.com/oxen-io/lokinet-flutter-app lokinet-mobile', + 'cp -av lokinet-jni-*/* lokinet-mobile/lokinet_lib/android/src/main/jniLibs/', + 'cd lokinet-mobile', + 'flutter build apk --debug', + 'cd ..', + 'cp lokinet-mobile/build/app/outputs/apk/debug/app-debug.apk lokinet.apk', + ] + extra_cmds, + }, + ], }; // windows cross compile on debian -local windows_cross_pipeline(name, image, - arch='amd64', - build_type='Release', - lto=false, - werror=false, - cmake_extra='', - toolchain='32', - extra_cmds=[], - jobs=6, - allow_fail=false) = { - kind: 'pipeline', - type: 'docker', - name: name, - platform: { arch: arch }, - trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } }, - steps: [ - submodules, - { - name: 'build', - image: image, - [if allow_fail then "failure"]: "ignore", - environment: { SSH_KEY: { from_secret: "SSH_KEY" }, WINDOWS_BUILD_NAME: toolchain+"bit" }, - commands: [ - 'echo "Building on ${DRONE_STAGE_MACHINE}"', - 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', - apt_get_quiet + ' update', - apt_get_quiet + ' install -y eatmydata', - 'eatmydata ' + apt_get_quiet + ' install -y build-essential cmake git pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool', - 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix', - 'update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix', - 'VERBOSE=1 JOBS=' + jobs + ' ./contrib/windows.sh' - ] + extra_cmds, - } - ], +local windows_cross_pipeline(name, + image, + arch='amd64', + build_type='Release', + lto=false, + werror=false, + cmake_extra='', + toolchain='32', + extra_cmds=[], + jobs=6, + allow_fail=false) = { + kind: 'pipeline', + type: 'docker', + name: name, + platform: { arch: arch }, + trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } }, + steps: [ + submodules, + { + name: 'build', + image: image, + [if allow_fail then 'failure']: 'ignore', + environment: { SSH_KEY: { from_secret: 'SSH_KEY' }, WINDOWS_BUILD_NAME: toolchain + 'bit' }, + commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + 'eatmydata ' + apt_get_quiet + ' install -y build-essential cmake git pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool', + 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix', + 'update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix', + 'VERBOSE=1 JOBS=' + jobs + ' ./contrib/windows.sh', + ] + extra_cmds, + }, + ], }; // Builds a snapshot .deb on a debian-like system by merging into the debian/* or ubuntu/* branch local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = { - kind: 'pipeline', - type: 'docker', - name: 'DEB (' + distro + (if arch == 'amd64' then '' else '/' + arch) + ')', - platform: { arch: arch }, - environment: { distro_branch: distro_branch, distro: distro }, - steps: [ - submodules, - { - name: 'build', - image: image, - failure: 'ignore', - environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, - commands: [ - 'echo "Building on ${DRONE_STAGE_MACHINE}"', - 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' - ] + (if loki_repo then [ - 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', - 'echo deb http://deb.loki.network $${distro} main >/etc/apt/sources.list.d/loki.network.list' - ] else []) + [ - apt_get_quiet + ' update', - apt_get_quiet + ' install -y eatmydata', - 'eatmydata ' + apt_get_quiet + ' install -y git devscripts equivs ccache git-buildpackage python3-dev', - ||| - # Look for the debian branch in this repo first, try upstream if that fails. - if ! git checkout $${distro_branch}; then - git remote add --fetch upstream https://github.com/oxen-io/loki-network.git && - git checkout $${distro_branch} - fi - |||, - # Tell the merge how to resolve conflicts in the source .drone.jsonnet (we don't - # care about it at all since *this* .drone.jsonnet is already loaded). - 'git config merge.ours.driver true', - 'echo .drone.jsonnet merge=ours >>.gitattributes', + kind: 'pipeline', + type: 'docker', + name: 'DEB (' + distro + (if arch == 'amd64' then '' else '/' + arch) + ')', + platform: { arch: arch }, + environment: { distro_branch: distro_branch, distro: distro }, + steps: [ + submodules, + { + name: 'build', + image: image, + failure: 'ignore', + environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, + commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', + ] + (if loki_repo then [ + 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', + 'echo deb http://deb.loki.network $${distro} main >/etc/apt/sources.list.d/loki.network.list', + ] else []) + [ + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + 'eatmydata ' + apt_get_quiet + ' install -y git devscripts equivs ccache git-buildpackage python3-dev', + ||| + # Look for the debian branch in this repo first, try upstream if that fails. + if ! git checkout $${distro_branch}; then + git remote add --fetch upstream https://github.com/oxen-io/loki-network.git && + git checkout $${distro_branch} + fi + |||, + // Tell the merge how to resolve conflicts in the source .drone.jsonnet (we don't + // care about it at all since *this* .drone.jsonnet is already loaded). + 'git config merge.ours.driver true', + 'echo .drone.jsonnet merge=ours >>.gitattributes', - 'git merge ${DRONE_COMMIT}', - 'export DEBEMAIL="${DRONE_COMMIT_AUTHOR_EMAIL}" DEBFULLNAME="${DRONE_COMMIT_AUTHOR_NAME}"', - 'gbp dch -S -s "HEAD^" --spawn-editor=never -U low', - 'eatmydata mk-build-deps --install --remove --tool "' + apt_get_quiet + ' -o Debug::pkgProblemResolver=yes --no-install-recommends -y"', - 'export DEB_BUILD_OPTIONS="parallel=$$(nproc)"', - #'grep -q lib debian/lokinet-bin.install || echo "/usr/lib/lib*.so*" >>debian/lokinet-bin.install', - 'debuild -e CCACHE_DIR -b', - './contrib/ci/drone-debs-upload.sh ' + distro, - ] - } - ] + 'git merge ${DRONE_COMMIT}', + 'export DEBEMAIL="${DRONE_COMMIT_AUTHOR_EMAIL}" DEBFULLNAME="${DRONE_COMMIT_AUTHOR_NAME}"', + 'gbp dch -S -s "HEAD^" --spawn-editor=never -U low', + 'eatmydata mk-build-deps --install --remove --tool "' + apt_get_quiet + ' -o Debug::pkgProblemResolver=yes --no-install-recommends -y"', + 'export DEB_BUILD_OPTIONS="parallel=$$(nproc)"', + //'grep -q lib debian/lokinet-bin.install || echo "/usr/lib/lib*.so*" >>debian/lokinet-bin.install', + 'debuild -e CCACHE_DIR -b', + './contrib/ci/drone-debs-upload.sh ' + distro, + ], + }, + ], }; local clang(version) = debian_pipeline( - 'Debian sid/clang-' + version + ' (amd64)', - docker_base + 'debian-sid', - deps='clang-' + version + ' ' + default_deps_nocxx, - cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' ' + 'Debian sid/clang-' + version + ' (amd64)', + docker_base + 'debian-sid', + deps='clang-' + version + ' ' + default_deps_nocxx, + cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' ' ); local full_llvm(version) = debian_pipeline( - 'Debian sid/llvm-' + version + ' (amd64)', - 'debian:sid', - deps='clang-' + version + ' lld-' + version + ' libc++-' + version + '-dev libc++abi-' + version + '-dev ' - + default_deps_nocxx, - cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + - ' -DCMAKE_CXX_COMPILER=clang++-' + version + - ' -DCMAKE_CXX_FLAGS=-stdlib=libc++ ' + - std.join(' ', [ - '-DCMAKE_' + type + '_LINKER_FLAGS=-fuse-ld=lld-' + version - for type in ['EXE', 'MODULE', 'SHARED'] - ]) + 'Debian sid/llvm-' + version + ' (amd64)', + 'debian:sid', + deps='clang-' + version + ' lld-' + version + ' libc++-' + version + '-dev libc++abi-' + version + '-dev ' + + default_deps_nocxx, + cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + + ' -DCMAKE_CXX_COMPILER=clang++-' + version + + ' -DCMAKE_CXX_FLAGS=-stdlib=libc++ ' + + std.join(' ', [ + '-DCMAKE_' + type + '_LINKER_FLAGS=-fuse-ld=lld-' + version + for type in ['EXE', 'MODULE', 'SHARED'] + ]) ); // Macos build local mac_builder(name, - build_type='Release', - werror=true, - cmake_extra='', - extra_cmds=[], - jobs=6, - allow_fail=false) = { - kind: 'pipeline', - type: 'exec', - name: name, - platform: { os: 'darwin', arch: 'amd64' }, - steps: [ - { name: 'submodules', commands: ['git fetch --tags', 'git submodule update --init --recursive'] }, - { - name: 'build', - environment: { SSH_KEY: { from_secret: "SSH_KEY" } }, - commands: [ - 'echo "Building on ${DRONE_STAGE_MACHINE}"', - // If you don't do this then the C compiler doesn't have an include path containing - // basic system headers. WTF apple: - 'export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"', - 'ulimit -n 1024', // because macos sets ulimit to 256 for some reason yeah idk - './contrib/mac.sh' - ] + extra_cmds, - } - ] + build_type='Release', + werror=true, + cmake_extra='', + extra_cmds=[], + jobs=6, + allow_fail=false) = { + kind: 'pipeline', + type: 'exec', + name: name, + platform: { os: 'darwin', arch: 'amd64' }, + steps: [ + { name: 'submodules', commands: ['git fetch --tags', 'git submodule update --init --recursive'] }, + { + name: 'build', + environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, + commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + // If you don't do this then the C compiler doesn't have an include path containing + // basic system headers. WTF apple: + 'export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"', + 'ulimit -n 1024', // because macos sets ulimit to 256 for some reason yeah idk + './contrib/mac.sh', + ] + extra_cmds, + }, + ], }; [ - { - name: 'lint check', - kind: 'pipeline', - type: 'docker', - steps: [{ - name: 'build', image: 'registry.oxen.rocks/lokinet-ci-lint', - commands: [ - 'echo "Building on ${DRONE_STAGE_MACHINE}"', - apt_get_quiet + ' update', - apt_get_quiet + ' install -y eatmydata', - 'eatmydata ' + apt_get_quiet + ' install -y git clang-format-11', - './contrib/ci/drone-format-verify.sh'] - }] - }, + { + name: 'lint check', + kind: 'pipeline', + type: 'docker', + steps: [{ + name: 'build', + image: 'registry.oxen.rocks/lokinet-ci-lint', + commands: [ + 'echo "Building on ${DRONE_STAGE_MACHINE}"', + apt_get_quiet + ' update', + apt_get_quiet + ' install -y eatmydata', + 'eatmydata ' + apt_get_quiet + ' install -y git clang-format-11 jsonnet', + './contrib/ci/drone-format-verify.sh', + ], + }], + }, - // Various debian builds - debian_pipeline("Debian sid (amd64)", "debian:sid"), - debian_pipeline("Debian sid/Debug (amd64)", "debian:sid", build_type='Debug'), - clang(13), - full_llvm(13), - debian_pipeline("Debian buster (i386)", "i386/debian:buster", cmake_extra='-DDOWNLOAD_SODIUM=ON'), - debian_pipeline("Ubuntu focal (amd64)", docker_base+'ubuntu-focal'), - debian_pipeline("Ubuntu bionic (amd64)", "ubuntu:bionic", deps='g++-8 ' + default_deps_nocxx, - cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8', loki_repo=true), + // Various debian builds + debian_pipeline('Debian sid (amd64)', 'debian:sid'), + debian_pipeline('Debian sid/Debug (amd64)', 'debian:sid', build_type='Debug'), + clang(13), + full_llvm(13), + debian_pipeline('Debian buster (i386)', 'i386/debian:buster', cmake_extra='-DDOWNLOAD_SODIUM=ON'), + debian_pipeline('Ubuntu focal (amd64)', docker_base + 'ubuntu-focal'), + debian_pipeline('Ubuntu bionic (amd64)', + 'ubuntu:bionic', + deps='g++-8 ' + default_deps_nocxx, + cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8', + loki_repo=true), - // ARM builds (ARM64 and armhf) - debian_pipeline("Debian sid (ARM64)", "debian:sid", arch="arm64", jobs=4), - debian_pipeline("Debian buster (armhf)", "arm32v7/debian:buster", arch="arm64", cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4), - // Static armhf build (gets uploaded) - debian_pipeline("Static (buster armhf)", "arm32v7/debian:buster", arch="arm64", deps='g++ python3-dev automake libtool', - cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + - '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp" -DCMAKE_C_FLAGS="-march=armv7-a+fp" -DNATIVE_BUILD=OFF ' + - '-DWITH_SYSTEMD=OFF', - extra_cmds=[ - '../contrib/ci/drone-check-static-libs.sh', - 'UPLOAD_OS=linux-armhf ../contrib/ci/drone-static-upload.sh' - ], - jobs=4), - // android apk builder - apk_builder("android apk", "registry.oxen.rocks/lokinet-ci-android", extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']), + // ARM builds (ARM64 and armhf) + debian_pipeline('Debian sid (ARM64)', 'debian:sid', arch='arm64', jobs=4), + debian_pipeline('Debian buster (armhf)', 'arm32v7/debian:buster', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4), + // Static armhf build (gets uploaded) + debian_pipeline('Static (buster armhf)', + 'arm32v7/debian:buster', + arch='arm64', + deps='g++ python3-dev automake libtool', + cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + + '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp" -DCMAKE_C_FLAGS="-march=armv7-a+fp" -DNATIVE_BUILD=OFF ' + + '-DWITH_SYSTEMD=OFF', + extra_cmds=[ + '../contrib/ci/drone-check-static-libs.sh', + 'UPLOAD_OS=linux-armhf ../contrib/ci/drone-static-upload.sh', + ], + jobs=4), + // android apk builder + apk_builder('android apk', 'registry.oxen.rocks/lokinet-ci-android', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']), - // Windows builds (x64) - windows_cross_pipeline("Windows (amd64)", docker_base+'debian-win32-cross', - toolchain='64', extra_cmds=[ - './contrib/ci/drone-static-upload.sh' - ]), + // Windows builds (x64) + windows_cross_pipeline('Windows (amd64)', + docker_base + 'debian-win32-cross', + toolchain='64', + extra_cmds=[ + './contrib/ci/drone-static-upload.sh', + ]), - // Static build (on bionic) which gets uploaded to builds.lokinet.dev: - debian_pipeline("Static (bionic amd64)", docker_base+'ubuntu-bionic', deps='g++-8 python3-dev automake libtool', lto=true, tests=false, - cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 ' + - '-DCMAKE_CXX_FLAGS="-march=x86-64 -mtune=haswell" -DCMAKE_C_FLAGS="-march=x86-64 -mtune=haswell" -DNATIVE_BUILD=OFF ' + - '-DWITH_SYSTEMD=OFF', - extra_cmds=[ - '../contrib/ci/drone-check-static-libs.sh', - '../contrib/ci/drone-static-upload.sh' - ]), + // Static build (on bionic) which gets uploaded to builds.lokinet.dev: + debian_pipeline('Static (bionic amd64)', + docker_base + 'ubuntu-bionic', + deps='g++-8 python3-dev automake libtool', + lto=true, + tests=false, + cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 ' + + '-DCMAKE_CXX_FLAGS="-march=x86-64 -mtune=haswell" -DCMAKE_C_FLAGS="-march=x86-64 -mtune=haswell" -DNATIVE_BUILD=OFF ' + + '-DWITH_SYSTEMD=OFF', + extra_cmds=[ + '../contrib/ci/drone-check-static-libs.sh', + '../contrib/ci/drone-static-upload.sh', + ]), - // integration tests - debian_pipeline("Router Hive", "ubuntu:focal", deps='python3-dev python3-pytest python3-pybind11 ' + default_deps, - cmake_extra='-DWITH_HIVE=ON'), + // integration tests + debian_pipeline('Router Hive', + 'ubuntu:focal', + deps='python3-dev python3-pytest python3-pybind11 ' + default_deps, + cmake_extra='-DWITH_HIVE=ON'), - // Deb builds: - deb_builder("debian:sid", "sid", "debian/sid"), - deb_builder("debian:buster", "buster", "debian/buster"), - deb_builder("ubuntu:focal", "focal", "ubuntu/focal"), - deb_builder("debian:sid", "sid", "debian/sid", arch='arm64'), + // Deb builds: + deb_builder('debian:sid', 'sid', 'debian/sid'), + deb_builder('debian:buster', 'buster', 'debian/buster'), + deb_builder('ubuntu:focal', 'focal', 'ubuntu/focal'), + deb_builder('debian:sid', 'sid', 'debian/sid', arch='arm64'), - // Macos builds: - mac_builder('macOS (Release)'), - mac_builder('macOS (Debug)', build_type='Debug'), + // Macos builds: + mac_builder('macOS (Release)'), + mac_builder('macOS (Debug)', build_type='Debug'), ] diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 2a7aef20a..bb500dfba 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -36,7 +36,8 @@ namespace llarp explicit NetID(const byte_t* val); NetID(const NetID& other) = default; - NetID& operator=(const NetID& other) = default; + NetID& + operator=(const NetID& other) = default; bool operator==(const NetID& other) const; From 1526b2a2eb8fd268da850f4a5c88055fba3fec22 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 14 Oct 2021 21:02:23 -0300 Subject: [PATCH 07/25] which => command -v; different format.sh exit codes `which` is a debian tool that is being deprecated in favour of posix-standard `command -v`, so which to that to avoid deprecation warnings. Change the exit codes of `contrib/format.sh verify` to be unique for each formatting program. --- contrib/format.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/format.sh b/contrib/format.sh index 28ceb65ce..956f5db92 100755 --- a/contrib/format.sh +++ b/contrib/format.sh @@ -2,12 +2,12 @@ CLANG_FORMAT_DESIRED_VERSION=11 -binary=$(which clang-format-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null) +binary=$(command -v clang-format-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null) if [ $? -ne 0 ]; then - binary=$(which clang-format-mp-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null) + binary=$(command -v clang-format-mp-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null) fi if [ $? -ne 0 ]; then - binary=$(which clang-format 2>/dev/null) + binary=$(command -v clang-format 2>/dev/null) if [ $? -ne 0 ]; then echo "Please install clang-format version $CLANG_FORMAT_DESIRED_VERSION and re-run this script." exit 1 @@ -22,18 +22,18 @@ fi cd "$(dirname $0)/../" if [ "$1" = "verify" ] ; then if [ $($binary --output-replacements-xml $(find jni daemon llarp include pybind | grep -E '\.([hc](pp)?|mm?)$' | grep -v '\#') | grep '' | wc -l) -ne 0 ] ; then - exit 1 + exit 2 fi else $binary -i $(find jni daemon llarp include pybind | grep -E '\.([hc](pp)?|mm)$' | grep -v '\#') &> /dev/null fi -swift_format=$(which swiftformat 2>/dev/null) +swift_format=$(command -v swiftformat 2>/dev/null) if [ $? -eq 0 ]; then if [ "$1" = "verify" ] ; then for f in $(find daemon | grep -E '\.swift$' | grep -v '\#') ; do if [ $($swift_format --quiet --dryrun < "$f" | diff "$f" - | wc -l) -ne 0 ] ; then - exit 1 + exit 3 fi done else From 9c32058a50834e348eb5c26012e16c9e21d6d447 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 14 Oct 2021 21:05:34 -0300 Subject: [PATCH 08/25] Remove windows linefeeds --- cmake/libatomic.cmake | 98 +++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/cmake/libatomic.cmake b/cmake/libatomic.cmake index d29155913..3a8234e45 100644 --- a/cmake/libatomic.cmake +++ b/cmake/libatomic.cmake @@ -1,49 +1,49 @@ -function(check_working_cxx_atomics64 varname) - set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) - if (EMBEDDED_CFG) - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -m32 -march=i486") - elseif(MSVC OR MSVC_VERSION) - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -arch:IA32 -std:c++14") - else() - # CMAKE_CXX_STANDARD does not propagate to cmake compile tests - set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=c++14") - endif() - check_cxx_source_compiles(" -#include -#include -std::atomic x (0); -int main() { - uint64_t i = x.load(std::memory_order_relaxed); - return 0; -} -" ${varname}) - set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) -endfunction() - -function(link_libatomic) - check_working_cxx_atomics64(HAVE_CXX_ATOMICS64_WITHOUT_LIB) - - if(HAVE_CXX_ATOMICS64_WITHOUT_LIB) - message(STATUS "Have working 64bit atomics") - return() - endif() - - if (NOT MSVC AND NOT MSVC_VERSION) - check_library_exists(atomic __atomic_load_8 "" HAVE_CXX_LIBATOMICS64) - if (HAVE_CXX_LIBATOMICS64) - message(STATUS "Have 64bit atomics via library") - list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic") - check_working_cxx_atomics64(HAVE_CXX_ATOMICS64_WITH_LIB) - if (HAVE_CXX_ATOMICS64_WITH_LIB) - message(STATUS "Can link with libatomic") - link_libraries(-latomic) - return() - endif() - endif() - endif() - if (MSVC OR MSVC_VERSION) - message(FATAL_ERROR "Host compiler must support 64-bit std::atomic! (What does MSVC do to inline atomics?)") - else() - message(FATAL_ERROR "Host compiler must support 64-bit std::atomic!") - endif() -endfunction() \ No newline at end of file +function(check_working_cxx_atomics64 varname) + set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) + if (EMBEDDED_CFG) + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -m32 -march=i486") + elseif(MSVC OR MSVC_VERSION) + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -arch:IA32 -std:c++14") + else() + # CMAKE_CXX_STANDARD does not propagate to cmake compile tests + set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=c++14") + endif() + check_cxx_source_compiles(" +#include +#include +std::atomic x (0); +int main() { + uint64_t i = x.load(std::memory_order_relaxed); + return 0; +} +" ${varname}) + set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) +endfunction() + +function(link_libatomic) + check_working_cxx_atomics64(HAVE_CXX_ATOMICS64_WITHOUT_LIB) + + if(HAVE_CXX_ATOMICS64_WITHOUT_LIB) + message(STATUS "Have working 64bit atomics") + return() + endif() + + if (NOT MSVC AND NOT MSVC_VERSION) + check_library_exists(atomic __atomic_load_8 "" HAVE_CXX_LIBATOMICS64) + if (HAVE_CXX_LIBATOMICS64) + message(STATUS "Have 64bit atomics via library") + list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic") + check_working_cxx_atomics64(HAVE_CXX_ATOMICS64_WITH_LIB) + if (HAVE_CXX_ATOMICS64_WITH_LIB) + message(STATUS "Can link with libatomic") + link_libraries(-latomic) + return() + endif() + endif() + endif() + if (MSVC OR MSVC_VERSION) + message(FATAL_ERROR "Host compiler must support 64-bit std::atomic! (What does MSVC do to inline atomics?)") + else() + message(FATAL_ERROR "Host compiler must support 64-bit std::atomic!") + endif() +endfunction() From 6f58648161336afde8ad34671df93c589a8dc227 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 15 Oct 2021 12:29:18 -0300 Subject: [PATCH 09/25] Add libzmq3-dev; change deps into arrays We're currently rebuilding libzmq3-dev all the time in most of the CI jobs. --- .drone.jsonnet | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index d8dae0803..02970c16b 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -1,7 +1,19 @@ -local default_deps_base = 'libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libcurl4-openssl-dev make'; -local default_deps_nocxx = 'libsodium-dev ' + default_deps_base; // libsodium-dev needs to be >= 1.0.18 -local default_deps = 'g++ ' + default_deps_nocxx; -local default_windows_deps = 'mingw-w64 zip nsis'; +local default_deps_base = [ + 'libsystemd-dev', + 'python3-dev', + 'libuv1-dev', + 'libunbound-dev', + 'nettle-dev', + 'libssl-dev', + 'libevent-dev', + 'libsqlite3-dev', + 'libcurl4-openssl-dev', + 'libzmq3-dev', + 'make', +]; +local default_deps_nocxx = ['libsodium-dev'] + default_deps_base; // libsodium-dev needs to be >= 1.0.18 +local default_deps = ['g++'] + default_deps_nocxx; +local default_windows_deps = ['mingw-w64', 'zip', 'nsis']; local docker_base = 'registry.oxen.rocks/lokinet-ci-'; local submodules = { @@ -52,7 +64,7 @@ local debian_pipeline(name, ] else [] ) + [ 'eatmydata ' + apt_get_quiet + ' dist-upgrade -y', - 'eatmydata ' + apt_get_quiet + ' install -y gdb cmake git pkg-config ccache ' + deps, + 'eatmydata ' + apt_get_quiet + ' install -y gdb cmake git pkg-config ccache ' + std.join(' ', deps), 'mkdir build', 'cd build', 'cmake .. -DWITH_SETCAP=OFF -DCMAKE_CXX_FLAGS=-fdiagnostics-color=always -DCMAKE_BUILD_TYPE=' + build_type + ' ' + @@ -182,14 +194,14 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = local clang(version) = debian_pipeline( 'Debian sid/clang-' + version + ' (amd64)', docker_base + 'debian-sid', - deps='clang-' + version + ' ' + default_deps_nocxx, + deps=['clang-' + version] + default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' ' ); local full_llvm(version) = debian_pipeline( 'Debian sid/llvm-' + version + ' (amd64)', 'debian:sid', - deps='clang-' + version + ' lld-' + version + ' libc++-' + version + '-dev libc++abi-' + version + '-dev ' + deps=['clang-' + version, ' lld-' + version, ' libc++-' + version + '-dev', 'libc++abi-' + version + '-dev'] + default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + @@ -257,7 +269,7 @@ local mac_builder(name, debian_pipeline('Ubuntu focal (amd64)', docker_base + 'ubuntu-focal'), debian_pipeline('Ubuntu bionic (amd64)', 'ubuntu:bionic', - deps='g++-8 ' + default_deps_nocxx, + deps=['g++-8'] + default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8', loki_repo=true), @@ -268,7 +280,7 @@ local mac_builder(name, debian_pipeline('Static (buster armhf)', 'arm32v7/debian:buster', arch='arm64', - deps='g++ python3-dev automake libtool', + deps=['g++', 'python3-dev', 'automake', 'libtool'], cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp" -DCMAKE_C_FLAGS="-march=armv7-a+fp" -DNATIVE_BUILD=OFF ' + '-DWITH_SYSTEMD=OFF', @@ -291,7 +303,7 @@ local mac_builder(name, // Static build (on bionic) which gets uploaded to builds.lokinet.dev: debian_pipeline('Static (bionic amd64)', docker_base + 'ubuntu-bionic', - deps='g++-8 python3-dev automake libtool', + deps=['g++-8', 'python3-dev', 'automake', 'libtool'], lto=true, tests=false, cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 ' + @@ -305,7 +317,7 @@ local mac_builder(name, // integration tests debian_pipeline('Router Hive', 'ubuntu:focal', - deps='python3-dev python3-pytest python3-pybind11 ' + default_deps, + deps=['python3-dev', 'python3-pytest', 'python3-pybind11'] + default_deps, cmake_extra='-DWITH_HIVE=ON'), // Deb builds: From ca9d979361f986b92a55e90c901edfd6675c7854 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 16 Oct 2021 12:11:39 -0300 Subject: [PATCH 10/25] Docker image updates - split debian sid into base/sid/clang images - similarly for debian stable - add jsonnet to lint - add `--pull` to docker build so that we always pull the latest images (otherwise we were building on whatever local cache we have for `debian:sid`, etc., which made the base image update layer much larger). - don't install Recommends by default - add libzmq3-dev - split android into android (base) and flutter - hard-code registry.oxen.rocks into the dockerfile stuff because that seems to be the only way to properly depend on other docker builds. - update a few CI builds that should have been using our images but weren't. - Update a few CI images to bullseye instead of buster Add openssh-client (for sftp to upload builds) --- .drone.jsonnet | 25 ++++++++++--------- .../docker/00-debian-bullseye-base.dockerfile | 3 +++ .../docker/00-debian-buster-base.dockerfile | 3 +++ .../ci/docker/00-debian-sid-base.dockerfile | 3 +++ .../docker/00-debian-stable-base.dockerfile | 1 + .../docker/00-debian-testing-base.dockerfile | 3 +++ .../ci/docker/10-debian-bullseye.dockerfile | 2 ++ contrib/ci/docker/10-debian-buster.dockerfile | 2 ++ contrib/ci/docker/10-debian-sid.dockerfile | 2 ++ contrib/ci/docker/10-debian-stable.dockerfile | 1 + ...dockerfile => 10-ubuntu-bionic.dockerfile} | 2 +- contrib/ci/docker/10-ubuntu-focal.dockerfile | 3 +++ contrib/ci/docker/50-android.dockerfile | 3 +++ contrib/ci/docker/android.dockerfile | 7 ------ contrib/ci/docker/debian-sid-clang.dockerfile | 2 ++ contrib/ci/docker/debian-sid.dockerfile | 3 --- contrib/ci/docker/debian-stable.dockerfile | 3 --- .../ci/docker/debian-win32-cross.dockerfile | 2 +- contrib/ci/docker/flutter.dockerfile | 2 ++ contrib/ci/docker/lint.dockerfile | 5 ++-- contrib/ci/docker/nodejs.dockerfile | 2 +- contrib/ci/docker/readme.md | 13 +++++++--- contrib/ci/docker/rebuild-docker-images.sh | 20 +++++++++------ contrib/ci/docker/ubuntu-focal.dockerfile | 3 --- 24 files changed, 70 insertions(+), 45 deletions(-) create mode 100644 contrib/ci/docker/00-debian-bullseye-base.dockerfile create mode 100644 contrib/ci/docker/00-debian-buster-base.dockerfile create mode 100644 contrib/ci/docker/00-debian-sid-base.dockerfile create mode 120000 contrib/ci/docker/00-debian-stable-base.dockerfile create mode 100644 contrib/ci/docker/00-debian-testing-base.dockerfile create mode 100644 contrib/ci/docker/10-debian-bullseye.dockerfile create mode 100644 contrib/ci/docker/10-debian-buster.dockerfile create mode 100644 contrib/ci/docker/10-debian-sid.dockerfile create mode 120000 contrib/ci/docker/10-debian-stable.dockerfile rename contrib/ci/docker/{ubuntu-bionic.dockerfile => 10-ubuntu-bionic.dockerfile} (67%) create mode 100644 contrib/ci/docker/10-ubuntu-focal.dockerfile create mode 100644 contrib/ci/docker/50-android.dockerfile delete mode 100644 contrib/ci/docker/android.dockerfile create mode 100644 contrib/ci/docker/debian-sid-clang.dockerfile delete mode 100644 contrib/ci/docker/debian-sid.dockerfile delete mode 100644 contrib/ci/docker/debian-stable.dockerfile create mode 100644 contrib/ci/docker/flutter.dockerfile delete mode 100644 contrib/ci/docker/ubuntu-focal.dockerfile diff --git a/.drone.jsonnet b/.drone.jsonnet index 02970c16b..588e538d9 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -193,14 +193,14 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = local clang(version) = debian_pipeline( 'Debian sid/clang-' + version + ' (amd64)', - docker_base + 'debian-sid', + docker_base + 'debian-sid-clang', deps=['clang-' + version] + default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' ' ); local full_llvm(version) = debian_pipeline( 'Debian sid/llvm-' + version + ' (amd64)', - 'debian:sid', + docker_base + 'debian-sid-clang', deps=['clang-' + version, ' lld-' + version, ' libc++-' + version + '-dev', 'libc++abi-' + version + '-dev'] + default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + @@ -249,7 +249,7 @@ local mac_builder(name, type: 'docker', steps: [{ name: 'build', - image: 'registry.oxen.rocks/lokinet-ci-lint', + image: docker_base + 'lint', commands: [ 'echo "Building on ${DRONE_STAGE_MACHINE}"', apt_get_quiet + ' update', @@ -261,21 +261,21 @@ local mac_builder(name, }, // Various debian builds - debian_pipeline('Debian sid (amd64)', 'debian:sid'), - debian_pipeline('Debian sid/Debug (amd64)', 'debian:sid', build_type='Debug'), + debian_pipeline('Debian sid (amd64)', docker_base + 'debian-sid'), + debian_pipeline('Debian sid/Debug (amd64)', docker_base + 'debian-sid', build_type='Debug'), clang(13), full_llvm(13), - debian_pipeline('Debian buster (i386)', 'i386/debian:buster', cmake_extra='-DDOWNLOAD_SODIUM=ON'), + debian_pipeline('Debian bullseye (i386)', 'i386/debian:bullseye', cmake_extra='-DDOWNLOAD_SODIUM=ON'), debian_pipeline('Ubuntu focal (amd64)', docker_base + 'ubuntu-focal'), debian_pipeline('Ubuntu bionic (amd64)', - 'ubuntu:bionic', + docker_base + 'ubuntu-bionic', deps=['g++-8'] + default_deps_nocxx, cmake_extra='-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8', loki_repo=true), // ARM builds (ARM64 and armhf) debian_pipeline('Debian sid (ARM64)', 'debian:sid', arch='arm64', jobs=4), - debian_pipeline('Debian buster (armhf)', 'arm32v7/debian:buster', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4), + debian_pipeline('Debian bullseye (armhf)', 'arm32v7/debian:bullseye', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4), // Static armhf build (gets uploaded) debian_pipeline('Static (buster armhf)', 'arm32v7/debian:buster', @@ -290,7 +290,7 @@ local mac_builder(name, ], jobs=4), // android apk builder - apk_builder('android apk', 'registry.oxen.rocks/lokinet-ci-android', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']), + apk_builder('android apk', docker_base + 'flutter', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']), // Windows builds (x64) windows_cross_pipeline('Windows (amd64)', @@ -321,9 +321,10 @@ local mac_builder(name, cmake_extra='-DWITH_HIVE=ON'), // Deb builds: - deb_builder('debian:sid', 'sid', 'debian/sid'), - deb_builder('debian:buster', 'buster', 'debian/buster'), - deb_builder('ubuntu:focal', 'focal', 'ubuntu/focal'), + deb_builder(docker_base + 'debian-sid', 'sid', 'debian/sid'), + deb_builder(docker_base + 'debian-bullseye', 'bullseye', 'debian/bullseye'), + deb_builder(docker_base + 'ubuntu-impish', 'impish', 'ubuntu/impish'), + deb_builder(docker_base + 'ubuntu-focal', 'focal', 'ubuntu/focal'), deb_builder('debian:sid', 'sid', 'debian/sid', arch='arm64'), // Macos builds: diff --git a/contrib/ci/docker/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/00-debian-bullseye-base.dockerfile new file mode 100644 index 000000000..ff4b77234 --- /dev/null +++ b/contrib/ci/docker/00-debian-bullseye-base.dockerfile @@ -0,0 +1,3 @@ +FROM debian:bullseye +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-buster-base.dockerfile b/contrib/ci/docker/00-debian-buster-base.dockerfile new file mode 100644 index 000000000..d2f2d0082 --- /dev/null +++ b/contrib/ci/docker/00-debian-buster-base.dockerfile @@ -0,0 +1,3 @@ +FROM debian:buster +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-sid-base.dockerfile b/contrib/ci/docker/00-debian-sid-base.dockerfile new file mode 100644 index 000000000..1e5e1e241 --- /dev/null +++ b/contrib/ci/docker/00-debian-sid-base.dockerfile @@ -0,0 +1,3 @@ +FROM debian:sid +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-stable-base.dockerfile b/contrib/ci/docker/00-debian-stable-base.dockerfile new file mode 120000 index 000000000..6b75d6395 --- /dev/null +++ b/contrib/ci/docker/00-debian-stable-base.dockerfile @@ -0,0 +1 @@ +00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/00-debian-testing-base.dockerfile b/contrib/ci/docker/00-debian-testing-base.dockerfile new file mode 100644 index 000000000..487bd376f --- /dev/null +++ b/contrib/ci/docker/00-debian-testing-base.dockerfile @@ -0,0 +1,3 @@ +FROM debian:testing +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile new file mode 100644 index 000000000..1d6be7072 --- /dev/null +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -0,0 +1,2 @@ +FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile new file mode 100644 index 000000000..ea711e5dd --- /dev/null +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -0,0 +1,2 @@ +FROM registry.oxen.rocks/lokinet-ci-debian-buster-base +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile new file mode 100644 index 000000000..5e0d08d1d --- /dev/null +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -0,0 +1,2 @@ +FROM registry.oxen.rocks/lokinet-ci-debian-sid-base +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile new file mode 120000 index 000000000..e37207772 --- /dev/null +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -0,0 +1 @@ +10-debian-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile similarity index 67% rename from contrib/ci/docker/ubuntu-bionic.dockerfile rename to contrib/ci/docker/10-ubuntu-bionic.dockerfile index b6bf4f09a..0c4937e71 100644 --- a/contrib/ci/docker/ubuntu-bionic.dockerfile +++ b/contrib/ci/docker/10-ubuntu-bionic.dockerfile @@ -1,3 +1,3 @@ FROM ubuntu:bionic RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client lsb-release libzmq3-dev libpgm-dev libuv1-dev openssh-client && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include' diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile new file mode 100644 index 000000000..dc726aa4a --- /dev/null +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -0,0 +1,3 @@ +FROM ubuntu:focal +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/50-android.dockerfile b/contrib/ci/docker/50-android.dockerfile new file mode 100644 index 000000000..e9f14c8b0 --- /dev/null +++ b/contrib/ci/docker/50-android.dockerfile @@ -0,0 +1,3 @@ +FROM registry.oxen.rocks/lokinet-ci-debian-testing-base +RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses' diff --git a/contrib/ci/docker/android.dockerfile b/contrib/ci/docker/android.dockerfile deleted file mode 100644 index 417735ae7..000000000 --- a/contrib/ci/docker/android.dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM debian:testing -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip' -RUN /bin/bash -c 'git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses' -RUN /bin/bash -c 'wget https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz -O /tmp/flutter.tar.xz && cd /opt && tar -xJvf /tmp/flutter.tar.xz && rm /tmp/flutter.tar.xz && ln -s /opt/flutter/bin/flutter /usr/local/bin/' -RUN /bin/bash -c 'flutter precache' \ No newline at end of file diff --git a/contrib/ci/docker/debian-sid-clang.dockerfile b/contrib/ci/docker/debian-sid-clang.dockerfile new file mode 100644 index 000000000..f113a1c0f --- /dev/null +++ b/contrib/ci/docker/debian-sid-clang.dockerfile @@ -0,0 +1,2 @@ +FROM registry.oxen.rocks/lokinet-ci-debian-sid +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y clang-13 lld-13 libc++-13-dev libc++abi-13-dev' diff --git a/contrib/ci/docker/debian-sid.dockerfile b/contrib/ci/docker/debian-sid.dockerfile deleted file mode 100644 index 9d45c5251..000000000 --- a/contrib/ci/docker/debian-sid.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM debian:sid -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache clang-11 libsodium-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev' diff --git a/contrib/ci/docker/debian-stable.dockerfile b/contrib/ci/docker/debian-stable.dockerfile deleted file mode 100644 index f5de1e581..000000000 --- a/contrib/ci/docker/debian-stable.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM debian:stable -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev' diff --git a/contrib/ci/docker/debian-win32-cross.dockerfile b/contrib/ci/docker/debian-win32-cross.dockerfile index 60bd1ffdf..aa4425848 100644 --- a/contrib/ci/docker/debian-win32-cross.dockerfile +++ b/contrib/ci/docker/debian-win32-cross.dockerfile @@ -1,4 +1,4 @@ FROM debian:testing RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client' RUN /bin/bash -c 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix' diff --git a/contrib/ci/docker/flutter.dockerfile b/contrib/ci/docker/flutter.dockerfile new file mode 100644 index 000000000..05a0b2580 --- /dev/null +++ b/contrib/ci/docker/flutter.dockerfile @@ -0,0 +1,2 @@ +FROM registry.oxen.rocks/lokinet-ci-android +RUN /bin/bash -c 'cd /opt && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz | tar xJv && ln -s /opt/flutter/bin/flutter /usr/local/bin/ && flutter precache' diff --git a/contrib/ci/docker/lint.dockerfile b/contrib/ci/docker/lint.dockerfile index f247fd6a7..d7310e8aa 100644 --- a/contrib/ci/docker/lint.dockerfile +++ b/contrib/ci/docker/lint.dockerfile @@ -1,3 +1,2 @@ -FROM debian:sid -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata git clang-format-11' +FROM registry.oxen.rocks/lokinet-ci-debian-sid-base +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata git clang-format-11 jsonnet' diff --git a/contrib/ci/docker/nodejs.dockerfile b/contrib/ci/docker/nodejs.dockerfile index 059d16a6f..399a59599 100644 --- a/contrib/ci/docker/nodejs.dockerfile +++ b/contrib/ci/docker/nodejs.dockerfile @@ -1,3 +1,3 @@ FROM node:14.16.1 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ wine' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ wine openssh-client' diff --git a/contrib/ci/docker/readme.md b/contrib/ci/docker/readme.md index 7b3cfbe87..767286b61 100644 --- a/contrib/ci/docker/readme.md +++ b/contrib/ci/docker/readme.md @@ -1,8 +1,13 @@ ## drone-ci docker jizz -To rebuild all ci images and push them to a registry server do: +To rebuild all ci images and push them to the oxen registry server do: - $ docker login your.registry.here - $ ./rebuild-docker-images.sh your.registry.here *.dockerfile + $ docker login registry.oxen.rocks + $ ./rebuild-docker-images.sh *.dockerfile -The docker images will be `your.registry.here/lokinet-ci-*`for each *.dockerfile in this directory +If you aren't part of the Oxen team, you'll likely need to set up your own registry and change +registry.oxen.rocks to your own domain name in order to do anything useful with this. + +The docker images will be `registry.oxen.rocks/lokinet-ci-*`for each \*.dockerfile in this +directory, with the leading numeric `NN-` removed, if present (so that you can ensure proper +ordering using two-digit numeric prefixes). diff --git a/contrib/ci/docker/rebuild-docker-images.sh b/contrib/ci/docker/rebuild-docker-images.sh index 959643cef..a95d326bf 100755 --- a/contrib/ci/docker/rebuild-docker-images.sh +++ b/contrib/ci/docker/rebuild-docker-images.sh @@ -1,13 +1,19 @@ #!/bin/bash -# the registry server to use -registry=$1 +set -o errexit -test "x$registry" != "x" || exit 1 +registry=registry.oxen.rocks -for file in ${@:2} ; do - name="$(echo $file | cut -d'.' -f1)" - echo "rebuild $name" - docker build -f $file -t $registry/lokinet-ci-$name . +if [[ $# -eq 0 ]]; then + files=(*.dockerfile) +else + files=("$@") +fi + +for file in "${files[@]}"; do + name="${file#[0-9][0-9]-}" # s/^\d\d-// + name="${name%.dockerfile}" # s/\.dockerfile$// + echo -e "\e[32;1mrebuilding $name\e[0m" + docker build --pull -f $file -t $registry/lokinet-ci-$name . docker push $registry/lokinet-ci-$name done diff --git a/contrib/ci/docker/ubuntu-focal.dockerfile b/contrib/ci/docker/ubuntu-focal.dockerfile deleted file mode 100644 index 5df8d737e..000000000 --- a/contrib/ci/docker/ubuntu-focal.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM ubuntu:focal -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev' From 1dccbb58140fc6e29e59db70833bcd6ba08c46a8 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sat, 16 Oct 2021 16:00:27 -0300 Subject: [PATCH 11/25] Update deb repo dns name --- .drone.jsonnet | 8 ++++---- contrib/{deb.loki.network.gpg => deb.oxen.io.gpg} | Bin 2 files changed, 4 insertions(+), 4 deletions(-) rename contrib/{deb.loki.network.gpg => deb.oxen.io.gpg} (100%) diff --git a/.drone.jsonnet b/.drone.jsonnet index 588e538d9..5e31427e1 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -58,8 +58,8 @@ local debian_pipeline(name, ] + ( if loki_repo then [ 'eatmydata ' + apt_get_quiet + ' install -y lsb-release', - 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', - 'echo deb http://deb.loki.network $$(lsb_release -sc) main >/etc/apt/sources.list.d/loki.network.list', + 'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d', + 'echo deb http://deb.oxen.io $$(lsb_release -sc) main >/etc/apt/sources.list.d/oxen.list', 'eatmydata ' + apt_get_quiet + ' update', ] else [] ) + [ @@ -160,8 +160,8 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = 'echo "Building on ${DRONE_STAGE_MACHINE}"', 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', ] + (if loki_repo then [ - 'cp contrib/deb.loki.network.gpg /etc/apt/trusted.gpg.d', - 'echo deb http://deb.loki.network $${distro} main >/etc/apt/sources.list.d/loki.network.list', + 'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d', + 'echo deb http://deb.oxen.io $${distro} main >/etc/apt/sources.list.d/oxen.list', ] else []) + [ apt_get_quiet + ' update', apt_get_quiet + ' install -y eatmydata', diff --git a/contrib/deb.loki.network.gpg b/contrib/deb.oxen.io.gpg similarity index 100% rename from contrib/deb.loki.network.gpg rename to contrib/deb.oxen.io.gpg From 47d8c95e6830b7845ff091a807b51cb577c12ea9 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 14:46:30 -0300 Subject: [PATCH 12/25] CI docker multiarch Adds multiarch docker images and starts using them. --- .drone.jsonnet | 16 +++++---- .../docker/00-debian-bullseye-base.dockerfile | 3 +- .../docker/00-debian-buster-base.dockerfile | 3 +- .../ci/docker/00-debian-sid-base.dockerfile | 3 +- .../docker/00-debian-stable-base.dockerfile | 5 ++- .../docker/00-debian-testing-base.dockerfile | 3 +- .../ci/docker/10-debian-bullseye.dockerfile | 3 +- contrib/ci/docker/10-debian-buster.dockerfile | 3 +- contrib/ci/docker/10-debian-sid.dockerfile | 3 +- contrib/ci/docker/10-debian-stable.dockerfile | 4 ++- contrib/ci/docker/10-ubuntu-bionic.dockerfile | 3 +- contrib/ci/docker/10-ubuntu-focal.dockerfile | 3 +- contrib/ci/docker/10-ubuntu-impish.dockerfile | 4 +++ contrib/ci/docker/10-ubuntu-lts.dockerfile | 4 +++ .../ci/docker/10-ubuntu-rolling.dockerfile | 4 +++ contrib/ci/docker/50-android.dockerfile | 3 +- .../arm32v7/00-debian-buster-base.dockerfile | 1 + .../arm32v7/00-debian-stable-base.dockerfile | 1 + .../arm32v7/10-debian-buster.dockerfile | 1 + .../arm32v7/10-debian-stable.dockerfile | 1 + .../arm64v8/00-debian-sid-base.dockerfile | 1 + .../docker/arm64v8/10-debian-sid.dockerfile | 1 + contrib/ci/docker/debian-sid-clang.dockerfile | 3 +- .../ci/docker/debian-win32-cross.dockerfile | 3 +- contrib/ci/docker/flutter.dockerfile | 3 +- .../i386/00-debian-stable-base.dockerfile | 1 + .../docker/i386/10-debian-stable.dockerfile | 1 + contrib/ci/docker/lint.dockerfile | 3 +- contrib/ci/docker/readme.md | 2 +- contrib/ci/docker/rebuild-docker-images.sh | 33 ++++++++++++++++--- 30 files changed, 93 insertions(+), 29 deletions(-) mode change 120000 => 100644 contrib/ci/docker/00-debian-stable-base.dockerfile mode change 120000 => 100644 contrib/ci/docker/10-debian-stable.dockerfile create mode 100644 contrib/ci/docker/10-ubuntu-impish.dockerfile create mode 100644 contrib/ci/docker/10-ubuntu-lts.dockerfile create mode 100644 contrib/ci/docker/10-ubuntu-rolling.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile create mode 120000 contrib/ci/docker/arm32v7/10-debian-buster.dockerfile create mode 120000 contrib/ci/docker/arm32v7/10-debian-stable.dockerfile create mode 120000 contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile create mode 120000 contrib/ci/docker/arm64v8/10-debian-sid.dockerfile create mode 120000 contrib/ci/docker/i386/00-debian-stable-base.dockerfile create mode 120000 contrib/ci/docker/i386/10-debian-stable.dockerfile diff --git a/.drone.jsonnet b/.drone.jsonnet index 5e31427e1..dec90b2bc 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -265,8 +265,10 @@ local mac_builder(name, debian_pipeline('Debian sid/Debug (amd64)', docker_base + 'debian-sid', build_type='Debug'), clang(13), full_llvm(13), - debian_pipeline('Debian bullseye (i386)', 'i386/debian:bullseye', cmake_extra='-DDOWNLOAD_SODIUM=ON'), - debian_pipeline('Ubuntu focal (amd64)', docker_base + 'ubuntu-focal'), + debian_pipeline('Debian stable (i386)', docker_base + 'debian-stable:i386'), + debian_pipeline('Debian buster (amd64)', docker_base + 'debian-buster', cmake_extra='-DDOWNLOAD_SODIUM=ON'), + debian_pipeline('Ubuntu latest (amd64)', docker_base + 'ubuntu-rolling'), + debian_pipeline('Ubuntu LTS (amd64)', docker_base + 'ubuntu-lts'), debian_pipeline('Ubuntu bionic (amd64)', docker_base + 'ubuntu-bionic', deps=['g++-8'] + default_deps_nocxx, @@ -274,11 +276,11 @@ local mac_builder(name, loki_repo=true), // ARM builds (ARM64 and armhf) - debian_pipeline('Debian sid (ARM64)', 'debian:sid', arch='arm64', jobs=4), - debian_pipeline('Debian bullseye (armhf)', 'arm32v7/debian:bullseye', arch='arm64', cmake_extra='-DDOWNLOAD_SODIUM=ON', jobs=4), + debian_pipeline('Debian sid (ARM64)', docker_base + 'debian-sid', arch='arm64', jobs=4), + debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable:arm32v7', arch='arm64', jobs=4), // Static armhf build (gets uploaded) debian_pipeline('Static (buster armhf)', - 'arm32v7/debian:buster', + docker_base + 'debian-buster:arm32v7', arch='arm64', deps=['g++', 'python3-dev', 'automake', 'libtool'], cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + @@ -316,7 +318,7 @@ local mac_builder(name, // integration tests debian_pipeline('Router Hive', - 'ubuntu:focal', + docker_base + 'ubuntu-lts', deps=['python3-dev', 'python3-pytest', 'python3-pybind11'] + default_deps, cmake_extra='-DWITH_HIVE=ON'), @@ -325,7 +327,7 @@ local mac_builder(name, deb_builder(docker_base + 'debian-bullseye', 'bullseye', 'debian/bullseye'), deb_builder(docker_base + 'ubuntu-impish', 'impish', 'ubuntu/impish'), deb_builder(docker_base + 'ubuntu-focal', 'focal', 'ubuntu/focal'), - deb_builder('debian:sid', 'sid', 'debian/sid', arch='arm64'), + deb_builder(docker_base + 'debian-sid', 'sid', 'debian/sid', arch='arm64'), // Macos builds: mac_builder('macOS (Release)'), diff --git a/contrib/ci/docker/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/00-debian-bullseye-base.dockerfile index ff4b77234..120c55c40 100644 --- a/contrib/ci/docker/00-debian-bullseye-base.dockerfile +++ b/contrib/ci/docker/00-debian-bullseye-base.dockerfile @@ -1,3 +1,4 @@ -FROM debian:bullseye +ARG ARCH=amd64 +FROM ${ARCH}/debian:bullseye RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-buster-base.dockerfile b/contrib/ci/docker/00-debian-buster-base.dockerfile index d2f2d0082..3a36cda42 100644 --- a/contrib/ci/docker/00-debian-buster-base.dockerfile +++ b/contrib/ci/docker/00-debian-buster-base.dockerfile @@ -1,3 +1,4 @@ -FROM debian:buster +ARG ARCH=amd64 +FROM ${ARCH}/debian:buster RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-sid-base.dockerfile b/contrib/ci/docker/00-debian-sid-base.dockerfile index 1e5e1e241..0ae5432ff 100644 --- a/contrib/ci/docker/00-debian-sid-base.dockerfile +++ b/contrib/ci/docker/00-debian-sid-base.dockerfile @@ -1,3 +1,4 @@ -FROM debian:sid +ARG ARCH=amd64 +FROM ${ARCH}/debian:sid RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-stable-base.dockerfile b/contrib/ci/docker/00-debian-stable-base.dockerfile deleted file mode 120000 index 6b75d6395..000000000 --- a/contrib/ci/docker/00-debian-stable-base.dockerfile +++ /dev/null @@ -1 +0,0 @@ -00-debian-bullseye-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/00-debian-stable-base.dockerfile b/contrib/ci/docker/00-debian-stable-base.dockerfile new file mode 100644 index 000000000..1313788ce --- /dev/null +++ b/contrib/ci/docker/00-debian-stable-base.dockerfile @@ -0,0 +1,4 @@ +ARG ARCH=amd64 +FROM ${ARCH}/debian:stable +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/00-debian-testing-base.dockerfile b/contrib/ci/docker/00-debian-testing-base.dockerfile index 487bd376f..047ee08c7 100644 --- a/contrib/ci/docker/00-debian-testing-base.dockerfile +++ b/contrib/ci/docker/00-debian-testing-base.dockerfile @@ -1,3 +1,4 @@ -FROM debian:testing +ARG ARCH=amd64 +FROM ${ARCH}/debian:testing RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile index 1d6be7072..07d2181b8 100644 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -1,2 +1,3 @@ -FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index ea711e5dd..27d33e88b 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -1,2 +1,3 @@ -FROM registry.oxen.rocks/lokinet-ci-debian-buster-base +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile index 5e0d08d1d..f09bdebd0 100644 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -1,2 +1,3 @@ -FROM registry.oxen.rocks/lokinet-ci-debian-sid-base +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile deleted file mode 120000 index e37207772..000000000 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ /dev/null @@ -1 +0,0 @@ -10-debian-bullseye.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile new file mode 100644 index 000000000..17c80be39 --- /dev/null +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -0,0 +1,3 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH} +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile index 0c4937e71..61a7eaff4 100644 --- a/contrib/ci/docker/10-ubuntu-bionic.dockerfile +++ b/contrib/ci/docker/10-ubuntu-bionic.dockerfile @@ -1,3 +1,4 @@ -FROM ubuntu:bionic +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:bionic RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client lsb-release libzmq3-dev libpgm-dev libuv1-dev openssh-client && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include' diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile index dc726aa4a..6ae238dc5 100644 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -1,3 +1,4 @@ -FROM ubuntu:focal +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:focal RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile new file mode 100644 index 000000000..e7dcc77f3 --- /dev/null +++ b/contrib/ci/docker/10-ubuntu-impish.dockerfile @@ -0,0 +1,4 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:impish +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile new file mode 100644 index 000000000..8e73db27e --- /dev/null +++ b/contrib/ci/docker/10-ubuntu-lts.dockerfile @@ -0,0 +1,4 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:latest +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile new file mode 100644 index 000000000..175961320 --- /dev/null +++ b/contrib/ci/docker/10-ubuntu-rolling.dockerfile @@ -0,0 +1,4 @@ +ARG ARCH=amd64 +FROM ${ARCH}/ubuntu:rolling +RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/50-android.dockerfile b/contrib/ci/docker/50-android.dockerfile index e9f14c8b0..1bed19fc5 100644 --- a/contrib/ci/docker/50-android.dockerfile +++ b/contrib/ci/docker/50-android.dockerfile @@ -1,3 +1,4 @@ -FROM registry.oxen.rocks/lokinet-ci-debian-testing-base +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses' diff --git a/contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile new file mode 120000 index 000000000..e4ab00108 --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-debian-buster-base.dockerfile @@ -0,0 +1 @@ +../00-debian-buster-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile b/contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile new file mode 120000 index 000000000..2d62f76d4 --- /dev/null +++ b/contrib/ci/docker/arm32v7/00-debian-stable-base.dockerfile @@ -0,0 +1 @@ +../00-debian-stable-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/10-debian-buster.dockerfile b/contrib/ci/docker/arm32v7/10-debian-buster.dockerfile new file mode 120000 index 000000000..592305776 --- /dev/null +++ b/contrib/ci/docker/arm32v7/10-debian-buster.dockerfile @@ -0,0 +1 @@ +../10-debian-buster.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm32v7/10-debian-stable.dockerfile b/contrib/ci/docker/arm32v7/10-debian-stable.dockerfile new file mode 120000 index 000000000..a10f6b3c5 --- /dev/null +++ b/contrib/ci/docker/arm32v7/10-debian-stable.dockerfile @@ -0,0 +1 @@ +../10-debian-stable.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile b/contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile new file mode 120000 index 000000000..3fdec500d --- /dev/null +++ b/contrib/ci/docker/arm64v8/00-debian-sid-base.dockerfile @@ -0,0 +1 @@ +../00-debian-sid-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/arm64v8/10-debian-sid.dockerfile b/contrib/ci/docker/arm64v8/10-debian-sid.dockerfile new file mode 120000 index 000000000..baf7ccaad --- /dev/null +++ b/contrib/ci/docker/arm64v8/10-debian-sid.dockerfile @@ -0,0 +1 @@ +../10-debian-sid.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/debian-sid-clang.dockerfile b/contrib/ci/docker/debian-sid-clang.dockerfile index f113a1c0f..7dc8e5cc1 100644 --- a/contrib/ci/docker/debian-sid-clang.dockerfile +++ b/contrib/ci/docker/debian-sid-clang.dockerfile @@ -1,2 +1,3 @@ -FROM registry.oxen.rocks/lokinet-ci-debian-sid +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-sid/${ARCH} RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y clang-13 lld-13 libc++-13-dev libc++abi-13-dev' diff --git a/contrib/ci/docker/debian-win32-cross.dockerfile b/contrib/ci/docker/debian-win32-cross.dockerfile index aa4425848..ba12ca8c0 100644 --- a/contrib/ci/docker/debian-win32-cross.dockerfile +++ b/contrib/ci/docker/debian-win32-cross.dockerfile @@ -1,4 +1,5 @@ -FROM debian:testing +ARG ARCH=amd64 +FROM ${ARCH}/debian:testing RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client' RUN /bin/bash -c 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix' diff --git a/contrib/ci/docker/flutter.dockerfile b/contrib/ci/docker/flutter.dockerfile index 05a0b2580..bf549edd1 100644 --- a/contrib/ci/docker/flutter.dockerfile +++ b/contrib/ci/docker/flutter.dockerfile @@ -1,2 +1,3 @@ -FROM registry.oxen.rocks/lokinet-ci-android +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-android/${ARCH} RUN /bin/bash -c 'cd /opt && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz | tar xJv && ln -s /opt/flutter/bin/flutter /usr/local/bin/ && flutter precache' diff --git a/contrib/ci/docker/i386/00-debian-stable-base.dockerfile b/contrib/ci/docker/i386/00-debian-stable-base.dockerfile new file mode 120000 index 000000000..2d62f76d4 --- /dev/null +++ b/contrib/ci/docker/i386/00-debian-stable-base.dockerfile @@ -0,0 +1 @@ +../00-debian-stable-base.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/i386/10-debian-stable.dockerfile b/contrib/ci/docker/i386/10-debian-stable.dockerfile new file mode 120000 index 000000000..a10f6b3c5 --- /dev/null +++ b/contrib/ci/docker/i386/10-debian-stable.dockerfile @@ -0,0 +1 @@ +../10-debian-stable.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/lint.dockerfile b/contrib/ci/docker/lint.dockerfile index d7310e8aa..62f97940a 100644 --- a/contrib/ci/docker/lint.dockerfile +++ b/contrib/ci/docker/lint.dockerfile @@ -1,2 +1,3 @@ -FROM registry.oxen.rocks/lokinet-ci-debian-sid-base +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata git clang-format-11 jsonnet' diff --git a/contrib/ci/docker/readme.md b/contrib/ci/docker/readme.md index 767286b61..5cd271f2d 100644 --- a/contrib/ci/docker/readme.md +++ b/contrib/ci/docker/readme.md @@ -3,7 +3,7 @@ To rebuild all ci images and push them to the oxen registry server do: $ docker login registry.oxen.rocks - $ ./rebuild-docker-images.sh *.dockerfile + $ ./rebuild-docker-images.sh If you aren't part of the Oxen team, you'll likely need to set up your own registry and change registry.oxen.rocks to your own domain name in order to do anything useful with this. diff --git a/contrib/ci/docker/rebuild-docker-images.sh b/contrib/ci/docker/rebuild-docker-images.sh index a95d326bf..6a3b97684 100755 --- a/contrib/ci/docker/rebuild-docker-images.sh +++ b/contrib/ci/docker/rebuild-docker-images.sh @@ -2,18 +2,41 @@ set -o errexit +trap 'echo -e "\n\n\n\e[31;1mAn error occurred!\e[1m\n\n"' ERR + registry=registry.oxen.rocks if [[ $# -eq 0 ]]; then - files=(*.dockerfile) + files=(*.dockerfile i386/*.dockerfile arm64v8/*.dockerfile arm32v7/*.dockerfile) else files=("$@") fi +declare -A manifests + for file in "${files[@]}"; do - name="${file#[0-9][0-9]-}" # s/^\d\d-// + if [[ "$file" == */* ]]; then + arch="${file%%/*}" + name="${file#*/}" + else + arch="amd64" + name="$file" + fi + + name="${name#[0-9][0-9]-}" # s/^\d\d-// name="${name%.dockerfile}" # s/\.dockerfile$// - echo -e "\e[32;1mrebuilding $name\e[0m" - docker build --pull -f $file -t $registry/lokinet-ci-$name . - docker push $registry/lokinet-ci-$name + namearch=$registry/lokinet-ci-$name/$arch + latest=$registry/lokinet-ci-$name:latest + echo -e "\e[32;1mrebuilding \e[35;1m$namearch\e[0m" + docker build --pull -f $file -t $namearch --build-arg ARCH=$arch . + docker push $namearch + + manifests[$latest]="${manifests[$latest]} $namearch" +done + +for latest in "${!manifests[@]}"; do + echo -e "\e[32;1mpushing new manifest for \e[33;1m$latest[\e[35;1m${manifests[$latest]} \e[33;1m]\e[0m" + docker manifest rm $latest 2>/dev/null || true + docker manifest create $latest ${manifests[$latest]} + docker manifest push $latest done From 326670b95941d00e7d865f721117b3ec5a78344b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 14:51:31 -0300 Subject: [PATCH 13/25] Docker package updates: - add `make` - add `patch` --- contrib/ci/docker/10-debian-bullseye.dockerfile | 2 +- contrib/ci/docker/10-debian-buster.dockerfile | 2 +- contrib/ci/docker/10-debian-sid.dockerfile | 2 +- contrib/ci/docker/10-debian-stable.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-focal.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-impish.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-lts.dockerfile | 2 +- contrib/ci/docker/10-ubuntu-rolling.dockerfile | 2 +- contrib/ci/docker/50-android.dockerfile | 2 +- contrib/ci/docker/debian-win32-cross.dockerfile | 5 ++--- contrib/ci/docker/nodejs.dockerfile | 2 +- 11 files changed, 12 insertions(+), 13 deletions(-) diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile index 07d2181b8..f1d93de55 100644 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -1,3 +1,3 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index 27d33e88b..21c3fa60b 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -1,3 +1,3 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile index f09bdebd0..3de3dd88d 100644 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -1,3 +1,3 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile index 17c80be39..0e2d3bf2c 100644 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -1,3 +1,3 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile index 6ae238dc5..20c2bd0d5 100644 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -1,4 +1,4 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:focal RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile index e7dcc77f3..e4bbc81cb 100644 --- a/contrib/ci/docker/10-ubuntu-impish.dockerfile +++ b/contrib/ci/docker/10-ubuntu-impish.dockerfile @@ -1,4 +1,4 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:impish RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile index 8e73db27e..69b52a8cc 100644 --- a/contrib/ci/docker/10-ubuntu-lts.dockerfile +++ b/contrib/ci/docker/10-ubuntu-lts.dockerfile @@ -1,4 +1,4 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:latest RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile index 175961320..7fefffc5e 100644 --- a/contrib/ci/docker/10-ubuntu-rolling.dockerfile +++ b/contrib/ci/docker/10-ubuntu-rolling.dockerfile @@ -1,4 +1,4 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:rolling RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' diff --git a/contrib/ci/docker/50-android.dockerfile b/contrib/ci/docker/50-android.dockerfile index 1bed19fc5..c6650a5f4 100644 --- a/contrib/ci/docker/50-android.dockerfile +++ b/contrib/ci/docker/50-android.dockerfile @@ -1,4 +1,4 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake make patch ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses' diff --git a/contrib/ci/docker/debian-win32-cross.dockerfile b/contrib/ci/docker/debian-win32-cross.dockerfile index ba12ca8c0..b5eedbbad 100644 --- a/contrib/ci/docker/debian-win32-cross.dockerfile +++ b/contrib/ci/docker/debian-win32-cross.dockerfile @@ -1,5 +1,4 @@ ARG ARCH=amd64 -FROM ${ARCH}/debian:testing -RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client' +FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client' RUN /bin/bash -c 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix' diff --git a/contrib/ci/docker/nodejs.dockerfile b/contrib/ci/docker/nodejs.dockerfile index 399a59599..48264e3ce 100644 --- a/contrib/ci/docker/nodejs.dockerfile +++ b/contrib/ci/docker/nodejs.dockerfile @@ -1,3 +1,3 @@ FROM node:14.16.1 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake git ninja-build pkg-config ccache g++ wine openssh-client' +RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ wine openssh-client' From d64e2ae0afa01e90f01db4e67e6a76cbf44bc1d7 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 15:13:43 -0300 Subject: [PATCH 14/25] Reformat RUN commands to make editing easier - split them on multiple lines - sort the installed package deps - don't use extra unnecessary `bash` invocation --- .../docker/00-debian-bullseye-base.dockerfile | 4 +- .../docker/00-debian-buster-base.dockerfile | 4 +- .../ci/docker/00-debian-sid-base.dockerfile | 4 +- .../docker/00-debian-stable-base.dockerfile | 4 +- .../docker/00-debian-testing-base.dockerfile | 4 +- .../ci/docker/10-debian-bullseye.dockerfile | 35 +++++++++++++++++- contrib/ci/docker/10-debian-buster.dockerfile | 35 +++++++++++++++++- contrib/ci/docker/10-debian-sid.dockerfile | 35 +++++++++++++++++- contrib/ci/docker/10-debian-stable.dockerfile | 35 +++++++++++++++++- contrib/ci/docker/10-ubuntu-bionic.dockerfile | 28 +++++++++++++- contrib/ci/docker/10-ubuntu-focal.dockerfile | 37 ++++++++++++++++++- contrib/ci/docker/10-ubuntu-impish.dockerfile | 37 ++++++++++++++++++- contrib/ci/docker/10-ubuntu-lts.dockerfile | 37 ++++++++++++++++++- .../ci/docker/10-ubuntu-rolling.dockerfile | 37 ++++++++++++++++++- contrib/ci/docker/50-android.dockerfile | 21 ++++++++++- contrib/ci/docker/debian-sid-clang.dockerfile | 6 ++- .../ci/docker/debian-win32-cross.dockerfile | 24 +++++++++++- contrib/ci/docker/flutter.dockerfile | 6 ++- contrib/ci/docker/lint.dockerfile | 6 ++- contrib/ci/docker/nodejs.dockerfile | 16 +++++++- 20 files changed, 394 insertions(+), 21 deletions(-) diff --git a/contrib/ci/docker/00-debian-bullseye-base.dockerfile b/contrib/ci/docker/00-debian-bullseye-base.dockerfile index 120c55c40..4d691d074 100644 --- a/contrib/ci/docker/00-debian-bullseye-base.dockerfile +++ b/contrib/ci/docker/00-debian-bullseye-base.dockerfile @@ -1,4 +1,6 @@ ARG ARCH=amd64 FROM ${ARCH}/debian:bullseye RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-buster-base.dockerfile b/contrib/ci/docker/00-debian-buster-base.dockerfile index 3a36cda42..43c6ca4be 100644 --- a/contrib/ci/docker/00-debian-buster-base.dockerfile +++ b/contrib/ci/docker/00-debian-buster-base.dockerfile @@ -1,4 +1,6 @@ ARG ARCH=amd64 FROM ${ARCH}/debian:buster RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-sid-base.dockerfile b/contrib/ci/docker/00-debian-sid-base.dockerfile index 0ae5432ff..9293b6e9e 100644 --- a/contrib/ci/docker/00-debian-sid-base.dockerfile +++ b/contrib/ci/docker/00-debian-sid-base.dockerfile @@ -1,4 +1,6 @@ ARG ARCH=amd64 FROM ${ARCH}/debian:sid RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-stable-base.dockerfile b/contrib/ci/docker/00-debian-stable-base.dockerfile index 1313788ce..62eedbd67 100644 --- a/contrib/ci/docker/00-debian-stable-base.dockerfile +++ b/contrib/ci/docker/00-debian-stable-base.dockerfile @@ -1,4 +1,6 @@ ARG ARCH=amd64 FROM ${ARCH}/debian:stable RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/00-debian-testing-base.dockerfile b/contrib/ci/docker/00-debian-testing-base.dockerfile index 047ee08c7..714e24542 100644 --- a/contrib/ci/docker/00-debian-testing-base.dockerfile +++ b/contrib/ci/docker/00-debian-testing-base.dockerfile @@ -1,4 +1,6 @@ ARG ARCH=amd64 FROM ${ARCH}/debian:testing RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q autoremove -y diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile index f1d93de55..c061cac9e 100644 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -1,3 +1,36 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index 21c3fa60b..b9cd4e6f6 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -1,3 +1,36 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile index 3de3dd88d..32ac1d46f 100644 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -1,3 +1,36 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile index 0e2d3bf2c..585de0a17 100644 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -1,3 +1,36 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile index 61a7eaff4..9097406e9 100644 --- a/contrib/ci/docker/10-ubuntu-bionic.dockerfile +++ b/contrib/ci/docker/10-ubuntu-bionic.dockerfile @@ -1,4 +1,30 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:bionic RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y eatmydata gdb cmake git ninja-build pkg-config ccache g++-8 python3-dev automake libtool autoconf make qttools5-dev file gperf patch openssh-client lsb-release libzmq3-dev libpgm-dev libuv1-dev openssh-client && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q --no-install-recommends install -y \ + autoconf \ + automake \ + ccache \ + cmake \ + eatmydata \ + file \ + g++-8 \ + gdb \ + git \ + gperf \ + libpgm-dev \ + libtool \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + ninja-build \ + openssh-client \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev \ + && mkdir -p /usr/lib/x86_64-linux-gnu/pgm-5.2/include diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile index 20c2bd0d5..aee953456 100644 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -1,4 +1,39 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:focal RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile index e4bbc81cb..472d12f50 100644 --- a/contrib/ci/docker/10-ubuntu-impish.dockerfile +++ b/contrib/ci/docker/10-ubuntu-impish.dockerfile @@ -1,4 +1,39 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:impish RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile index 69b52a8cc..f2763cfa1 100644 --- a/contrib/ci/docker/10-ubuntu-lts.dockerfile +++ b/contrib/ci/docker/10-ubuntu-lts.dockerfile @@ -1,4 +1,39 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:latest RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile index 7fefffc5e..de4401479 100644 --- a/contrib/ci/docker/10-ubuntu-rolling.dockerfile +++ b/contrib/ci/docker/10-ubuntu-rolling.dockerfile @@ -1,4 +1,39 @@ ARG ARCH=amd64 FROM ${ARCH}/ubuntu:rolling RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ libsodium-dev libzmq3-dev libsystemd-dev python3-dev libuv1-dev libunbound-dev nettle-dev libssl-dev libevent-dev libsqlite3-dev libboost-thread-dev libboost-serialization-dev libboost-program-options-dev libgtest-dev libminiupnpc-dev libunwind8-dev libreadline-dev libhidapi-dev libusb-1.0.0-dev qttools5-dev libcurl4-openssl-dev lsb-release openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + libboost-program-options-dev \ + libboost-serialization-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + libevent-dev \ + libgtest-dev \ + libhidapi-dev \ + libminiupnpc-dev \ + libreadline-dev \ + libsodium-dev \ + libsqlite3-dev \ + libssl-dev \ + libsystemd-dev \ + libunbound-dev \ + libunwind8-dev \ + libusb-1.0.0-dev \ + libuv1-dev \ + libzmq3-dev \ + lsb-release \ + make \ + nettle-dev \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + python3-dev \ + qttools5-dev diff --git a/contrib/ci/docker/50-android.dockerfile b/contrib/ci/docker/50-android.dockerfile index c6650a5f4..04163fc63 100644 --- a/contrib/ci/docker/50-android.dockerfile +++ b/contrib/ci/docker/50-android.dockerfile @@ -1,4 +1,23 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} RUN /bin/bash -c 'sed -i "s/main/main contrib/g" /etc/apt/sources.list' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y android-sdk google-android-ndk-installer wget git pkg-config automake libtool cmake make patch ccache curl zip xz-utils openssh-client && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses && rm -rf /tmp/android-sdk-licenses' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + android-sdk \ + automake \ + ccache \ + cmake \ + curl \ + git \ + google-android-ndk-installer \ + libtool \ + make \ + openssh-client \ + patch \ + pkg-config \ + wget \ + xz-utils \ + zip \ + && git clone https://github.com/Shadowstyler/android-sdk-licenses.git /tmp/android-sdk-licenses \ + && cp -a /tmp/android-sdk-licenses/*-license /usr/lib/android-sdk/licenses \ + && rm -rf /tmp/android-sdk-licenses diff --git a/contrib/ci/docker/debian-sid-clang.dockerfile b/contrib/ci/docker/debian-sid-clang.dockerfile index 7dc8e5cc1..6d21df7fe 100644 --- a/contrib/ci/docker/debian-sid-clang.dockerfile +++ b/contrib/ci/docker/debian-sid-clang.dockerfile @@ -1,3 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-sid/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y clang-13 lld-13 libc++-13-dev libc++abi-13-dev' +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + clang-13 \ + libc++-13-dev \ + libc++abi-13-dev \ + lld-13 diff --git a/contrib/ci/docker/debian-win32-cross.dockerfile b/contrib/ci/docker/debian-win32-cross.dockerfile index b5eedbbad..f2d852644 100644 --- a/contrib/ci/docker/debian-win32-cross.dockerfile +++ b/contrib/ci/docker/debian-win32-cross.dockerfile @@ -1,4 +1,24 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-testing-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata build-essential cmake git ninja-build pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool autoconf make qttools5-dev file gperf patch openssh-client' -RUN /bin/bash -c 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix' +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + autoconf \ + automake \ + build-essential \ + ccache \ + cmake \ + eatmydata \ + file \ + g++-mingw-w64-x86-64-posix \ + git \ + gperf \ + libtool \ + make \ + ninja-build \ + nsis \ + openssh-client \ + patch \ + pkg-config \ + qttools5-dev \ + zip \ + && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix \ + && update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix diff --git a/contrib/ci/docker/flutter.dockerfile b/contrib/ci/docker/flutter.dockerfile index bf549edd1..17b39a980 100644 --- a/contrib/ci/docker/flutter.dockerfile +++ b/contrib/ci/docker/flutter.dockerfile @@ -1,3 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-android/${ARCH} -RUN /bin/bash -c 'cd /opt && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz | tar xJv && ln -s /opt/flutter/bin/flutter /usr/local/bin/ && flutter precache' +RUN cd /opt \ + && curl https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_2.2.2-stable.tar.xz \ + | tar xJv \ + && ln -s /opt/flutter/bin/flutter /usr/local/bin/ \ + && flutter precache diff --git a/contrib/ci/docker/lint.dockerfile b/contrib/ci/docker/lint.dockerfile index 62f97940a..d9636827d 100644 --- a/contrib/ci/docker/lint.dockerfile +++ b/contrib/ci/docker/lint.dockerfile @@ -1,3 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata git clang-format-11 jsonnet' +RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + clang-format-11 \ + eatmydata \ + git \ + jsonnet diff --git a/contrib/ci/docker/nodejs.dockerfile b/contrib/ci/docker/nodejs.dockerfile index 48264e3ce..625251173 100644 --- a/contrib/ci/docker/nodejs.dockerfile +++ b/contrib/ci/docker/nodejs.dockerfile @@ -1,3 +1,17 @@ FROM node:14.16.1 RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections' -RUN /bin/bash -c 'apt-get -o=Dpkg::Use-Pty=0 -q update && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y eatmydata gdb cmake make patch git ninja-build pkg-config ccache g++ wine openssh-client' +RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ + && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ + && apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + ccache \ + cmake \ + eatmydata \ + g++ \ + gdb \ + git \ + make \ + ninja-build \ + openssh-client \ + patch \ + pkg-config \ + wine From 0a9b020555009194168c287766562c080e7dd9ba Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 16:30:43 -0300 Subject: [PATCH 15/25] Update Catch2 to latest to fix build on impish --- test/Catch2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Catch2 b/test/Catch2 index 81d52c4a5..dba29b60d 160000 --- a/test/Catch2 +++ b/test/Catch2 @@ -1 +1 @@ -Subproject commit 81d52c4a5ffc1e527ce158c7570aa94dadc8c002 +Subproject commit dba29b60d639bf8d206a9a12c223e6ed4284fb13 From 4351611859b244af6f7efc69c8cfd196ead5d669 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 16:37:51 -0300 Subject: [PATCH 16/25] Add automake, libtool, libjemalloc-dev --- contrib/ci/docker/10-debian-bullseye.dockerfile | 3 +++ contrib/ci/docker/10-debian-buster.dockerfile | 3 +++ contrib/ci/docker/10-debian-sid.dockerfile | 3 +++ contrib/ci/docker/10-debian-stable.dockerfile | 3 +++ contrib/ci/docker/10-ubuntu-bionic.dockerfile | 1 + contrib/ci/docker/10-ubuntu-focal.dockerfile | 3 +++ contrib/ci/docker/10-ubuntu-impish.dockerfile | 3 +++ contrib/ci/docker/10-ubuntu-lts.dockerfile | 3 +++ contrib/ci/docker/10-ubuntu-rolling.dockerfile | 3 +++ 9 files changed, 25 insertions(+) diff --git a/contrib/ci/docker/10-debian-bullseye.dockerfile b/contrib/ci/docker/10-debian-bullseye.dockerfile index c061cac9e..d92ea1b97 100644 --- a/contrib/ci/docker/10-debian-bullseye.dockerfile +++ b/contrib/ci/docker/10-debian-bullseye.dockerfile @@ -1,6 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-bullseye-base/${ARCH} RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -14,12 +15,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index b9cd4e6f6..7349189f7 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -1,6 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-buster-base/${ARCH} RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -14,12 +15,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-debian-sid.dockerfile b/contrib/ci/docker/10-debian-sid.dockerfile index 32ac1d46f..7bb2f8c3c 100644 --- a/contrib/ci/docker/10-debian-sid.dockerfile +++ b/contrib/ci/docker/10-debian-sid.dockerfile @@ -1,6 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-sid-base/${ARCH} RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -14,12 +15,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-debian-stable.dockerfile b/contrib/ci/docker/10-debian-stable.dockerfile index 585de0a17..cd50a4651 100644 --- a/contrib/ci/docker/10-debian-stable.dockerfile +++ b/contrib/ci/docker/10-debian-stable.dockerfile @@ -1,6 +1,7 @@ ARG ARCH=amd64 FROM registry.oxen.rocks/lokinet-ci-debian-stable-base/${ARCH} RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -14,12 +15,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-ubuntu-bionic.dockerfile b/contrib/ci/docker/10-ubuntu-bionic.dockerfile index 9097406e9..767d2f3d1 100644 --- a/contrib/ci/docker/10-ubuntu-bionic.dockerfile +++ b/contrib/ci/docker/10-ubuntu-bionic.dockerfile @@ -14,6 +14,7 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ gdb \ git \ gperf \ + libjemalloc-dev \ libpgm-dev \ libtool \ libuv1-dev \ diff --git a/contrib/ci/docker/10-ubuntu-focal.dockerfile b/contrib/ci/docker/10-ubuntu-focal.dockerfile index aee953456..a9d38599e 100644 --- a/contrib/ci/docker/10-ubuntu-focal.dockerfile +++ b/contrib/ci/docker/10-ubuntu-focal.dockerfile @@ -4,6 +4,7 @@ RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-s RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -17,12 +18,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-ubuntu-impish.dockerfile b/contrib/ci/docker/10-ubuntu-impish.dockerfile index 472d12f50..0e7c354e0 100644 --- a/contrib/ci/docker/10-ubuntu-impish.dockerfile +++ b/contrib/ci/docker/10-ubuntu-impish.dockerfile @@ -4,6 +4,7 @@ RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-s RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -17,12 +18,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-ubuntu-lts.dockerfile b/contrib/ci/docker/10-ubuntu-lts.dockerfile index f2763cfa1..3b2dfc413 100644 --- a/contrib/ci/docker/10-ubuntu-lts.dockerfile +++ b/contrib/ci/docker/10-ubuntu-lts.dockerfile @@ -4,6 +4,7 @@ RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-s RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -17,12 +18,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ diff --git a/contrib/ci/docker/10-ubuntu-rolling.dockerfile b/contrib/ci/docker/10-ubuntu-rolling.dockerfile index de4401479..989dc0a5c 100644 --- a/contrib/ci/docker/10-ubuntu-rolling.dockerfile +++ b/contrib/ci/docker/10-ubuntu-rolling.dockerfile @@ -4,6 +4,7 @@ RUN /bin/bash -c 'echo "man-db man-db/auto-update boolean false" | debconf-set-s RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ && apt-get -o=Dpkg::Use-Pty=0 -q dist-upgrade -y \ && apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + automake \ ccache \ cmake \ eatmydata \ @@ -17,12 +18,14 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q update \ libevent-dev \ libgtest-dev \ libhidapi-dev \ + libjemalloc-dev \ libminiupnpc-dev \ libreadline-dev \ libsodium-dev \ libsqlite3-dev \ libssl-dev \ libsystemd-dev \ + libtool \ libunbound-dev \ libunwind8-dev \ libusb-1.0.0-dev \ From 76d69a3d705c9d77b47fc1240135945c33caf629 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 16:53:06 -0300 Subject: [PATCH 17/25] Add cached images with debhelper & related tools for deb pipelines --- .drone.jsonnet | 12 ++++++------ .../docker/40-debian-bullseye-debhelper.dockerfile | 10 ++++++++++ contrib/ci/docker/40-debian-sid-debhelper.dockerfile | 10 ++++++++++ .../ci/docker/40-ubuntu-focal-debhelper.dockerfile | 10 ++++++++++ .../ci/docker/40-ubuntu-impish-debhelper.dockerfile | 10 ++++++++++ .../arm64v8/40-debian-sid-debhelper.dockerfile | 1 + contrib/ci/docker/rebuild-docker-images.sh | 2 ++ 7 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile create mode 100644 contrib/ci/docker/40-debian-sid-debhelper.dockerfile create mode 100644 contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile create mode 100644 contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile create mode 120000 contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile diff --git a/.drone.jsonnet b/.drone.jsonnet index dec90b2bc..a23bd10cb 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -169,7 +169,7 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = ||| # Look for the debian branch in this repo first, try upstream if that fails. if ! git checkout $${distro_branch}; then - git remote add --fetch upstream https://github.com/oxen-io/loki-network.git && + git remote add --fetch upstream https://github.com/oxen-io/lokinet.git && git checkout $${distro_branch} fi |||, @@ -323,11 +323,11 @@ local mac_builder(name, cmake_extra='-DWITH_HIVE=ON'), // Deb builds: - deb_builder(docker_base + 'debian-sid', 'sid', 'debian/sid'), - deb_builder(docker_base + 'debian-bullseye', 'bullseye', 'debian/bullseye'), - deb_builder(docker_base + 'ubuntu-impish', 'impish', 'ubuntu/impish'), - deb_builder(docker_base + 'ubuntu-focal', 'focal', 'ubuntu/focal'), - deb_builder(docker_base + 'debian-sid', 'sid', 'debian/sid', arch='arm64'), + deb_builder(docker_base + 'debian-sid-debhelper', 'sid', 'debian/sid'), + deb_builder(docker_base + 'debian-bullseye-debhelper', 'bullseye', 'debian/bullseye'), + deb_builder(docker_base + 'ubuntu-impish-debhelper', 'impish', 'ubuntu/impish'), + deb_builder(docker_base + 'ubuntu-focal-debhelper', 'focal', 'ubuntu/focal'), + deb_builder(docker_base + 'debian-sid-debhelper', 'sid', 'debian/sid', arch='arm64'), // Macos builds: mac_builder('macOS (Release)'), diff --git a/contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile b/contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile new file mode 100644 index 000000000..05236d48c --- /dev/null +++ b/contrib/ci/docker/40-debian-bullseye-debhelper.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-bullseye/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + debhelper \ + devscripts \ + equivs \ + git \ + git-buildpackage \ + python3-dev diff --git a/contrib/ci/docker/40-debian-sid-debhelper.dockerfile b/contrib/ci/docker/40-debian-sid-debhelper.dockerfile new file mode 100644 index 000000000..e05764508 --- /dev/null +++ b/contrib/ci/docker/40-debian-sid-debhelper.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-debian-sid/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + debhelper \ + devscripts \ + equivs \ + git \ + git-buildpackage \ + python3-dev diff --git a/contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile b/contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile new file mode 100644 index 000000000..288179538 --- /dev/null +++ b/contrib/ci/docker/40-ubuntu-focal-debhelper.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-ubuntu-focal/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + debhelper \ + devscripts \ + equivs \ + git \ + git-buildpackage \ + python3-dev diff --git a/contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile b/contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile new file mode 100644 index 000000000..e73bb07ec --- /dev/null +++ b/contrib/ci/docker/40-ubuntu-impish-debhelper.dockerfile @@ -0,0 +1,10 @@ +ARG ARCH=amd64 +FROM registry.oxen.rocks/lokinet-ci-ubuntu-impish/${ARCH} +RUN apt-get -o=Dpkg::Use-Pty=0 --no-install-recommends -q install -y \ + ccache \ + debhelper \ + devscripts \ + equivs \ + git \ + git-buildpackage \ + python3-dev diff --git a/contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile b/contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile new file mode 120000 index 000000000..e7f04dc7a --- /dev/null +++ b/contrib/ci/docker/arm64v8/40-debian-sid-debhelper.dockerfile @@ -0,0 +1 @@ +../40-debian-sid-debhelper.dockerfile \ No newline at end of file diff --git a/contrib/ci/docker/rebuild-docker-images.sh b/contrib/ci/docker/rebuild-docker-images.sh index 6a3b97684..a0773a0b1 100755 --- a/contrib/ci/docker/rebuild-docker-images.sh +++ b/contrib/ci/docker/rebuild-docker-images.sh @@ -40,3 +40,5 @@ for latest in "${!manifests[@]}"; do docker manifest create $latest ${manifests[$latest]} docker manifest push $latest done + +echo -e "\n\n\n\e[32;1mAll done!\e[1m\n\n" From 55356face21c09ac8704fcbe0f44e758e9e1d3a7 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 17:33:30 -0300 Subject: [PATCH 18/25] No recommends --- .drone.jsonnet | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index a23bd10cb..f5d299a71 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -57,14 +57,14 @@ local debian_pipeline(name, apt_get_quiet + ' install -y eatmydata', ] + ( if loki_repo then [ - 'eatmydata ' + apt_get_quiet + ' install -y lsb-release', + 'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y lsb-release', 'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d', 'echo deb http://deb.oxen.io $$(lsb_release -sc) main >/etc/apt/sources.list.d/oxen.list', 'eatmydata ' + apt_get_quiet + ' update', ] else [] ) + [ 'eatmydata ' + apt_get_quiet + ' dist-upgrade -y', - 'eatmydata ' + apt_get_quiet + ' install -y gdb cmake git pkg-config ccache ' + std.join(' ', deps), + 'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y gdb cmake git pkg-config ccache ' + std.join(' ', deps), 'mkdir build', 'cd build', 'cmake .. -DWITH_SETCAP=OFF -DCMAKE_CXX_FLAGS=-fdiagnostics-color=always -DCMAKE_BUILD_TYPE=' + build_type + ' ' + @@ -133,7 +133,7 @@ local windows_cross_pipeline(name, 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', apt_get_quiet + ' update', apt_get_quiet + ' install -y eatmydata', - 'eatmydata ' + apt_get_quiet + ' install -y build-essential cmake git pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool', + 'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y build-essential cmake git pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip automake libtool', 'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix', 'update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix', 'VERBOSE=1 JOBS=' + jobs + ' ./contrib/windows.sh', @@ -165,7 +165,7 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = ] else []) + [ apt_get_quiet + ' update', apt_get_quiet + ' install -y eatmydata', - 'eatmydata ' + apt_get_quiet + ' install -y git devscripts equivs ccache git-buildpackage python3-dev', + 'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y git devscripts equivs ccache git-buildpackage python3-dev', ||| # Look for the debian branch in this repo first, try upstream if that fails. if ! git checkout $${distro_branch}; then @@ -254,7 +254,7 @@ local mac_builder(name, 'echo "Building on ${DRONE_STAGE_MACHINE}"', apt_get_quiet + ' update', apt_get_quiet + ' install -y eatmydata', - 'eatmydata ' + apt_get_quiet + ' install -y git clang-format-11 jsonnet', + 'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y git clang-format-11 jsonnet', './contrib/ci/drone-format-verify.sh', ], }], @@ -284,8 +284,8 @@ local mac_builder(name, arch='arm64', deps=['g++', 'python3-dev', 'automake', 'libtool'], cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + - '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp" -DCMAKE_C_FLAGS="-march=armv7-a+fp" -DNATIVE_BUILD=OFF ' + - '-DWITH_SYSTEMD=OFF', + '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp" -DCMAKE_C_FLAGS="-march=armv7-a+fp" ' + + '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', extra_cmds=[ '../contrib/ci/drone-check-static-libs.sh', 'UPLOAD_OS=linux-armhf ../contrib/ci/drone-static-upload.sh', @@ -308,9 +308,11 @@ local mac_builder(name, deps=['g++-8', 'python3-dev', 'automake', 'libtool'], lto=true, tests=false, - cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 ' + - '-DCMAKE_CXX_FLAGS="-march=x86-64 -mtune=haswell" -DCMAKE_C_FLAGS="-march=x86-64 -mtune=haswell" -DNATIVE_BUILD=OFF ' + - '-DWITH_SYSTEMD=OFF', + cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + + '-DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 ' + + '-DCMAKE_CXX_FLAGS="-march=x86-64 -mtune=haswell" ' + + '-DCMAKE_C_FLAGS="-march=x86-64 -mtune=haswell" ' + + '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', extra_cmds=[ '../contrib/ci/drone-check-static-libs.sh', '../contrib/ci/drone-static-upload.sh', From f71d795f1d0cf7da201d2f57a9d53fb27b45d356 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 17:36:33 -0300 Subject: [PATCH 19/25] Make sure we always pull the latest image --- .drone.jsonnet | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.drone.jsonnet b/.drone.jsonnet index f5d299a71..7fe7dc2ac 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -48,6 +48,7 @@ local debian_pipeline(name, { name: 'build', image: image, + pull: 'always', [if allow_fail then 'failure']: 'ignore', environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, commands: [ @@ -90,6 +91,7 @@ local apk_builder(name, image, extra_cmds=[], allow_fail=false, jobs=6) = { { name: 'build', image: image, + pull: 'always', [if allow_fail then 'failure']: 'ignore', environment: { SSH_KEY: { from_secret: 'SSH_KEY' }, ANDROID: 'android' }, commands: [ @@ -126,6 +128,7 @@ local windows_cross_pipeline(name, { name: 'build', image: image, + pull: 'always', [if allow_fail then 'failure']: 'ignore', environment: { SSH_KEY: { from_secret: 'SSH_KEY' }, WINDOWS_BUILD_NAME: toolchain + 'bit' }, commands: [ @@ -154,6 +157,7 @@ local deb_builder(image, distro, distro_branch, arch='amd64', loki_repo=true) = { name: 'build', image: image, + pull: 'always', failure: 'ignore', environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, commands: [ @@ -250,6 +254,7 @@ local mac_builder(name, steps: [{ name: 'build', image: docker_base + 'lint', + pull: 'always', commands: [ 'echo "Building on ${DRONE_STAGE_MACHINE}"', apt_get_quiet + ' update', From d27a095f1fc9a410688b4b3d4092c3ec7f4702cd Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 18:11:55 -0300 Subject: [PATCH 20/25] Fix arch-specific docker repo names --- .drone.jsonnet | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 7fe7dc2ac..ae0213348 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -270,7 +270,7 @@ local mac_builder(name, debian_pipeline('Debian sid/Debug (amd64)', docker_base + 'debian-sid', build_type='Debug'), clang(13), full_llvm(13), - debian_pipeline('Debian stable (i386)', docker_base + 'debian-stable:i386'), + debian_pipeline('Debian stable (i386)', docker_base + 'debian-stable/i386'), debian_pipeline('Debian buster (amd64)', docker_base + 'debian-buster', cmake_extra='-DDOWNLOAD_SODIUM=ON'), debian_pipeline('Ubuntu latest (amd64)', docker_base + 'ubuntu-rolling'), debian_pipeline('Ubuntu LTS (amd64)', docker_base + 'ubuntu-lts'), @@ -282,10 +282,10 @@ local mac_builder(name, // ARM builds (ARM64 and armhf) debian_pipeline('Debian sid (ARM64)', docker_base + 'debian-sid', arch='arm64', jobs=4), - debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable:arm32v7', arch='arm64', jobs=4), + debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable/arm32v7', arch='arm64', jobs=4), // Static armhf build (gets uploaded) debian_pipeline('Static (buster armhf)', - docker_base + 'debian-buster:arm32v7', + docker_base + 'debian-buster/arm32v7', arch='arm64', deps=['g++', 'python3-dev', 'automake', 'libtool'], cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + From 31d2242cc63a45ed872955016f9c553b04b2ab8d Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 23:11:17 -0300 Subject: [PATCH 21/25] Disable ABI warnings on buster/armhf --- .drone.jsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index ae0213348..0fb5a686b 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -289,7 +289,7 @@ local mac_builder(name, arch='arm64', deps=['g++', 'python3-dev', 'automake', 'libtool'], cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + - '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp" -DCMAKE_C_FLAGS="-march=armv7-a+fp" ' + + '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp -Wno-psabi" -DCMAKE_C_FLAGS="-march=armv7-a+fp" ' + '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', extra_cmds=[ '../contrib/ci/drone-check-static-libs.sh', From b64a38ff2c84585560b7014d777971592f727ca8 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 23:42:13 -0300 Subject: [PATCH 22/25] Add missing xz-utils to buster --- contrib/ci/docker/10-debian-buster.dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/ci/docker/10-debian-buster.dockerfile b/contrib/ci/docker/10-debian-buster.dockerfile index 7349189f7..f77b9deda 100644 --- a/contrib/ci/docker/10-debian-buster.dockerfile +++ b/contrib/ci/docker/10-debian-buster.dockerfile @@ -36,4 +36,5 @@ RUN apt-get -o=Dpkg::Use-Pty=0 -q install --no-install-recommends -y \ patch \ pkg-config \ python3-dev \ - qttools5-dev + qttools5-dev \ + xz-utils From 7bc86ff98336b301cf011cc4d95e24237696535b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 17 Oct 2021 23:44:03 -0300 Subject: [PATCH 23/25] Reorder pipelines to put static builds together --- .drone.jsonnet | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 0fb5a686b..91d76da80 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -283,19 +283,6 @@ local mac_builder(name, // ARM builds (ARM64 and armhf) debian_pipeline('Debian sid (ARM64)', docker_base + 'debian-sid', arch='arm64', jobs=4), debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable/arm32v7', arch='arm64', jobs=4), - // Static armhf build (gets uploaded) - debian_pipeline('Static (buster armhf)', - docker_base + 'debian-buster/arm32v7', - arch='arm64', - deps=['g++', 'python3-dev', 'automake', 'libtool'], - cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + - '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp -Wno-psabi" -DCMAKE_C_FLAGS="-march=armv7-a+fp" ' + - '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', - extra_cmds=[ - '../contrib/ci/drone-check-static-libs.sh', - 'UPLOAD_OS=linux-armhf ../contrib/ci/drone-static-upload.sh', - ], - jobs=4), // android apk builder apk_builder('android apk', docker_base + 'flutter', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']), @@ -322,6 +309,19 @@ local mac_builder(name, '../contrib/ci/drone-check-static-libs.sh', '../contrib/ci/drone-static-upload.sh', ]), + // Static armhf build (gets uploaded) + debian_pipeline('Static (buster armhf)', + docker_base + 'debian-buster/arm32v7', + arch='arm64', + deps=['g++', 'python3-dev', 'automake', 'libtool'], + cmake_extra='-DBUILD_STATIC_DEPS=ON -DBUILD_SHARED_LIBS=OFF -DSTATIC_LINK=ON ' + + '-DCMAKE_CXX_FLAGS="-march=armv7-a+fp -Wno-psabi" -DCMAKE_C_FLAGS="-march=armv7-a+fp" ' + + '-DNATIVE_BUILD=OFF -DWITH_SYSTEMD=OFF', + extra_cmds=[ + '../contrib/ci/drone-check-static-libs.sh', + 'UPLOAD_OS=linux-armhf ../contrib/ci/drone-static-upload.sh', + ], + jobs=4), // integration tests debian_pipeline('Router Hive', From 45249422b16c01f5a8502a045bdd3960a84eb62d Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 18 Oct 2021 13:50:43 -0300 Subject: [PATCH 24/25] Fetch CI submodules in parallel --- .drone.jsonnet | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 91d76da80..744ad55bc 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -16,10 +16,11 @@ local default_deps = ['g++'] + default_deps_nocxx; local default_windows_deps = ['mingw-w64', 'zip', 'nsis']; local docker_base = 'registry.oxen.rocks/lokinet-ci-'; +local submodule_commands = ['git fetch --tags', 'git submodule update --init --recursive --depth=1 --jobs=4']; local submodules = { name: 'submodules', image: 'drone/git', - commands: ['git fetch --tags', 'git submodule update --init --recursive --depth=1'], + commands: submodule_commands, }; local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q'; @@ -229,7 +230,7 @@ local mac_builder(name, name: name, platform: { os: 'darwin', arch: 'amd64' }, steps: [ - { name: 'submodules', commands: ['git fetch --tags', 'git submodule update --init --recursive'] }, + { name: 'submodules', commands: submodule_commands }, { name: 'build', environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, From 1a360c1a360d79a508b25b5dd3652c6ee3c461a4 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 19 Oct 2021 15:17:42 -0400 Subject: [PATCH 25/25] version bump to 0.9.7 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fee4f2a0..7ab561a0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ endif() project(lokinet - VERSION 0.9.6 + VERSION 0.9.7 DESCRIPTION "lokinet - IP packet onion router" LANGUAGES ${LANGS})