Merge namecoin/ncdns-repro#174: Bump ncdns to v0.3

3ee637e39f python-windows: Move Bash comments before the lines they apply to (Jeremy Rand)
94692af34c python-windows: Exclude unit tests from output (Jeremy Rand)
f42532efa2 python-windows: Don't hardcode arch in cross-file path (Jeremy Rand)
75aac284e6 python-windows: Add libffi dependency (Jeremy Rand)
68c386eec3 python-windows: Add openssl dependency (Jeremy Rand)
0aecf8f5f4 Add openssl project (Jeremy Rand)
775a02d643 plain-binaries: Add winsvcwrap dependency (Jeremy Rand)
fa019cf1a5 Add winsvcwrap project (Jeremy Rand)
059bb77096 ncdns-nsis: Add stemns dependency (Jeremy Rand)
846ae29e20 Add stemns project (Jeremy Rand)
df9c1b1d25 ncdns-nsis: Add stem dependency (Jeremy Rand)
2063d66397 ncdns-nsis: Add python-windows dependency (Jeremy Rand)
a93a0e881c Add reproducible build of Python 3.8.5 for Windows (Jeremy Rand)
ad6b387549 Add keyring for Meson (key of Jussi Pakkanen) (Jeremy Rand)
8343e94839 Update Python keyring to add Łukasz Langa (3.8.x source files and tags) (Jeremy Rand)
0e4b64ff6f certinject: Support building as library (Jeremy Rand)
66c0a96544 ncdns: Add certinject dependency (Jeremy Rand)
3174662922 Bump ncdns to v0.3 (NamecoinBot)

Pull request description:

Top commit has no ACKs.

keyring/python.gpg

@@ -1 +0,0 @@
-../tor-browser-build/keyring/python.gpg

projects/certinject/build

@@ -23,10 +23,22 @@ go install -ldflags '-s -buildid=' github.com/namecoin/certinject/cmd/certinject
   GOPATHBIN="${GOPATH}/bin/${GOOS}_${GOARCH}"
 [% END -%]
 
-cp -a $GOPATHBIN/certinject[% IF c("var/windows") %].exe[% END %] $distdir/
+cd /var/tmp/dist
+[% IF !c("var/go_lib_no_output") -%]
+  # Build as library
+  [% c('tar', {
+    tar_src => [ 'gopath' ],
+    tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+  }) %]
+[% ELSE %]
+  # Build as executable
+  ls $GOPATHBIN
 
-cd $distdir
-[% c('tar', {
+  cp -a $GOPATHBIN/certinject[% IF c("var/windows") %].exe[% END %] $distdir/
+
+  cd $distdir
+  [% c('tar', {
      tar_src   => [ '.' ],
      tar_args  => '-czf ' _ dest_dir _ '/' _ c('filename'),
    }) %]
+[% END %]

projects/certinject/config

@@ -13,6 +13,7 @@ var:
     - goeasyconfig
     - goxlog
     - goxsys
+  go_lib_no_output: 1
   cgo: 0
   build_go_lib_pre: |
     export CGO_ENABLED=[% c("var/cgo") %] 

projects/ncdns-nsis/build

@@ -45,6 +45,15 @@ cp $rootdir/[% c('input_files_by_name/dnssec-trigger') %] ${ARTIFACTS}
 cp $rootdir/[% c('input_files_by_name/consensusj-namecoin') %] ${ARTIFACTS}/bitcoinj-daemon.jar
 cp $rootdir/[% c('input_files_by_name/electrum-nmc') %] ${ARTIFACTS}
 
+tar -C /var/tmp/dist/ -xf $rootdir/[% c('input_files_by_name/python-windows') %]
+cp -a /var/tmp/dist/python-windows/bin "${ARTIFACTS}/python"
+
+tar -C /var/tmp/dist/ -xf $rootdir/[% c('input_files_by_name/stem') %]
+cp -a /var/tmp/dist/stem-*/stem "${NEUTRAL_ARTIFACTS}/stem"
+
+tar -C /var/tmp/dist/ -xf $rootdir/[% c('input_files_by_name/stemns') %]
+cp /var/tmp/dist/StemNS/stemns.py "${NEUTRAL_ARTIFACTS}/stemns/"
+
 # Windows PE metadata requires that the version be a dot-delimited 4-tuple of
 # numbers (no leading v)
 NCDNS_NSIS_VERSION=[% c('version') %]

projects/ncdns-nsis/config

@@ -12,6 +12,7 @@ var:
   dnssec_trigger_version: '0.17'
   namecoin_core_version: '0.21.0.1'
   electrum_nmc_version: '3.3.10'
+  stem_version: '1.8.0'
   container:
     use_container: 1
   deps:
@@ -53,6 +54,14 @@ input_files:
     sha256sum: 6b35d5a31eb74b4870c8a6c37dd53563aa63b64810fdedb5192f2a77396e190f
   - project: nsis
     name: nsis
+  - project: python-windows
+    name: python-windows
+  - name: stem
+    URL: 'https://files.pythonhosted.org/packages/source/s/stem/stem-[% c("var/stem_version") %].tar.gz'
+    sha256sum: a0b48ea6224e95f22aa34c0bc3415f0eb4667ddeae3dfb5e32a6920c185568c2
+    # TODO: This sha256sum was grabbed from PyPI and hasn't been verified.  We should probably switch to using an OpenPGP sig.
+  - project: stemns
+    name: stemns
   - filename: pe_checksum_fix.py
   - name: python-pefile
     URL: https://files.pythonhosted.org/packages/7e/9b/f99171190f04cd23768547dd34533b4016bd582842f53cd9fe9585a74c74/pefile-2017.11.5.tar.gz

projects/ncdns/config

@@ -1,4 +1,4 @@
-version: '0.2.2'
+version: '0.3'
 git_url: https://github.com/namecoin/ncdns.git
 git_hash: 'v[% c("version") %]'
 tag_gpg_id: 1
@@ -36,12 +36,12 @@ var:
   enable_namecoin_tlsa: 1
   os_go_lib_deps: []
   optional_go_lib_deps:
+    - certinject
     - gosplicesign
     - gotlsrestrictnss
     - gox509compressed
   optional_go_lib_install:
     - github.com/namecoin/ncdns/certdehydrate
-    - github.com/namecoin/ncdns/certinject
     - github.com/namecoin/ncdns/ncdumpzone
     - github.com/namecoin/ncdns/ncdumpzone/ncdumpzone
     - github.com/namecoin/ncdns/tlsoverridefirefox
@@ -83,6 +83,13 @@ input_files:
   - name: binutils
     project: binutils
     enable: '[% c("var/linux") && c("var/cgo") %]'
+  - name: certinject
+    project: certinject
+    # Build certinject as a library, not an executable.
+    # TODO: refactor this once our build_go_lib executable patch is merged by upstream Tor
+    var:
+      go_lib_no_output: 0
+    enable: '[% c("var/enable_namecoin_tlsa") %]'
   - name: goxlog
     project: goxlog
   - name: godexlogconfig

projects/openssl

@@ -0,0 +1 @@
+../tor-browser-build/projects/openssl

projects/plain-binaries/build

@@ -15,6 +15,9 @@ tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncdns') %]
 [% END %]
 tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncprop279') %]
 tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/q') %]
+[% IF c("var/windows") %]
+  tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/winsvcwrap') %]
+[% END %]
 
 cd $distdir
 [% c('tar', {

projects/plain-binaries/config

@@ -23,3 +23,7 @@ input_files:
     project: ncprop279
   - name: q
     project: q
+  - name: winsvcwrap
+    project: winsvcwrap
+    enable: '[% c("var/windows") %]'
+

projects/python-windows/build

@@ -0,0 +1,93 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+distdir=/var/tmp/dist/[% project %]
+[% pc(c('var/compiler'), 'var/setup', { compiler_tarfile => c('input_files_by_name/' _ c('var/compiler')) }) %]
+# Set up compiler
+
+shopt -s globstar
+
+# Required for "import ssl"
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/openssl') %]
+openssldir=/var/tmp/dist/openssl/openssl
+cp -a $openssldir/include/openssl /var/tmp/dist/mingw-w64/[% c("arch") %]-w64-mingw32/include/openssl
+cp -a $openssldir/lib/* $openssldir/bin/*.dll /var/tmp/dist/mingw-w64/[% c("arch") %]-w64-mingw32/lib/
+
+# Required for "import ctypes"
+tar -C /var/tmp/dist -xf [% c('input_files_by_name/libffi-src') %]
+pushd /var/tmp/dist/*libffi*
+# The Python devs got high on LSD and appear to have committed the libffi
+# source code from a Windows machine, which stripped off all the +x bits.
+# Kids, drugs and coding don't mix.
+chmod -R +x ./
+./configure --host=[% c("arch") %]-w64-mingw32
+# Always errors on this step, but still produces the .dll files we want.
+make || true
+cp ./[% c("arch") %]-w64-mingw32/include/*.h /var/tmp/dist/mingw-w64/[% c("arch") %]-w64-mingw32/include
+cp ./[% c("arch") %]-w64-mingw32/.libs/* /var/tmp/dist/mingw-w64/[% c("arch") %]-w64-mingw32/lib/
+popd
+
+# Unpack dependencies
+mkdir -p /var/tmp/build
+tar -C /var/tmp/build -xf [% c('input_files_by_name/python') %]
+tar -C /var/tmp/build -xf [% c('input_files_by_name/meson') %]
+tar -C /var/tmp/build -xf [% c('input_files_by_name/ninja') %]
+tar -C /var/tmp/build -xf [% c('input_files_by_name/python-buildscript') %]
+# Add dependencies to PATH
+PATH=/var/tmp/build/meson-[% c("meson_version") %]:"$PATH"
+PATH=/var/tmp/build/ninja:"$PATH"
+
+# Python includes build timestamp by default
+patch /var/tmp/build/Python-[% c("version") %]/Modules/getbuildinfo.c < $rootdir/omit-build-timestamp.patch
+# Patch build script to statically link pthread
+patch /var/tmp/build/cross-python-[% c("cp_githash") %]/meson.build << 'EOF'
+113c113
+<     add_project_link_arguments('-municode', '-mconsole', '-static-libgcc', language:'c')
+---
+>     add_project_link_arguments('-municode', '-mconsole', '-static-libgcc', '-Wl,-Bstatic', '-lwinpthread', '-Wl,-Bdynamic', language:'c')
+1076c1076
+< python_libraries = both_libraries(
+---
+> libpython = shared_library(
+1086,1087d1085
+< 
+< libpython = python_libraries.get_shared_lib()
+EOF
+# Prepare libwinpthread for static linking
+# This is a hack. (TODO: refactor)
+# We need to link this statically, because we aren't shipping the mingw runtime.
+# https://stackoverflow.com/questions/14665691/static-linking-with-libwinpthread
+ranlib /var/tmp/dist/mingw-w64/[% c("arch") %]-w64-mingw32/lib/libwinpthread.a
+
+# Project pulls in sqlite, zlib, and build files.
+# Download them outside, in rbm, to avoid container network access.
+mkdir -p /var/tmp/build/cross-python-[% c("cp_githash") %]/subprojects/packagecache
+cp [% c('input_files_by_name/sqlite-src') %] /var/tmp/build/cross-python-[% c("cp_githash") %]/subprojects/packagecache/sqlite-amalgamation-3250100.zip
+cp [% c('input_files_by_name/sqlite-patch') %] /var/tmp/build/cross-python-[% c("cp_githash") %]/subprojects/packagecache/sqlite-3250100-1-wrap.zip
+cp [% c('input_files_by_name/zlib-src') %] /var/tmp/build/cross-python-[% c("cp_githash") %]/subprojects/packagecache/zlib-1.2.11.tar.gz
+cp [% c('input_files_by_name/zlib-patch') %] /var/tmp/build/cross-python-[% c("cp_githash") %]/subprojects/packagecache/zlib-1.2.11-4-wrap.zip
+
+# Enter the build directory
+cd /var/tmp/build/cross-python-[% c("cp_githash") %]
+
+# cf. ./configure
+LDFLAGS="-Wl,--no-insert-timestamp" meson.py -Dsource=../Python-[% c("version") %] --cross-file cross-files/[% c("arch") %]-w64-mingw32.txt --prefix=$distdir builddir
+# cf. make
+ninja -C builddir
+# cf. make install
+ninja -C builddir install
+
+# These files don't reproduce, and are useless
+find $distdir -name '*.a' -exec rm '{}' +
+
+# Required for "import ssl" and "import ctypes"
+cp $openssldir/bin/*.dll /var/tmp/dist/*libffi*/[% c("arch") %]-w64-mingw32/.libs/*.dll $gcclibs/*.dll $distdir/bin/
+
+# Delete unit tests to save space (71 to 49 MB)
+cd $distdir/bin
+rm -rf ./**/test ./**/tests ./**/test_*
+
+cd /var/tmp/dist
+[% c('tar', {
+        tar_src => [ project ],
+        tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+    }) %]

projects/python-windows/config

@@ -0,0 +1,63 @@
+# vim: filetype=yaml sw=2
+version: 3.8.5
+meson_version: 0.55.1
+cp_githash: cb160097d0df4b510e011aa263a37150bfab8939
+filename: 'python-[% c("lsb_release/id") %]-[% c("lsb_release/release") %]-[% c("var/build_id") %].tar.gz'
+var:
+  container:
+    use_container: 1
+  deps:
+    - xz-utils
+    - build-essential
+    - python3
+    - pkg-config
+    - zip
+    - git # maybe possible to omit these
+  compiler: mingw-w64
+  post_pkginst: |
+    mkdir -p /usr/i686-w64-mingw32/sys-root/mingw/lib/libffi-3.1/
+    # Yes, Meson uses "i686" in the sysroot name even for 64-bit.  v0v
+    ln -s -T /var/tmp/dist/cpython-source-deps-libffi-3.4.2/[% c("arch") %]-w64-mingw32/include /usr/i686-w64-mingw32/sys-root/mingw/lib/libffi-3.1/include
+
+input_files:
+  - project: container-image
+  - project: '[% c("var/compiler") %]'
+    name: '[% c("var/compiler") %]'
+  - project: ninja
+    name: ninja
+  - name: python
+    URL: 'https://www.python.org/ftp/python/[% c("version") %]/Python-[% c("version") %].tar.xz'
+    sig_ext: asc
+    gpg_keyring: python.gpg
+  - name: meson
+    URL: 'https://github.com/mesonbuild/meson/releases/download/[% c("meson_version") %]/meson-[% c("meson_version") %].tar.gz'
+    sig_ext: asc
+    gpg_keyring: meson.gpg
+  - name: python-buildscript
+    URL: 'https://github.com/v-finance/cross-python/archive/[% c("cp_githash") %].tar.gz'
+    sha256sum: 4f427bbd1601235ad26553ec2ca10a309124849df4436e76afdb84af7bd6584b
+  - name: '[% c("var/compiler") %]'
+    project: '[% c("var/compiler") %]'
+  - name: zlib-src
+    filename: zlib-1.2.11.tar.gz
+    URL: http://zlib.net/fossils/zlib-1.2.11.tar.gz
+    sha256sum: c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+  - name: zlib-patch
+    filename: zlib-1.2.11-4-wrap.zip
+    URL: https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/4/get_zip
+    sha256sum: f733976fbfc59e0bcde01aa9469a24eeb16faf0a4280b17e9eaa60a301d75657
+  - name: sqlite-src
+    filename: sqlite-amalgamation-3250100.zip
+    URL: https://www.sqlite.org/2018/sqlite-amalgamation-3250100.zip
+    sha256sum: 6c68b6364fa0e01d9d05088ba473ac4e7a6fbc64f9b9aeb6d830f944e2a1fa03
+  - name: sqlite-patch
+    filename: sqlite-3250100-1-wrap.zip
+    URL: https://wrapdb.mesonbuild.com/v1/projects/sqlite/3250100/1/get_zip
+    sha256sum: cbb697a1e4649cf6b7f28d82048109d87ccbf01106e1d5a68f763d4685d8607c
+  - filename: omit-build-timestamp.patch
+  - name: openssl
+    project: openssl
+  - name: libffi-src
+    URL: https://github.com/python/cpython-source-deps/archive/refs/tags/libffi-3.4.2.tar.gz
+    sha256sum: 9f9edfbb59a142a8624cec574eeac0caf98c2e51a15e6c9f099b83aaf0942c22
+

projects/python-windows/omit-build-timestamp.patch

@@ -0,0 +1,8 @@
+8c8
+< #ifdef __DATE__
+---
+> #if defined(__DATE__) && defined(EMIT_BUILD_TIMESTAMP)
+16c16
+< #ifdef __TIME__
+---
+> #if defined(__TIME__) && defined(EMIT_BUILD_TIMESTAMP)

projects/stemns/build

@@ -0,0 +1,18 @@
+#!/bin/sh
+[% c("var/set_default_env") -%]
+distdir=/var/tmp/dist/StemNS
+mkdir /var/tmp/build
+mkdir /var/tmp/dist
+
+# Extract StemNS
+tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz
+rm /var/tmp/build/[% project %]-[% c('version') %]/settings_*.py
+
+cp -a /var/tmp/build/[% project %]-[% c('version') %] ${distdir}
+
+cd /var/tmp/dist/
+
+[% c('tar', {
+        tar_src => '.',
+        tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
+        }) %]

projects/stemns/config

@@ -0,0 +1,13 @@
+# vim: filetype=yaml sw=2
+version: '0.2.0'
+git_url: https://github.com/namecoin/StemNS.git
+git_hash: 'v[% c("version") %]'
+tag_gpg_id: 1
+gpg_keyring: namecoin.gpg
+filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %].tar.gz'
+var:
+  container:
+    use_container: 1
+
+input_files:
+  - project: container-image

projects/winsvcwrap/build

@@ -0,0 +1,37 @@
+#!/bin/bash
+[% c("var/set_default_env") -%]
+[% pc('go', 'var/setup', { go_tarfile => c('input_files_by_name/go') }) %]
+export CGO_ENABLED=0
+distdir=/var/tmp/dist/[% project %]
+mkdir -p $distdir
+
+[% FOREACH dep = c("var/go_lib_deps") -%]
+  tar -C /var/tmp/dist -xf [% c('input_files_by_name/' _ dep) %]
+[% END -%]
+
+mkdir -p $GOPATH/src/github.com/hlandau
+tar -C $GOPATH/src/github.com/hlandau -xf [% project %]-[% c('version') %].tar.gz
+mv $GOPATH/src/github.com/hlandau/winsvcwrap-[% c('version') %] $GOPATH/src/github.com/hlandau/winsvcwrap
+
+[% FOREACH inst IN c("var/go_lib_install") %]
+  # TODO: Remove static build ID after migrating to Go 1.13.3+
+  go install -ldflags '-s -buildid=' [% inst %]
+[% END %]
+
+[% IF c("var/linux-x86_64") -%]
+  GOPATHBIN="${GOPATH}/bin"
+[% ELSE -%]
+  GOPATHBIN="${GOPATH}/bin/${GOOS}_${GOARCH}"
+[% END -%]
+
+ls $GOPATHBIN
+
+for x in winsvcwrap; do
+  cp -a $GOPATHBIN/"$x"[% IF c("var/windows") %].exe[% END %] $distdir/
+done
+
+cd $distdir
+[% c('tar', {
+     tar_src   => [ '.' ],
+     tar_args  => '-czf ' _ dest_dir _ '/' _ c('filename'),
+   }) %]

projects/winsvcwrap/config

@@ -0,0 +1,26 @@
+version: '[% c("abbrev") %]'
+git_url:  https://github.com/hlandau/winsvcwrap.git
+git_hash: 'c0800f931ae97f9662297b22f8c069a488209d05'
+filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
+
+var:
+  container:
+    use_container: 1
+  go_lib_deps:
+    - godexlogconfig
+    - goservice
+  go_lib_install:
+    - github.com/hlandau/winsvcwrap
+  cgo: 0
+  build_go_lib_pre: |
+    export CGO_ENABLED=[% c("var/cgo") %]
+
+input_files:
+  - project: container-image
+  - name: go
+    project: go
+  - name: godexlogconfig
+    project: godexlogconfig
+  - name: goservice
+    project: goservice
+