Merge #18: Shrink diff against upstream Tor as much as possible

0993230 Bring rbm.conf closer to upstream (JeremyRand)
8b8029d Bring Makefile closer to upstream (JeremyRand)
9596bba Add tools/clean-old symlink (JeremyRand)
385ff12 Bring .gitignore closer to upstream (JeremyRand)
d6add12 Add symlink from rbm to tor-browser-build/rbm (JeremyRand)
838080b Remove our rbm submodule (JeremyRand)
fdf931e Pull latest tor-browser-build (JeremyRand)
3da6339 Pull latest rbm (JeremyRand)

Pull request description:

  This PR eliminates most of the diff between ncdns-repro and upstream tor-browser-build.  It wasn't feasible to properly audit most of the diff, and upstream has substantially better QA resources than we do.  If any parts of the diff were actually required for ncdns, I suggest either submitting them as follow-up PR's here (with comments explaining why they're needed), or else submitting them upstream to Tor.

  I can confirm that `make release-linux-x86_64` produces a working binary (modulo cgo issues that I'll file separately) with this PR applied.  I haven't tried any of the other targets.

.gitignore

@@ -1,3 +1,12 @@
-out/
+/git_clones
-tmp/
+/hg_clones
-*.swp
+/gclient
+/out
+/release
+/alpha
+/alpha_nightly
+/nightly
+/testbuild
+/rbm.local.conf
+/logs
+/tmp

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "rbm"]
-	path = rbm
-	url = https://git.torproject.org/builders/rbm.git
 [submodule "tor-browser-build"]
 	path = tor-browser-build
 	url = https://git.torproject.org/builders/tor-browser-build.git

Makefile

@@ -1,12 +1,41 @@
-RBM=./rbm/rbm
+rbm=./rbm/rbm
 
 all: release
 
-release:
+release: submodule-update
-	$(RBM) build ncdns --target ncdns-linux-x86_64
+	$(rbm) build ncdns --target release --target ncdns-all
+
+release-android-armv7: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-android-armv7
+
+release-android-x86: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-android-x86
+
+release-linux-x86_64: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-linux-x86_64
+
+release-linux-i686: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-linux-i686
+
+release-windows-i686: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-windows-i686
+
+release-windows-x86_64: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-windows-x86_64
+
+release-osx-x86_64: submodule-update
+	$(rbm) build ncdns --target release --target ncdns-osx-x86_64
 
 submodule-update:
 	git submodule update --init
+	$(MAKE) -C tor-browser-build submodule-update
 
 fetch: submodule-update
-	$(RBM) fetch
+	$(rbm) fetch
+
+clean: submodule-update
+	./tools/clean-old
+
+clean-dry-run: submodule-update
+	./tools/clean-old --dry-run
+

rbm

@@ -1 +0,0 @@
-Subproject commit eb500fa9467fb4d7229c9ca87f202ef18603d023

rbm

@@ -0,0 +1 @@
+tor-browser-build/rbm

rbm.conf

@@ -1,17 +1,40 @@
+# vim: filetype=yaml sw=2
 debug: '[% GET ! ENV.RBM_NO_DEBUG %]'
 compress_tar: gz
-output_dir: 'out/[% project %]'
+output_dir: "out/[% project %]"
 tmp_dir: '[% c("basedir") %]/tmp'
-_build_log: 'logs/[% project %][% IF c("var/osname") %]-[% c("var/osname") %][% END %].log'
+build_log: '[% GET ENV.RBM_LOGS_DIR ? ENV.RBM_LOGS_DIR : "logs" %]/[% project %][% IF c("var/osname") %]-[% c("var/osname") %][% END %].log'
+
 pkg_type: build
 
+steps:
+  src-tarballs:
+    compress_tar: xz
+    src-tarballs: |
+      #!/bin/bash
+      set -e
+      mkdir -p '[% dest_dir %]'
+      mv -vf '[% project %]-[% c("version") %].tar.xz' '[% dest_dir %]/[% c("filename") %]'
+
+# buildconf contains build options that the user can change in rbm.local.conf
+# When adding a new option to buildconf, a default value should be defined
+# in var/build_id, so that changing this option does not affect the build_id.
+buildconf:
+  num_procs: '[% GET ENV.RBM_NUM_PROCS ? ENV.RBM_NUM_PROCS : "4" %]'
+  git_signtag_opt: '-s'
+
 var:
+  project_name: tor-browser
+  # By default, we sort the list of installed packages. This allows sharing
+  # containers with identical list of packages, even if they are not listed
+  # in the same order. In the cases where the installation order is
+  # important, sort_deps should be set to 0.
   sort_deps: 1
   build_id: '[% sha256(c("var/build_id_txt", { buildconf => { num_procs => 4 } })).substr(0, 6) %]'
   build_id_txt: |
     [% c("version") %]
     [% IF c("git_hash") || c("hg_hash"); GET c("abbrev"); END; %]
-    [% IF c("var/container/use_container") -%]
+    [% IF c("var/container/use_container") && ! c("var/container/global_disable") -%]
     [% c("var/container/suite") %]
     [% c("var/container/arch") %]
     [% END -%]
@@ -21,51 +44,266 @@ var:
   container:
     dir: '[% c("rbm_tmp_dir") %]/rbm-containers/[% sha256(c("build_id")) %]'
     user: rbm
+    disable_network:
+      # disable network in the build scripts
+      build: 1
   input_files_list: |
     [% FOREACH file IN c("input_files_by_name").keys.sort -%]
     [% c("input_files_by_name/" _ file) %]
     [% END -%]
+
+  faketime: "faketime -f \"[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]\""
+  touch: "[% USE date %]touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %]"
+
+  sign_build: '[% ENV.RBM_SIGN_BUILD %]'
+  sign_build_gpg_opts: '[% ENV.RBM_GPG_OPTS %]'
+
+  rezip: |
+    rezip_tmpdir=$(mktemp -d)
+    mkdir -p "$rezip_tmpdir/z"
+    unzip -d "$rezip_tmpdir/z" -- [% c("rezip_file") %] || [ $? -lt 3 ]
+    pushd "$rezip_tmpdir/z"
+    [% c("zip", {
+      zip_src => [ '.' ],
+      zip_args => '$rezip_tmpdir/new.zip',
+    }) %]
+    popd
+    mv -f -- "$rezip_tmpdir/new.zip" [% c("rezip_file") %]
+    rm -Rf "$rezip_tmpdir"
+
   set_default_env: |
     set -e
     [% FOREACH env = c('ENV') -%]
     export [% env.key %]="[% env.value %]"
     [% END -%]
     rootdir=$(pwd)
-    export SHELL=/bin/sh
+    export SHELL=/bin/bash
     export HOME=$rootdir
     umask 0022
+    [% IF c("var/container/global_disable") -%]
+      rm -Rf /var/tmp/build /var/tmp/dist
+    [% END -%]
 
 targets:
   notarget: linux-x86_64
   noint:
     debug: 0
 
+  release:
+    var:
+      release: 1
+      channel: release
+
+  # The common-stretch target is used to build components that are common to all
+  # platforms, using Debian stretch.
+  common-stretch:
+    var:
+      common: 1
+      container:
+        suite: stretch
+        arch: amd64
+      pre_pkginst: ''
+      deps:
+        - build-essential
+        - python
+        - bison
+        - automake
+        - libtool
+        - zip
+        - unzip
+
+  ncdns-android-armv7:
+    - android-armv7
+    - android
+  android-armv7:
+    arch: armv7
+    var:
+      android-armv7: 1
+      osname: android-armv7
+  ncdns-android-x86:
+    - android-x86
+    - android
+  android-x86:
+    arch: x86
+    var:
+      android-x86: 1
+      osname: android-x86
+  android:
+    var:
+      android: 1
+      compiler: android-toolchain
+      # API 16 is the minimum we currently support for Tor Browser on Android
+      android_min_api: 16
+      container:
+        suite: stretch
+        arch: amd64
+      deps:
+        - build-essential
+        - python
+        - bison
+        - automake
+        - libtool
+        - zip
+        - unzip
+
   ncdns-linux-x86_64:
     - linux-x86_64
     - linux
+  ncdns-linux-i686:
+    - linux-i686
+    - linux
   linux-x86_64:
     arch: x86_64
     var:
       linux-x86_64: 1
       osname: linux-x86_64
-      container:
+  linux-i686:
-        arch: amd64
+    arch: i686
+    var:
+      linux-i686: 1
+      osname: linux-i686
+      configure_opt_i686: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32'
+      configure_opt: '[% c("var/configure_opt_i686") %]'
   linux:
     var:
       linux: 1
       compiler: gcc
       container:
         suite: wheezy
+        arch: amd64
+      pre_pkginst: dpkg --add-architecture i386
       deps:
+        - libc6-dev-i386
+        - lib32stdc++6
         - build-essential
+        - python
+        - bison
+        - hardening-wrapper
+        - automake
+        - libtool
         - zip
         - unzip
 
-remote_start: '[% IF c("var/container/use_container") %][% c("runc/remote_start") %][% END %]'
+  ncdns-windows-i686:
-remote_exec: '[% IF c("var/container/use_container") %][% c("runc/remote_exec") %][% END %]'
+    - windows-i686
-remote_put: '[% IF c("var/container/use_container") %][% c("runc/remote_put") %][% END %]'
+    - windows
-remote_get: '[% IF c("var/container/use_container") %][% c("runc/remote_get") %][% END %]'
+  ncdns-windows-x86_64:
-remote_finish: '[% IF c("var/container/use_container") %][% c("runc/remote_finish") %][% END %]'
+    - windows-x86_64
+    - windows
+  windows-x86_64:
+    arch: x86_64
+    var:
+      windows-x86_64: 1
+      osname: windows-x86_64
+      container:
+        arch: amd64
+      faketime_path: /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+      # HEASLR is 64 bit only (see bug 12968)
+      flag_HEASLR: '-Wl,--high-entropy-va'
+  windows-i686:
+    arch: i686
+    var:
+      windows-i686: 1
+      osname: windows-i686
+      container:
+        arch: i386
+      faketime_path: /usr/lib/i386-linux-gnu/faketime/libfaketime.so.1
+      setarch: |
+        if test -z "$RBM_SETARCH"
+        then
+           export RBM_SETARCH=1
+           exec setarch i686 ./build
+        fi
+  windows:
+    var:
+      windows: 1
+      container:
+        suite: jessie
+      configure_opt: '--host=[% c("arch") %]-w64-mingw32 CFLAGS="[% c("var/CFLAGS") %]" LDFLAGS="[% c("var/LDFLAGS") %]"'
+      CFLAGS: '-fstack-protector-strong -fno-strict-overflow -Wno-missing-field-initializers -Wformat -Wformat-security [% c("var/flag_mwindows") %]'
+      LDFLAGS: '-Wl,--dynamicbase -Wl,--nxcompat -Wl,--enable-reloc-section -Wl,--no-insert-timestamp -lssp -L$gcclibs [% c("var/flag_HEASLR") %] [% c("var/flag_mwindows") %]'
+      flag_mwindows: '-mwindows'
+      compiler: mingw-w64
+      deps:
+        - build-essential
+        - python
+        - bison
+        - automake
+        - libtool
+        - zip
+        - unzip
+
+  ncdns-osx-x86_64:
+    - osx-x86_64
+  osx-x86_64:
+    arch: x86_64
+    var:
+      osx: 1
+      osname: osx-x86_64
+      container:
+        suite: jessie
+        arch: amd64
+      compiler: 'macosx-toolchain'
+      configure_opt: '--host=x86_64-apple-darwin11 CC="x86_64-apple-darwin11-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin11-clang++ [% c("var/FLAGS") %]"'
+      FLAGS: "-target x86_64-apple-darwin11 -B $cctoolsdir -isysroot $sysrootdir"
+      LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie"
+      deps:
+        - build-essential
+        - python
+        - bison
+        - automake
+        - libtool
+        - zip
+        - unzip
+      faketime_path: /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+
+  # The no_build_id target can be useful if you want to quickly display
+  # a build template or other option but don't want to spend time to
+  # compute the various build ids
+  no_build_id:
+    # The defaut timestamp value will use the commit time of the
+    # selected commit for the project, which will require cloning the
+    # git repository if it is not present. When we use the no_build_id
+    # target to display a script, we usually don't care about such
+    # details, so we set timestamp to 0 to avoid unnecessary cloning.
+    timestamp: 0
+    var:
+      build_id: 1
+
+  no_containers:
+    var:
+      container:
+        global_disable: 1
+
+# change the default gpg_wrapper to allow git tag signed using an
+# expired key.
+# https://bugs.torproject.org/19737
+gpg_wrapper: |
+  #!/bin/bash
+  export LC_ALL=C
+  [%
+      IF c('gpg_keyring');
+          SET gpg_kr = '--keyring ' _ path(c('gpg_keyring'), path(c('gpg_keyring_dir'))) _ ' --no-default-keyring';
+      END;
+  -%]
+  gpg_verify=0
+  for opt in "$@"
+  do
+    test "$opt" = '--verify' && gpg_verify=1
+  done
+  if [ $gpg_verify = 1 ]
+  then
+        [% c('gpg_bin') %] [% c('gpg_args') %] --with-fingerprint [% gpg_kr %] "$@" | sed 's/^\[GNUPG:\] EXPKEYSIG /\[GNUPG:\] GOODSIG /'
+        exit ${PIPESTATUS[0]}
+  else
+        exec [% c('gpg_bin') %] [% c('gpg_args') %] --with-fingerprint [% gpg_kr %] "$@"
+  fi
+
+remote_start: '[% IF c("var/container/use_container") && ! c("var/container/global_disable") %][% c("runc/remote_start") %][% END %]'
+remote_exec: '[% IF c("var/container/use_container") && ! c("var/container/global_disable") %][% c("runc/remote_exec") %][% END %]'
+remote_put: '[% IF c("var/container/use_container") && ! c("var/container/global_disable") %][% c("runc/remote_put") %][% END %]'
+remote_get: '[% IF c("var/container/use_container") && ! c("var/container/global_disable") %][% c("runc/remote_get") %][% END %]'
+remote_finish: '[% IF c("var/container/use_container") && ! c("var/container/global_disable") %][% c("runc/remote_finish") %][% END %]'
 
 runc:
   remote_start: |
@@ -97,10 +335,16 @@ runc:
     echo 'su - [% user %] -c /rbm/cmd' >> '[% c("var/container/dir") %]'/rootfs/rbm/run
     chmod +x '[% c("var/container/dir") %]'/rootfs/rbm/cmd
     chmod +x '[% c("var/container/dir") %]'/rootfs/rbm/run
-    cat > '[% c("var/container/dir") %]'/config.json <<EOF
+    cat > '[% c("var/container/dir") %]'/config.json << EOF
     [% INCLUDE 'runc-config.json' %]
     EOF
+    [% IF c("var/container/disable_network/" _ c("exec_name")) -%]
+      sudo ip netns add 'rbm-[% sha256(c("build_id", { error_if_undef => 1 })) %]'
+    [% END -%]
     sudo runc [% IF c("var_p/runc100") %]run[% ELSE %]start[% END %] -b '[% c("var/container/dir") %]' rbm-[% sha256(c("build_id", { error_if_undef => 1 })) %] [% IF c("runc_hide_stderr") %]2>/dev/null[% END %]
+    [% IF c("var/container/disable_network/" _ c("exec_name")) -%]
+      sudo ip netns delete 'rbm-[% sha256(c("build_id", { error_if_undef => 1 })) %]'
+    [% END -%]
 
   remote_put: |
     #!/bin/sh
@@ -172,7 +416,7 @@ ENV:
       # We will need to update this when there is a new spec version available
       runc_spec100 => sub {
         my ($out) = capture_exec('sudo', 'runc', '--version');
-        return $out =~ m/^.*spec: 1\.[0-9]+\.[0-9]+$/m;
+        return $out =~ m/^.*spec: 1\.[0-9]+\.[0-9]+(?:-dev)?$/m;
       },
 
       id => $hashlist,

tools/clean-old

@@ -0,0 +1 @@
+../tor-browser-build/tools/clean-old

tor-browser-build

@@ -1 +1 @@
-Subproject commit d8f156e110afe00e0b366cff8ff0e0c53b4a58c9
+Subproject commit 0fc15a39f2adcf029676810c2ec37c566886879e