diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..b313986 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,9 @@ +Repository Tools +--------------------- + +### [Developer tools](/contrib/devtools) ### +Specific tools for developers working on this repository. +Contains the script `github-merge.py` for merging github pull requests securely and signing them using GPG. + +### [Verify-Commits](/contrib/verify-commits) ### +Tool to verify that every merge commit was signed by a developer using the above `github-merge.py` script. diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index c8dcaae..f82362f 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright (c) 2016 Bitcoin Core Developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -19,6 +19,11 @@ import os,sys from sys import stdin,stdout,stderr import argparse import subprocess +import json,codecs +try: + from urllib.request import Request,urlopen +except: + from urllib2 import Request,urlopen # External tools (can be overridden using environment) GIT = os.getenv('GIT','git') @@ -38,38 +43,39 @@ def git_config_get(option, default=None): Get named configuration option from git repository. ''' try: - return subprocess.check_output([GIT,'config','--get',option]).rstrip() + return subprocess.check_output([GIT,'config','--get',option]).rstrip().decode('utf-8') except subprocess.CalledProcessError as e: return default -def retrieve_pr_title(repo,pull): +def retrieve_pr_info(repo,pull): ''' - Retrieve pull request title from github. + Retrieve pull request information from github. Return None if no title can be found, or an error happens. ''' - import urllib2,json try: - req = urllib2.Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) - result = urllib2.urlopen(req) - result = json.load(result) - return result['title'] + req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) + result = urlopen(req) + reader = codecs.getreader('utf-8') + obj = json.load(reader(result)) + return obj except Exception as e: - print('Warning: unable to retrieve pull title from github: %s' % e) + print('Warning: unable to retrieve pull information from github: %s' % e) return None def ask_prompt(text): print(text,end=" ",file=stderr) + stderr.flush() reply = stdin.readline().rstrip() print("",file=stderr) return reply -def parse_arguments(branch): +def parse_arguments(): epilog = ''' In addition, you can set the following git configuration variables: githubmerge.repository (mandatory), user.signingkey (mandatory), githubmerge.host (default: git@github.com), - githubmerge.branch (default: master), + githubmerge.branch (no default), githubmerge.testcmd (default: none). ''' parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', @@ -77,14 +83,14 @@ def parse_arguments(branch): parser.add_argument('pull', metavar='PULL', type=int, nargs=1, help='Pull request ID to merge') parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', - default=branch, help='Branch to merge against (default: '+branch+')') + default=None, help='Branch to merge against (default: githubmerge.branch setting, or base branch for pull, or \'master\')') return parser.parse_args() def main(): # Extract settings from git repo repo = git_config_get('githubmerge.repository') host = git_config_get('githubmerge.host','git@github.com') - branch = git_config_get('githubmerge.branch','master') + opt_branch = git_config_get('githubmerge.branch',None) testcmd = git_config_get('githubmerge.testcmd') signingkey = git_config_get('user.signingkey') if repo is None: @@ -99,9 +105,20 @@ def main(): host_repo = host+":"+repo # shortcut for push/pull target # Extract settings from command line - args = parse_arguments(branch) + args = parse_arguments() pull = str(args.pull[0]) - branch = args.branch + + # Receive pull information from github + info = retrieve_pr_info(repo,pull) + if info is None: + exit(1) + title = info['title'] + # precedence order for destination branch argument: + # - command line argument + # - githubmerge.branch setting + # - base branch for pull (as retrieved from github) + # - 'master' + branch = args.branch or opt_branch or info['base']['ref'] or 'master' # Initialize source branches head_branch = 'pull/'+pull+'/head' @@ -141,7 +158,6 @@ def main(): try: # Create unsigned merge commit. - title = retrieve_pr_title(repo,pull) if title: firstline = 'Merge #%s: %s' % (pull,title) else: @@ -159,7 +175,7 @@ def main(): print("ERROR: Creating merge failed (already merged?).",file=stderr) exit(4) - print('%s#%s%s %s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title)) + print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) print() # Run test command if configured. diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md new file mode 100644 index 0000000..e9e3f65 --- /dev/null +++ b/contrib/verify-commits/README.md @@ -0,0 +1,26 @@ +Tooling for verification of PGP signed commits +---------------------------------------------- + +This is an incomplete work in progress, but currently includes a pre-push hook +script (`pre-push-hook.sh`) for maintainers to ensure that their own commits +are PGP signed (nearly always merge commits), as well as a script to verify +commits against a trusted keys list. + + +Using verify-commits.sh safely +------------------------------ + +Remember that you can't use an untrusted script to verify itself. This means +that checking out code, then running `verify-commits.sh` against `HEAD` is +_not_ safe, because the version of `verify-commits.sh` that you just ran could +be backdoored. Instead, you need to use a trusted version of verify-commits +prior to checkout to make sure you're checking out only code signed by trusted +keys: + + git fetch origin && \ + ./contrib/verify-commits/verify-commits.sh origin/master && \ + git checkout origin/master + +Note that the above isn't a good UI/UX yet, and needs significant improvements +to make it more convenient and reduce the chance of errors; pull-reqs +improving this process would be much appreciated. diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits new file mode 100644 index 0000000..e69de29 diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh new file mode 100755 index 0000000..09ff237 --- /dev/null +++ b/contrib/verify-commits/gpg.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +INPUT=$(cat /dev/stdin) +VALID=false +REVSIG=false +IFS=' +' +for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do + case "$LINE" in + "[GNUPG:] VALIDSIG "*) + while read KEY; do + case "$LINE" in "[GNUPG:] VALIDSIG $KEY "*) VALID=true;; esac + done < ./contrib/verify-commits/trusted-keys + ;; + "[GNUPG:] REVKEYSIG "*) + [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 + while read KEY; do + case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY#????????????????????????} "*) + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${KEY#????????????????????????} " + esac + done < ./contrib/verify-commits/trusted-keys + ;; + esac +done +if ! $VALID; then + exit 1 +fi +if $VALID && $REVSIG; then + echo "$INPUT" | gpg --trust-model always "$@" | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" 2>/dev/null + echo "$GOODREVSIG" +else + echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null +fi diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh new file mode 100755 index 0000000..5cd449d --- /dev/null +++ b/contrib/verify-commits/pre-push-hook.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)devrandom/gitian-builder(.git)?$ ]]; then + exit 0 +fi + +while read LINE; do + set -- A $LINE + if [ "$4" != "refs/heads/master" ]; then + continue + fi + if ! ./contrib/verify-commits/verify-commits.sh $3 > /dev/null 2>&1; then + echo "ERROR: A commit is not signed, can't push" + ./contrib/verify-commits/verify-commits.sh + exit 1 + fi +done < /dev/stdin diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root new file mode 100644 index 0000000..8048b8f --- /dev/null +++ b/contrib/verify-commits/trusted-git-root @@ -0,0 +1 @@ +bb4f92f6cbde6ee78e39ae35b0934da3b55e154d diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys new file mode 100644 index 0000000..d3e500e --- /dev/null +++ b/contrib/verify-commits/trusted-keys @@ -0,0 +1 @@ +498FA3769A88C4AD1B187A7428EB4B0FB7AAF6B0 diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh new file mode 100755 index 0000000..cfe4f11 --- /dev/null +++ b/contrib/verify-commits/verify-commits.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# Not technically POSIX-compliant due to use of "local", but almost every +# shell anyone uses today supports it, so its probably fine + +DIR=$(dirname "$0") +[ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0") + +VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") +REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") + +HAVE_FAILED=false +IS_SIGNED () { + if [ $1 = $VERIFIED_ROOT ]; then + return 0; + fi + if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then + export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1 + else + export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0 + fi + if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then + return 1; + fi + local PARENTS + PARENTS=$(git show -s --format=format:%P $1) + for PARENT in $PARENTS; do + if IS_SIGNED $PARENT > /dev/null; then + return 0; + fi + done + if ! "$HAVE_FAILED"; then + echo "No parent of $1 was signed with a trusted key!" > /dev/stderr + echo "Parents are:" > /dev/stderr + for PARENT in $PARENTS; do + git show -s $PARENT > /dev/stderr + done + HAVE_FAILED=true + fi + return 1; +} + +if [ x"$1" = "x" ]; then + TEST_COMMIT="HEAD" +else + TEST_COMMIT="$1" +fi + +IS_SIGNED "$TEST_COMMIT" +RES=$? +if [ "$RES" = 1 ]; then + if ! "$HAVE_FAILED"; then + echo "$TEST_COMMIT was not signed with a trusted key!" + fi +else + echo "There is a valid path from $TEST_COMMIT to $VERIFIED_ROOT where all commits are signed!" +fi + +exit $RES