You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
git-secret/src/_utils/_git_secret_tools.sh

830 lines
20 KiB
Bash

#!/usr/bin/env bash
# Folders:
_SECRETS_DIR=${SECRETS_DIR:-".gitsecret"}
# if SECRETS_DIR env var is set, use that instead of .gitsecret
# for full path to secrets dir, use _get_secrets_dir()
# from _git_secret_tools.sh
_SECRETS_DIR_KEYS="${_SECRETS_DIR}/keys"
_SECRETS_DIR_PATHS="${_SECRETS_DIR}/paths"
# Files:
_SECRETS_DIR_KEYS_TRUSTDB="${_SECRETS_DIR_KEYS}/trustdb.gpg"
_SECRETS_DIR_PATHS_MAPPING="${_SECRETS_DIR_PATHS}/mapping.cfg"
# shellcheck disable=SC2153
if [[ -n "$SECRETS_VERBOSE" ]] && [[ "$SECRETS_VERBOSE" -ne 0 ]]; then
# shellcheck disable=SC2034
_SECRETS_VERBOSE='1'
# _SECRETS_VERBOSE is empty or '1'.
# Empty means 'off', any other value means 'on'.
fi
: "${SECRETS_EXTENSION:=".secret"}"
# Commands:
: "${SECRETS_GPG_COMMAND:="gpg"}"
: "${SECRETS_CHECKSUM_COMMAND:="_os_based __sha256"}"
: "${SECRETS_OCTAL_PERMS_COMMAND:="_os_based __get_octal_perms"}"
: "${SECRETS_EPOCH_TO_DATE:="_os_based __epoch_to_date"}"
# Temp Dir:
: "${TMPDIR:=/tmp}"
# AWK scripts:
# shellcheck disable=SC2016
AWK_FSDB_HAS_RECORD='
BEGIN { FS=":"; OFS=":"; cnt=0; }
{
if ( key == $1 )
{
cnt++
}
}
END { if ( cnt > 0 ) print "0"; else print "1"; }
'
# shellcheck disable=SC2016
AWK_FSDB_RM_RECORD='
BEGIN { FS=":"; OFS=":"; }
{
if ( key != $1 )
{
print $1,$2;
}
}
'
# shellcheck disable=SC2016
AWK_FSDB_CLEAR_HASHES='
BEGIN { FS=":"; OFS=":"; }
{
print $1,"";
}
'
# shellcheck disable=SC2016
AWK_GPG_VER_CHECK='
/^gpg/{
version=$3
n=split(version,array,".")
if( n >= 2) {
if(array[1] >= 2)
{
if(array[2] >= 1)
{
print 1
}
else
{
print 0
}
}
else
{
print 0
}
}
else if(array[1] >= 2)
{
print 1
}
else
{
print 0
}
}
'
# This is 1 for gpg version 2.1 or greater, otherwise 0
GPG_VER_MIN_21="$($SECRETS_GPG_COMMAND --version | gawk "$AWK_GPG_VER_CHECK")"
# Bash:
# echos 0 if function exists, otherwise non-zero
function _function_exists {
local function_name="$1" # required
declare -f -F "$function_name" > /dev/null 2>&1
echo $?
}
# OS based:
function _os_based {
# Pass function name as first parameter.
# It will be invoked as os-based function with the postfix.
case "$(uname -s)" in
Darwin)
"$1_osx" "${@:2}"
;;
Linux)
"$1_linux" "${@:2}"
;;
MINGW*)
"$1_linux" "${@:2}"
;;
CYGWIN*)
"$1_linux" "${@:2}"
;;
FreeBSD)
"$1_freebsd" "${@:2}"
;;
# TODO: add MS Windows support.
# MINGW32*|MSYS*)
# $1_ms ${@:2}
# ;;
*)
_abort 'unsupported OS.'
;;
esac
}
# File System:
function _clean_windows_path {
# This function transforms windows paths to *nix paths
# such as c:\this\that.file -> /c/this/that/file
# shellcheck disable=SC2001
echo "$1" | sed 's#^\([a-zA-Z]\):/#/\1/#'
}
function _set_config {
# This function creates a line in the config, or alters it.
local key="$1" # required
local value="$2" # required
local filename="$3" # required
# The exit status is 0 (true) if the name was found, 1 (false) if not:
local contains
contains=$(grep -Fq "$key" "$filename"; echo "$?")
# Append or alter?
if [[ "$contains" -eq 0 ]]; then
_os_based __replace_in_file "$@"
elif [[ "$contains" -eq 1 ]]; then
echo "${key} = ${value}" >> "$filename"
fi
}
# this sets the global variable 'temporary_filename'
# currently this function is only used by 'hide'
function _temporary_file {
# This function creates temporary file
# which will be removed on system exit.
temporary_filename=$(_os_based __temp_file) # is not `local` on purpose.
trap 'if [[ -f "$temporary_filename" ]]; then if [[ -n "$_SECRETS_VERBOSE" ]] || [[ "$SECRETS_TEST_VERBOSE" == 1 ]]; then echo "git-secret: cleaning up: $temporary_filename"; fi; rm -f "$temporary_filename"; fi;' EXIT
}
# Helper function
function _gawk_inplace {
local parms="$*"
local dest_file
dest_file="$(echo "$parms" | gawk -v RS="'" -v FS="'" 'END{ gsub(/^\s+/,""); print $1 }')"
_temporary_file
bash -c "gawk ${parms}" > "$temporary_filename"
mv "$temporary_filename" "$dest_file"
}
# File System Database (fsdb):
function _get_record_filename {
# Returns 1st field from passed record
local record="$1"
local filename
filename=$(echo "$record" | awk -F: '{print $1}')
echo "$filename"
}
function _get_record_hash {
# Returns 2nd field from passed record
local record="$1"
local hash
hash=$(echo "$record" | awk -F: '{print $2}')
echo "$hash"
}
function _fsdb_has_record {
# First parameter is the key
# Second is the fsdb
local key="$1" # required
local fsdb="$2" # required
# 0 on contains, 1 for error.
gawk -v key="$key" "$AWK_FSDB_HAS_RECORD" "$fsdb"
}
function _fsdb_rm_record {
# First parameter is the key (filename)
# Second is the path to fsdb
local key="$1" # required
local fsdb="$2" # required
_gawk_inplace -v key="'$key'" "'$AWK_FSDB_RM_RECORD'" "$fsdb"
}
function _fsdb_clear_hashes {
# First parameter is the path to fsdb
local fsdb="$1" # required
_gawk_inplace "'$AWK_FSDB_CLEAR_HASHES'" "$fsdb"
}
# Manuals:
function _show_manual_for {
local function_name="$1" # required
man "git-secret-${function_name}"
exit 0
}
# Invalid options
function _invalid_option_for {
local function_name="$1" # required
man "git-secret-${function_name}"
exit 1
}
# VCS:
function _check_ignore {
local filename="$1" # required
local result
result="$(git check-ignore -q "$filename"; echo $?)"
# returns 1 when not ignored, and 0 when ignored
echo "$result"
}
function _git_normalize_filename {
local filename="$1" # required
local result
result=$(git ls-files --full-name -o "$filename")
echo "$result"
}
function _maybe_create_gitignore {
# This function creates '.gitignore' if it was missing.
local full_path
full_path=$(_prepend_root_path '.gitignore')
if [[ ! -f "$full_path" ]]; then
touch "$full_path"
fi
}
function _add_ignored_file {
# This function adds a line with the filename into the '.gitignore' file.
# It also creates '.gitignore' if it's not there
local filename="$1" # required
_maybe_create_gitignore
local full_path
full_path=$(_prepend_root_path '.gitignore')
printf '%q\n' "$filename" >> "$full_path"
}
function _is_inside_git_tree {
# Checks if we are working inside the `git` tree.
local result
result=$(git rev-parse --is-inside-work-tree > /dev/null 2>&1; echo $?)
echo "$result"
}
function _is_tracked_in_git {
local filename="$1" # required
local result
result="$(git ls-files --error-unmatch "$filename" >/dev/null 2>&1; echo $?)"
if [[ "$result" -eq 0 ]]; then
echo "1"
else
echo "0"
fi
}
# This can give unexpected .git dir when used in a _subdirectory_
# of another git repo; See #431 and #433.
function _get_git_root_path {
# We need this function to get the location of the `.git` folder,
# since `.gitsecret` (or value set by SECRETS_DIR env var)
# must be in the same dir.
local result
result=$(_clean_windows_path "$(git rev-parse --show-toplevel)")
echo "$result"
}
# Relative paths:
function _prepend_root_path {
# This function adds root path to any other path.
local path="$1" # required
local root_path
root_path=$(_get_git_root_path)
echo "$root_path/$path"
}
# if passed a name like 'filename.txt', returns a full path in the repo
# For #710: if we are in a subdir, fixup the path with the subdir
function _prepend_relative_root_path {
local path="$1" # required
local full_path
full_path=$(_prepend_root_path "$path")
local subdir
subdir=$(git rev-parse --show-prefix) # get the subdir of repo, like "subdir/"
if [ -n "$subdir" ]; then
full_path="$(dirname "$full_path")/${subdir}/$(basename "$full_path")"
fi
echo "$full_path"
}
function _get_secrets_dir {
_prepend_root_path "${_SECRETS_DIR}"
}
function _get_secrets_dir_keys {
_prepend_root_path "${_SECRETS_DIR_KEYS}"
}
function _get_secrets_dir_path {
_prepend_root_path "${_SECRETS_DIR_PATHS}"
}
function _get_secrets_dir_keys_trustdb {
_prepend_root_path "${_SECRETS_DIR_KEYS_TRUSTDB}"
}
function _get_secrets_dir_paths_mapping {
_prepend_root_path "${_SECRETS_DIR_PATHS_MAPPING}"
}
# Logic:
function _message {
local message="$1" # required
echo "git-secret: $message"
}
function _abort {
local message="$1" # required
local exit_code=${2:-"1"} # defaults to 1
>&2 echo "git-secret: abort: $message"
exit "$exit_code"
}
# _warn() sends warnings to stdout so user sees them
function _warn {
local message="$1" # required
>&2 echo "git-secret: warning: $message"
}
# _warn_or_abort "$error_message" "$exit_code" "$error_ok"
function _warn_or_abort {
local message="$1" # required
local exit_code=${2:-"1"} # defaults to 1
local error_ok=${3:-0} # can be 0 or 1
if [[ "$error_ok" -eq "0" ]]; then
if [[ "$exit_code" -eq "0" ]]; then
# if caller sends an exit_code of 0, we change it to 1 before aborting.
exit_code=1
fi
_abort "$message" "$exit_code"
else
_warn "$message" "$exit_code"
fi
}
function _find_and_remove_secrets_formatted {
local filenames
_list_all_added_files # sets array variable 'filenames'
for filename in "${filenames[@]}"; do
local path # absolute path
encrypted_filename=$(_get_encrypted_filename "$filename")
if [[ -f "$encrypted_filename" ]]; then
rm "$encrypted_filename"
if [[ -n "$_SECRETS_VERBOSE" ]]; then
echo "git-secret: deleted: $encrypted_filename"
fi
fi
done
}
# this sets the global array variable 'filenames'
function _list_all_added_files {
local path_mappings
path_mappings=$(_get_secrets_dir_paths_mapping)
if [[ ! -s "$path_mappings" ]]; then
_abort "path_mappings file is missing or empty: $path_mappings"
fi
local filename
filenames=() # not local
while read -r line; do
filename=$(_get_record_filename "$line")
filenames+=("$filename")
done < "$path_mappings"
declare -a filenames # so caller can get list from filenames array
}
function _secrets_dir_exists {
# This function checks if "$_SECRETS_DIR" exists and.
local full_path
full_path=$(_get_secrets_dir)
if [[ ! -d "$full_path" ]]; then
local name
name=$(basename "$full_path")
_abort "directory '$name' does not exist. Use 'git secret init' to initialize git-secret"
fi
}
function _secrets_dir_is_not_ignored {
# This function checks that "$_SECRETS_DIR" is not ignored.
local git_secret_dir
git_secret_dir=$(_get_secrets_dir)
local ignores
ignores=$(_check_ignore "$git_secret_dir")
if [[ ! $ignores -eq 1 ]]; then
_abort "entry already in .gitignore: $git_secret_dir"
fi
}
function _exe_is_busybox {
local exe
exe="$1"
# we assume stat is from busybox if it's a symlink
local is_busybox=0
local stat_path
stat_path=$(command -v "$exe")
if [ -L "$stat_path" ]; then
is_busybox=1
fi
echo "$is_busybox"
}
# this is used by just about every command
function _user_required {
# This function does a bunch of validations:
# 1. It calls `_secrets_dir_exists` to verify that "$_SECRETS_DIR" exists.
# 2. It ensures that "$_SECRETS_DIR_KEYS_TRUSTDB" exists.
# 3. It ensures that there are added public keys.
_secrets_dir_exists
local trustdb
trustdb=$(_get_secrets_dir_keys_trustdb)
local error_message="no public keys for users found. run 'git secret tell email@address'."
if [[ ! -f "$trustdb" ]]; then
_abort "$error_message"
fi
local secrets_dir_keys
secrets_dir_keys=$(_get_secrets_dir_keys)
# see https://github.com/bats-core/bats-core#file-descriptor-3-read-this-if-bats-hangs for info about 3>&-
local keys_exist
keys_exist=$($SECRETS_GPG_COMMAND --homedir "$secrets_dir_keys" --no-permission-warning -n --list-keys 3>&-)
local exit_code=$?
if [[ -z "$keys_exist" ]]; then
_abort "$error_message"
fi
if [[ "$exit_code" -ne 0 ]]; then
# this might catch corner case where gpg --list-keys shows
# 'gpg: skipped packet of type 12 in keybox' warnings but succeeds?
# See #136
echo "$keys_exist" # show whatever _did_ come out of gpg
_abort "problem listing public keys with gpg: exit code $exit_code"
fi
}
# note: this has the same 'username matching' issue described in
# https://github.com/sobolevn/git-secret/issues/268
# where it will match emails that have other emails as substrings.
# we need to use fingerprints for a unique key id with gpg.
function _get_user_key_expiry {
# This function returns the user's key's expiry, as an epoch.
# It will return the empty string
# if there is no expiry date for the user's key
local username="$1"
local line
local secrets_dir_keys
secrets_dir_keys=$(_get_secrets_dir_keys)
# 3>&- closes fd 3 for bats, see https://github.com/bats-core/bats-core#file-descriptor-3-read-this-if-bats-hangs
line=$($SECRETS_GPG_COMMAND --homedir "$secrets_dir_keys" --no-permission-warning --list-public-keys --with-colon --fixed-list-mode "$username" | grep ^pub: 3>&-)
local expiry_epoch
expiry_epoch=$(echo "$line" | cut -d: -f7)
echo "$expiry_epoch"
}
function _assert_keyring_contains_emails {
local homedir="$1"
local keyring_name="$2"
local emails="$3"
# 1 here means 'expect $emails in keyring':
_assert_keyring_emails "$homedir" "$keyring_name" "$emails" 1
}
function _assert_keyring_doesnt_contain_emails {
local homedir="$1"
local keyring_name="$2"
local emails="$3"
# 0 here means 'don't expect $emails in keyring':
_assert_keyring_emails "$homedir" "$keyring_name" "$emails" 0
}
function _assert_keyring_contains_emails_at_least_once {
local homedir=$1
local keyring_name=$2
local emails=$3
_assert_keyring_emails "$homedir" "$keyring_name" "$emails" 1 1 # expect the email at least once in the keyring
}
function _assert_keyring_emails {
local homedir="$1"
local keyring_name="$2"
local emails="$3"
# set this to:
# 0 to not expect the email in the keyring;
# 1 to expect the email in the keyring
local expected="$4"
local allow_duplicates=$5 # set this to 0 to not allow duplicate emails in the keyring when processing assertion (optional)
local gpg_uids
gpg_uids=$(_get_users_in_gpg_keyring "$homedir")
for email in "${emails[@]}"; do
if [[ $email != *"@"* ]]; then
_abort "does not appear to be an email: $email"
fi
local emails_found=0
for uid in $gpg_uids; do
if [[ "$uid" == "$email" ]]; then
emails_found=$((emails_found+1))
fi
done
if [[ $expected -eq 1 ]]; then
if [[ $emails_found -eq 0 ]]; then
_abort "no key found in gpg $keyring_name for: $email"
elif [[ $emails_found -gt 1 ]]; then
if [[ $allow_duplicates -ne 1 ]]; then
_abort "$emails_found keys found in gpg $keyring_name for: $email"
fi
fi
else
if [[ $emails_found -gt 0 ]]; then
_abort "$emails_found keys found in gpg $keyring_name for: $email"
fi
fi
done
}
function _get_encrypted_filename {
local filename
filename="$(dirname "$1")/$(basename "$1" "$SECRETS_EXTENSION")"
echo "${filename}${SECRETS_EXTENSION}" | sed -e 's#^\./##'
}
# this is used throughout this file, and in 'whoknows'
function _get_users_in_gpg_keyring {
# show the users in the gpg keyring.
# `whoknows` command uses it internally.
# parses the `gpg` public keys
local homedir=$1
local result
local args=()
if [[ -n "$homedir" ]]; then
args+=( "--homedir" "$homedir" )
fi
## We use --fixed-list-mode so older versions of gpg emit 'uid:' lines.
## Gawk splits on colon as --with-colon, matches field 1 as 'uid',
result=$($SECRETS_GPG_COMMAND "${args[@]}" --no-permission-warning --list-public-keys --with-colon --fixed-list-mode | \
gawk -F: '$1=="uid"' )
local emails
emails=$(_extract_emails_from_gpg_output "$result")
# For #508 / #552: warn user if gpg indicates keys are one of:
# i=invalid, d=disabled, r=revoked, e=expired, n=not valid
# See https://github.com/gpg/gnupg/blob/master/doc/DETAILS#field-2---validity # for more on gpg 'validity codes'.
local invalid_lines
invalid_lines=$(echo "$result" | gawk -F: '$2=="i" || $2=="d" || $2=="r" || $2=="e" || $2=="n"')
local emails_with_invalid_keys
emails_with_invalid_keys=$(_extract_emails_from_gpg_output "$invalid_lines")
if [[ -n "$emails_with_invalid_keys" ]]; then
_warn "at least one key for email(s) is revoked, expired, or otherwise invalid: $emails_with_invalid_keys"
fi
echo "$emails"
}
function _extract_emails_from_gpg_output {
local result=$1
# gensub() outputs email from <> within field 10, "User-ID". If there's no <>, then field is just an email address
# (and maybe a comment) and we pass it through.
# Sed at the end removes any 'comment' that appears in parentheses, for #530
# 3>&- closes fd 3 for bats, see https://github.com/bats-core/bats-core#file-descriptor-3-read-this-if-bats-hangs
local emails
emails=$(echo "$result" | gawk -F: '{print gensub(/.*<(.*)>.*/, "\\1", "g", $10); }' | sed 's/([^)]*)//g' 3>&-)
echo "$emails"
}
function _get_users_in_gitsecret_keyring {
# show the users in the gitsecret keyring.
local secrets_dir_keys
secrets_dir_keys=$(_get_secrets_dir_keys)
local result
result=$(_get_users_in_gpg_keyring "$secrets_dir_keys")
echo "$result"
}
function _get_recipients {
# This function is required to create an encrypted file for different users.
# These users are called 'recipients' in the `gpg` terms.
# It basically just parses the `gpg` public keys
local result
# put -r before each user:
result=$(_get_users_in_gitsecret_keyring | sed 's/^/-r/')
echo "$result"
}
function _decrypt {
# required:
local filename="$1"
# optional:
local write_to_file=${2:-1} # can be 0 or 1
local force=${3:-0} # can be 0 or 1
local homedir=${4:-""}
local passphrase=${5:-""}
local error_ok=${6:-0} # can be 0 or 1
local encrypted_filename
encrypted_filename=$(_get_encrypted_filename "$filename")
if [ ! -f "$encrypted_filename" ]; then
_warn_or_abort "cannot find file to decrypt: $encrypted_filename" "1" "$error_ok"
fi
local args=( "--use-agent" "--decrypt" )
if [[ "$write_to_file" -eq 1 ]]; then
args+=( "-o" "$filename" )
fi
if [[ "$force" -eq 1 ]]; then
args+=( "--yes" )
fi
if [[ -n "$homedir" ]]; then
args+=( "--homedir" "$homedir" )
fi
if [[ "$GPG_VER_MIN_21" -eq 1 ]]; then
if [[ -n "$SECRETS_PINENTRY" ]]; then
args+=( "--pinentry-mode" "$SECRETS_PINENTRY" )
else
args+=( "--pinentry-mode" "loopback" )
fi
fi
if [[ -z "$_SECRETS_VERBOSE" ]]; then
# we no longer use --no-permission-warning here, for #811
args+=( "--quiet" )
fi
set +e # disable 'set -e' so we can capture exit_code
#echo "# gpg passphrase: $passphrase" >&3
local exit_code
if [[ -n "$passphrase" ]]; then
exec 5<<<"$passphrase" # use 5, because descriptors 3 and 4 are used by bats
$SECRETS_GPG_COMMAND "${args[@]}" --batch --yes --no-tty --passphrase-fd 5 "$encrypted_filename"
exit_code=$?
exec 5>&- # close file descriptor 5
else
$SECRETS_GPG_COMMAND "${args[@]}" "$encrypted_filename"
exit_code=$?
fi
set -e # re-enable set -e
# note that according to https://github.com/sobolevn/git-secret/issues/238 ,
# it's possible for gpg to return a 0 exit code but not have decrypted the file
#echo "# gpg exit code: $exit_code, error_ok: $error_ok" >&3
if [[ "$exit_code" -ne "0" ]]; then
local msg="problem decrypting file with gpg: exit code $exit_code: $filename"
_warn_or_abort "$msg" "$exit_code" "$error_ok"
fi
# at this point the file should be written to disk or output to stdout
}