diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 46ee5ee..dc3fd52 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -2,12 +2,14 @@ #TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ? #Check dryruns with nosuffix mode for timestampList +#WIP: currently TRAVIS_RUN debug lines in n_osync and run_tests for conflictLog + PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2018 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-beta1 -PROGRAM_BUILD=2018100105 +PROGRAM_BUILD=2018100106 IS_STABLE=no ##### Execution order #__WITH_PARANOIA_DEBUG @@ -40,22 +42,10 @@ IS_STABLE=no # UnlockReplicas yes #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.3.0-RC1 -_OFUNCTIONS_BUILD=2018100103 +_OFUNCTIONS_VERSION=2.3.0-RC2 +_OFUNCTIONS_BUILD=2018100204 _OFUNCTIONS_BOOTSTRAP=true -## To use in a program, define the following variables: -## PROGRAM=program-name -## INSTANCE_ID=program-instance-name -## _DEBUG=yes/no -## _LOGGER_SILENT=true/false -## _LOGGER_VERBOSE=true/false -## _LOGGER_ERR_ONLY=true/false -## _LOGGER_PREFIX="date"/"time"/"" - -## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel -## When called from subprocesses, variable of main process cannot be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID.$TSTAMP - if ! type "$BASH" > /dev/null; then echo "Please run this script only with bash shell. Tested on bash >= 3.2" exit 127 @@ -131,7 +121,6 @@ else RUN_DIR=. fi -#### PoorMansRandomGenerator SUBSET #### # Get a random number on Windows BusyBox alike, also works on most Unixes function PoorMansRandomGenerator { local digits="${1}" # The number of digits to generate @@ -146,7 +135,6 @@ function PoorMansRandomGenerator { fi done } -#### PoorMansRandomGenerator SUBSET END #### # Initial TSTMAP value before function declaration TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) @@ -159,13 +147,6 @@ set -o pipefail set -o errtrace -function Dummy { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG - - sleep $SLEEP_TIME -} - - # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array function joinString { @@ -411,6 +392,35 @@ function KillAllChilds { return $errorcount } +function CleanUp { + __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG + + if [ "$_DEBUG" != "yes" ]; then + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP" + # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" + fi +} + +function GenericTrapQuit { + local exitcode=0 + + # Get ERROR / WARN alert flags from subprocesses that call Logger + if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + WARN_ALERT=true + exitcode=2 + fi + if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + ERROR_ALERT=true + exitcode=1 + fi + + CleanUp + exit $exitcode +} + + + # osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending function SendAlert { local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run @@ -1216,16 +1226,6 @@ function ExecTasks { fi } -function CleanUp { - __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG - - if [ "$_DEBUG" != "yes" ]; then - rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP" - # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) - rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" - fi -} - # Usage: var=$(StripSingleQuotes "$var") function StripSingleQuotes { local string="${1}" @@ -1264,27 +1264,37 @@ function EscapeDoubleQuotes { echo "${value//\"/\\\"}" } -function IsNumericExpand { - eval "local value=\"${1}\"" # Needed eval so variable variables can be processed - - if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then - echo 1 - else - echo 0 - fi -} - # Usage [ $(IsNumeric $var) -eq 1 ] function IsNumeric { local value="${1}" - if [[ $value =~ ^[0-9]+([.][0-9]+)?$ ]]; then - echo 1 + if type expr > /dev/null 2>&1; then + expr "$value" : "^[-+]\?[0-9]*\.\?[0-9]\+$" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo 1 + else + echo 0 + fi else - echo 0 + if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then + echo 1 + else + echo 0 + fi fi } +function IsNumericExpand { + eval "local value=\"${1}\"" # Needed eval so variable variables can be processed + + echo $(IsNumeric "$value") + # if [[ $value =~ ^-?[0-9]+([.][0-9]+)?$ ]]; then + # echo 1 + #else + # echo 0 + #fi +} + # Function is busybox compatible since busybox ash does not understand direct regex, we use expr function IsInteger { local value="${1}" @@ -1874,9 +1884,9 @@ function RsyncPatternsAdd { rest="${rest#*$PATH_SEPARATOR_CHAR}" fi if [ "$RSYNC_PATTERNS" == "" ]; then - RSYNC_PATTERNS="--"$patternType"=\"$str\"" + RSYNC_PATTERNS="--$patternType=\"$str\"" else - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType=\"$str\"" fi done set +f @@ -1893,7 +1903,7 @@ function RsyncPatternsFromAdd { fi if [ -e "$patternFrom" ]; then - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType-from=\"$patternFrom\"" fi } @@ -1983,14 +1993,10 @@ function PostInit { # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - #WIP - #SSH_REVERSE_CMD="$(type -p ssh) $SSH_COMP -q -i $INITIATOR_SSH_RSA_PRIVATE_KEY $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" elif [ -f "$SSH_PASSWORD_FILE" ]; then SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - #WIP - #SSH_REVERSE_CMD="$(type -p sshpass) -f $INITIATOR_SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT" SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT" else @@ -2221,7 +2227,7 @@ function VerComp () { return 1 fi - if [[ $1 == $2 ]] + if [[ "$1" == "$2" ]] then echo 0 return @@ -3580,6 +3586,15 @@ function conflictList { join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP" retval=$? + + #WIP + if [ $TRAVIS_RUN == true ]; then + echo "conflictList debug retval=$retval" + cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" + cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" + cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP" + fi + if [ $retval -ne 0 ]; then Logger "Cannot create conflict list file." "ERROR" return $retval @@ -5169,6 +5184,12 @@ function LogConflicts { local subject local body + #WIP + if [ $TRAVIS_RUN == true ]; then + cat "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" + fi + + # We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" @@ -5176,6 +5197,7 @@ function LogConflicts { ( _LOGGER_PREFIX="" + if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" while read -r line; do diff --git a/install.sh b/install.sh index 570ed76..cf89a42 100755 --- a/install.sh +++ b/install.sh @@ -10,83 +10,113 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2018090301 +SCRIPT_BUILD=2018100201 INSTANCE_ID="installer-$SCRIPT_BUILD" ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Please adapt this to fit your distro needs +_OFUNCTIONS_VERSION=2.3.0-RC2 +_OFUNCTIONS_BUILD=2018100204 _OFUNCTIONS_BOOTSTRAP=true -# Get current install.sh path from http://stackoverflow.com/a/246128/2635443 -SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if ! type "$BASH" > /dev/null; then + echo "Please run this script only with bash shell. Tested on bash >= 3.2" + exit 127 +fi + +## Correct output of sort command (language agnostic sorting) +export LC_ALL=C + +## Default umask for file creation +umask 0077 +# Standard alert mail body +MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." + +# Environment variables that can be overriden by programs +_DRYRUN=false _LOGGER_SILENT=false -_STATS=1 -ACTION="install" -FAKEROOT="" +_LOGGER_VERBOSE=false +_LOGGER_ERR_ONLY=false +_LOGGER_PREFIX="date" +if [ "$KEEP_LOGGING" == "" ]; then + KEEP_LOGGING=1801 +fi -function GetCommandlineArguments { - for i in "$@"; do - case $i in - --prefix=*) - FAKEROOT="${i##*=}" - ;; - --silent) - _LOGGER_SILENT=true - ;; - --no-stats) - _STATS=0 - ;; - --remove) - ACTION="uninstall" - ;; - --help|-h|-?) - Usage - ;; - *) - Logger "Unknown option '$i'" "SIMPLE" - Usage - exit - ;; - esac - done -} +# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags +ERROR_ALERT=false +WARN_ALERT=false -GetCommandlineArguments "$@" -CONF_DIR=$FAKEROOT/etc/$PROGRAM -BIN_DIR="$FAKEROOT/usr/local/bin" -SERVICE_DIR_INIT=$FAKEROOT/etc/init.d -# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora -SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system -SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user -SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d +## allow debugging from command line with _DEBUG=yes +if [ ! "$_DEBUG" == "yes" ]; then + _DEBUG=no + _LOGGER_VERBOSE=false +else + trap 'TrapError ${LINENO} $?' ERR + _LOGGER_VERBOSE=true +fi -if [ "$PROGRAM" == "osync" ]; then - SERVICE_NAME="osync-srv" -elif [ "$PROGRAM" == "pmocr" ]; then - SERVICE_NAME="pmocr-srv" +if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console + SLEEP_TIME=.05 fi -SERVICE_FILE_INIT="$SERVICE_NAME" -SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service" -SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user" -SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc" +SCRIPT_PID=$$ -## Generic code +LOCAL_USER=$(whoami) +LOCAL_HOST=$(hostname) -## Default log file -if [ -w "$FAKEROOT/var/log" ]; then - LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log" +if [ "$PROGRAM" == "" ]; then + PROGRAM="ofunctions" +fi + +## Default log file until config file is loaded +if [ -w /var/log ]; then + LOG_FILE="/var/log/$PROGRAM.log" elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then - LOG_FILE="$HOME/$PROGRAM-install.log" + LOG_FILE="$HOME/$PROGRAM.log" +elif [ -w . ]; then + LOG_FILE="./$PROGRAM.log" else - LOG_FILE="./$PROGRAM-install.log" + LOG_FILE="/tmp/$PROGRAM.log" +fi + +## Default directory where to store temporary run files +if [ -w /tmp ]; then + RUN_DIR=/tmp +elif [ -w /var/tmp ]; then + RUN_DIR=/var/tmp +else + RUN_DIR=. fi -#### RemoteLogger SUBSET #### +# Get a random number on Windows BusyBox alike, also works on most Unixes +function PoorMansRandomGenerator { + local digits="${1}" # The number of digits to generate + local number + + # Some read bytes can't be used, se we read twice the number of required bytes + dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do + number=$number$(printf "%d" "'$char") + if [ ${#number} -ge $digits ]; then + echo ${number:0:$digits} + break; + fi + done +} + +# Initial TSTMAP value before function declaration +TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) + +# Default alert attachment filename +ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log" + +# Set error exit code if a piped command fails +set -o pipefail +set -o errtrace + # Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array # usage: joinString separaratorChar Array @@ -170,17 +200,11 @@ function RemoteLogger { _Logger "" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG - _Logger "" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG else _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true _Logger "" "Value was: $prefix$value" true fi } -#### RemoteLogger SUBSET END #### # General log function with log levels: @@ -249,11 +273,6 @@ function Logger { _Logger "$prefix$value" "$prefix$value" return fi - elif [ "$level" == "PARANOIA_DEBUG" ]; then #__WITH_PARANOIA_DEBUG - if [ "$_PARANOIA_DEBUG" == "yes" ]; then #__WITH_PARANOIA_DEBUG - _Logger "$prefix$value" "$prefix\e[35m$value\e[0m" #__WITH_PARANOIA_DEBUG - return #__WITH_PARANOIA_DEBUG - fi #__WITH_PARANOIA_DEBUG elif [ "$level" == "SIMPLE" ]; then if [ "$_LOGGER_SILENT" == true ]; then _Logger "$preix$value" @@ -266,6 +285,167 @@ function Logger { _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } + +# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X +function KillChilds { + local pid="${1}" # Parent pid to kill childs + local self="${2:-false}" # Should parent be killed too ? + + # Paranoid checks, we can safely assume that $pid should not be 0 nor 1 + if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then + Logger "Bogus pid given [$pid]." "CRITICAL" + return 1 + fi + + if kill -0 "$pid" > /dev/null 2>&1; then + if children="$(pgrep -P "$pid")"; then + if [[ "$pid" == *"$children"* ]]; then + Logger "Bogus pgrep implementation." "CRITICAL" + children="${children/$pid/}" + fi + for child in $children; do + KillChilds "$child" true + done + fi + fi + + # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing + if [ "$self" == true ]; then + # We need to check for pid again because it may have disappeared after recursive function call + if kill -0 "$pid" > /dev/null 2>&1; then + kill -s TERM "$pid" + Logger "Sent SIGTERM to process [$pid]." "DEBUG" + if [ $? != 0 ]; then + sleep 15 + Logger "Sending SIGTERM to process [$pid] failed." "DEBUG" + kill -9 "$pid" + if [ $? != 0 ]; then + Logger "Sending SIGKILL to process [$pid] failed." "DEBUG" + return 1 + fi # Simplify the return 0 logic here + else + return 0 + fi + else + return 0 + fi + else + return 0 + fi +} + +function KillAllChilds { + local pids="${1}" # List of parent pids to kill separated by semi-colon + local self="${2:-false}" # Should parent be killed too ? + + + local errorcount=0 + + IFS=';' read -a pidsArray <<< "$pids" + for pid in "${pidsArray[@]}"; do + KillChilds $pid $self + if [ $? != 0 ]; then + errorcount=$((errorcount+1)) + fi + done + return $errorcount +} + +function CleanUp { + + if [ "$_DEBUG" != "yes" ]; then + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP" + # Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements) + rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp" + fi +} + +function GenericTrapQuit { + local exitcode=0 + + # Get ERROR / WARN alert flags from subprocesses that call Logger + if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + WARN_ALERT=true + exitcode=2 + fi + if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + ERROR_ALERT=true + exitcode=1 + fi + + CleanUp + exit $exitcode +} + + + +# Get current install.sh path from http://stackoverflow.com/a/246128/2635443 +SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +_LOGGER_SILENT=false +_STATS=1 +ACTION="install" +FAKEROOT="" + +function GetCommandlineArguments { + for i in "$@"; do + case $i in + --prefix=*) + FAKEROOT="${i##*=}" + ;; + --silent) + _LOGGER_SILENT=true + ;; + --no-stats) + _STATS=0 + ;; + --remove) + ACTION="uninstall" + ;; + --help|-h|-?) + Usage + ;; + *) + Logger "Unknown option '$i'" "SIMPLE" + Usage + exit + ;; + esac + done +} + +GetCommandlineArguments "$@" + +CONF_DIR=$FAKEROOT/etc/$PROGRAM +BIN_DIR="$FAKEROOT/usr/local/bin" +SERVICE_DIR_INIT=$FAKEROOT/etc/init.d +# Should be /usr/lib/systemd/system, but /lib/systemd/system exists on debian & rhel / fedora +SERVICE_DIR_SYSTEMD_SYSTEM=$FAKEROOT/lib/systemd/system +SERVICE_DIR_SYSTEMD_USER=$FAKEROOT/etc/systemd/user +SERVICE_DIR_OPENRC=$FAKEROOT/etc/init.d + +if [ "$PROGRAM" == "osync" ]; then + SERVICE_NAME="osync-srv" +elif [ "$PROGRAM" == "pmocr" ]; then + SERVICE_NAME="pmocr-srv" +fi + +SERVICE_FILE_INIT="$SERVICE_NAME" +SERVICE_FILE_SYSTEMD_SYSTEM="$SERVICE_NAME@.service" +SERVICE_FILE_SYSTEMD_USER="$SERVICE_NAME@.service.user" +SERVICE_FILE_OPENRC="$SERVICE_NAME-openrc" + +## Generic code + +## Default log file +if [ -w "$FAKEROOT/var/log" ]; then + LOG_FILE="$FAKEROOT/var/log/$PROGRAM-install.log" +elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then + LOG_FILE="$HOME/$PROGRAM-install.log" +else + LOG_FILE="./$PROGRAM-install.log" +fi + ## Modified version of https://gist.github.com/cdown/1163649 function UrlEncode { local length="${#1}" @@ -658,19 +838,27 @@ function Usage { exit 127 } +function TrapQuit { + local exitcode=0 + + # Get ERROR / WARN alert flags from subprocesses that call Logger + if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then + WARN_ALERT=true + exitcode=2 + fi + if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then + ERROR_ALERT=true + exitcode=1 + fi + + CleanUp + exit $exitcode +} + ############################## Script entry point -if [ "$LOGFILE" == "" ]; then - if [ -w /var/log ]; then - LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log" - elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then - LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log" - else - LOG_FILE="./$PROGRAM.$INSTANCE_ID.log" - fi -else - LOG_FILE="$LOGFILE" -fi +trap TrapQuit TERM EXIT HUP QUIT + if [ ! -w "$(dirname $LOG_FILE)" ]; then echo "Cannot write to log [$(dirname $LOG_FILE)]." else diff --git a/osync-batch.sh b/osync-batch.sh index f1f55f4..18dc441 100755 --- a/osync-batch.sh +++ b/osync-batch.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash SUBPROGRAM=osync PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones -AUTHOR="(L) 2013-2017 by Orsiris de Jong" +AUTHOR="(L) 2013-2018 by Orsiris de Jong" CONTACT="http://www.netpower.fr - ozy@netpower.fr" -PROGRAM_BUILD=2016120401 +PROGRAM_BUILD=2018100101 ## Runs an osync /obackup instance for every conf file found ## If an instance fails, run it again if time permits @@ -26,34 +26,185 @@ else LOG_FILE=./$SUBPROGRAM-batch.log fi +## Default directory where to store temporary run files +if [ -w /tmp ]; then + RUN_DIR=/tmp +elif [ -w /var/tmp ]; then + RUN_DIR=/var/tmp +else + RUN_DIR=. +fi + # No need to edit under this line ############################################################## -function _logger { - local value="${1}" # What to log - echo -e "$value" >> "$LOG_FILE" +#### RemoteLogger SUBSET #### + +# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array +# usage: joinString separaratorChar Array +function joinString { + local IFS="$1"; shift; echo "$*"; +} + +# Sub function of Logger +function _Logger { + local logValue="${1}" # Log to file + local stdValue="${2}" # Log to screeen + local toStdErr="${3:-false}" # Log to stderr instead of stdout + + if [ "$logValue" != "" ]; then + echo -e "$logValue" >> "$LOG_FILE" + + # Build current log file for alerts if we have a sufficient environment + if [ "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" != "" ]; then + echo -e "$logValue" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" + fi + fi + + if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then + if [ $toStdErr == true ]; then + # Force stderr color in subshell + (>&2 echo -e "$stdValue") + + else + echo -e "$stdValue" + fi + fi +} + +# Remote logger similar to below Logger, without log to file and alert flags +function RemoteLogger { + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level + local retval="${3:-undef}" # optional return value of command + + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then + prefix="R $(date) - " + else + prefix="" + fi + + if [ "$level" == "CRITICAL" ]; then + _Logger "" "$prefix\e[1;33;41m$value\e[0m" true + if [ $_DEBUG == "yes" ]; then + _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true + fi + return + elif [ "$level" == "ERROR" ]; then + _Logger "" "$prefix\e[31m$value\e[0m" true + if [ $_DEBUG == "yes" ]; then + _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true + fi + return + elif [ "$level" == "WARN" ]; then + _Logger "" "$prefix\e[33m$value\e[0m" true + if [ $_DEBUG == "yes" ]; then + _Logger -e "" "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$" true + fi + return + elif [ "$level" == "NOTICE" ]; then + if [ $_LOGGER_ERR_ONLY != true ]; then + _Logger "" "$prefix$value" + fi + return + elif [ "$level" == "VERBOSE" ]; then + if [ $_LOGGER_VERBOSE == true ]; then + _Logger "" "$prefix$value" + fi + return + elif [ "$level" == "ALWAYS" ]; then + _Logger "" "$prefix$value" + return + elif [ "$level" == "DEBUG" ]; then + if [ "$_DEBUG" == "yes" ]; then + _Logger "" "$prefix$value" + return + fi + else + _Logger "" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "" "Value was: $prefix$value" true + fi } +#### RemoteLogger SUBSET END #### + +# General log function with log levels: + +# Environment variables +# _LOGGER_SILENT: Disables any output to stdout & stderr +# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel +# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout +# Loglevels +# Except for VERBOSE, all loglevels are ALWAYS sent to log file + +# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged +# NOTICE sent to stdout +# VERBOSE sent to stdout if _LOGGER_VERBOSE = true +# ALWAYS is sent to stdout unless _LOGGER_SILENT = true +# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=yes +# SIMPLE is a wrapper for QuickLogger that does not use advanced functionality function Logger { - local value="${1}" # What to log - local level="${2}" # Log level: DEBUG, NOTICE, WARN, ERROR, CRITIAL + local value="${1}" # Sentence to log (in double quotes) + local level="${2}" # Log level + local retval="${3:-undef}" # optional return value of command - prefix="$(date) - " + if [ "$_LOGGER_PREFIX" == "time" ]; then + prefix="TIME: $SECONDS - " + elif [ "$_LOGGER_PREFIX" == "date" ]; then + prefix="$(date '+%Y-%m-%d %H:%M:%S') - " + else + prefix="" + fi + + ## Obfuscate _REMOTE_TOKEN in logs (for ssh_filter usage only in osync and obackup) + value="${value/env _REMOTE_TOKEN=$_REMOTE_TOKEN/__(o_O)__}" + value="${value/env _REMOTE_TOKEN=\$_REMOTE_TOKEN/__(o_O)__}" if [ "$level" == "CRITICAL" ]; then - _logger "$prefix\e[41m$value\e[0m" + _Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true + ERROR_ALERT=true + # ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag. + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + return elif [ "$level" == "ERROR" ]; then - _logger "$prefix\e[91m$value\e[0m" + _Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true + ERROR_ALERT=true + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP" + return elif [ "$level" == "WARN" ]; then - _logger "$prefix\e[93m$value\e[0m" + _Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true + WARN_ALERT=true + echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP" + return elif [ "$level" == "NOTICE" ]; then - _logger "$prefix$value" + if [ "$_LOGGER_ERR_ONLY" != true ]; then + _Logger "$prefix$value" "$prefix$value" + fi + return + elif [ "$level" == "VERBOSE" ]; then + if [ $_LOGGER_VERBOSE == true ]; then + _Logger "$prefix($level):$value" "$prefix$value" + fi + return + elif [ "$level" == "ALWAYS" ]; then + _Logger "$prefix$value" "$prefix$value" + return elif [ "$level" == "DEBUG" ]; then - if [ "$DEBUG" == "yes" ]; then - _logger "$prefix$value" + if [ "$_DEBUG" == "yes" ]; then + _Logger "$prefix$value" "$prefix$value" + return + fi + elif [ "$level" == "SIMPLE" ]; then + if [ "$_LOGGER_SILENT" == true ]; then + _Logger "$preix$value" + else + _Logger "$preix$value" "$prefix$value" fi + return else - _logger "\e[41mLogger function called without proper loglevel.\e[0m" - _logger "$prefix$value" + _Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true + _Logger "Value was: $prefix$value" "Value was: $prefix$value" true fi } diff --git a/osync.sh b/osync.sh index d763d54..ae01f1d 100755 --- a/osync.sh +++ b/osync.sh @@ -2,17 +2,19 @@ #TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ? #Check dryruns with nosuffix mode for timestampList +#WIP: currently TRAVIS_RUN debug lines in n_osync and run_tests for conflictLog + PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2018 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" PROGRAM_VERSION=1.3.0-beta1 -PROGRAM_BUILD=2018100105 +PROGRAM_BUILD=2018100106 IS_STABLE=no _OFUNCTIONS_VERSION=2.3.0-RC1 -_OFUNCTIONS_BUILD=2018100103 +_OFUNCTIONS_BUILD=2018100105 _OFUNCTIONS_BOOTSTRAP=true ## To use in a program, define the following variables: @@ -98,7 +100,6 @@ else RUN_DIR=. fi -#### PoorMansRandomGenerator SUBSET #### # Get a random number on Windows BusyBox alike, also works on most Unixes function PoorMansRandomGenerator { local digits="${1}" # The number of digits to generate @@ -113,7 +114,6 @@ function PoorMansRandomGenerator { fi done } -#### PoorMansRandomGenerator SUBSET END #### # Initial TSTMAP value before function declaration TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 4) @@ -1729,9 +1729,9 @@ function RsyncPatternsAdd { rest="${rest#*$PATH_SEPARATOR_CHAR}" fi if [ "$RSYNC_PATTERNS" == "" ]; then - RSYNC_PATTERNS="--"$patternType"=\"$str\"" + RSYNC_PATTERNS="--$patternType=\"$str\"" else - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"=\"$str\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType=\"$str\"" fi done set +f @@ -1747,7 +1747,7 @@ function RsyncPatternsFromAdd { fi if [ -e "$patternFrom" ]; then - RSYNC_PATTERNS="$RSYNC_PATTERNS --"$patternType"-from=\"$patternFrom\"" + RSYNC_PATTERNS="$RSYNC_PATTERNS --$patternType-from=\"$patternFrom\"" fi } @@ -1834,14 +1834,10 @@ function PostInit { # Define remote commands if [ -f "$SSH_RSA_PRIVATE_KEY" ]; then SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - #WIP - #SSH_REVERSE_CMD="$(type -p ssh) $SSH_COMP -q -i $INITIATOR_SSH_RSA_PRIVATE_KEY $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT" SCP_CMD="$(type -p scp) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY -P $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p ssh) $SSH_COMP -q -i $SSH_RSA_PRIVATE_KEY $SSH_OPTS -p $REMOTE_PORT" elif [ -f "$SSH_PASSWORD_FILE" ]; then SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $REMOTE_USER@$REMOTE_HOST -p $REMOTE_PORT" - #WIP - #SSH_REVERSE_CMD="$(type -p sshpass) -f $INITIATOR_SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS $INITIATOR_REMOTE_USER@$INITIATOR_REMOTE_HOST -p $INITIATOR_REMOTE_PORT" SCP_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p scp) $SSH_COMP -q -P $REMOTE_PORT" RSYNC_SSH_CMD="$(type -p sshpass) -f $SSH_PASSWORD_FILE $(type -p ssh) $SSH_COMP -q $SSH_OPTS -p $REMOTE_PORT" else @@ -2070,7 +2066,7 @@ function VerComp () { return 1 fi - if [[ $1 == $2 ]] + if [[ "$1" == "$2" ]] then echo 0 return @@ -3393,6 +3389,15 @@ function conflictList { join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP" retval=$? + + #WIP + if [ $TRAVIS_RUN == true ]; then + echo "conflictList debug retval=$retval" + cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${INITIATOR[$__type]}.$SCRIPT_PID.$TSTAMP" + cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.${TARGET[$__type]}.$SCRIPT_PID.$TSTAMP" + cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.compare.$SCRIPT_PID.$TSTAMP" + fi + if [ $retval -ne 0 ]; then Logger "Cannot create conflict list file." "ERROR" return $retval @@ -4947,6 +4952,12 @@ function LogConflicts { local subject local body + #WIP + if [ $TRAVIS_RUN == true ]; then + cat "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" + fi + + # We keep this in a separate if check because of the subshell used for Logger with _LOGGER_PREFIX if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then Logger "File conflicts: INITIATOR << >> TARGET" "ALWAYS" @@ -4954,6 +4965,7 @@ function LogConflicts { ( _LOGGER_PREFIX="" + if [ -f "$RUN_DIR/$PROGRAM.conflictList.compare.$SCRIPT_PID.$TSTAMP" ]; then echo "" > "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/${INITIATOR[$__conflictListFile]}" while read -r line; do