From 3bd6dd09c725e346eef32a3a664659d58c4c5542 Mon Sep 17 00:00:00 2001 From: deajan Date: Wed, 8 Feb 2017 08:39:59 +0100 Subject: [PATCH] Rebuilt targets for v1.2RC2 --- dev/debug_osync.sh | 154 +++++++++++++++++++++++++++++++++++++-------- install.sh | 62 ++++++++++++------ osync-batch.sh | 6 +- osync.sh | 154 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 303 insertions(+), 73 deletions(-) diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index 0b6858a..728bd88 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -3,8 +3,8 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-RC1+dev -PROGRAM_BUILD=2016121901 +PROGRAM_VERSION=1.2-RC2 +PROGRAM_BUILD=2017020702 IS_STABLE=no # Execution order #__WITH_PARANOIA_DEBUG @@ -41,8 +41,8 @@ IS_STABLE=no # CleanUp no #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2017010401 +_OFUNCTIONS_VERSION=2.1-RC2 +_OFUNCTIONS_BUILD=2017020703 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -159,7 +159,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -168,7 +168,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -418,7 +418,12 @@ function SendAlert { attachment=true fi if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" ]; then - body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)" + if [ "$MAIL_BODY_CHARSET" != "" ] && type iconv > /dev/null 2>&1; then + iconv -f UTF-8 -t $MAIL_BODY_CHARSET "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP" + body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP)" + else + body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)" + fi fi if [ $ERROR_ALERT == true ]; then @@ -1104,6 +1109,8 @@ function ArrayContains () { function GetLocalOS { local localOsVar + local localOsName + local localOsVer # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then @@ -1164,8 +1171,14 @@ function GetLocalOS { Logger "Local OS: [$localOsVar]." "DEBUG" fi + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName=$(GetConfFileValue "/etc/os-release" "NAME") + localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + fi + # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar" + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" } @@ -1182,6 +1195,8 @@ $SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$T function GetOs { local localOsVar + local localOsName + local localOsVer # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then @@ -1200,7 +1215,13 @@ function GetOs { fi fi fi - echo "$localOsVar" + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName=$(GetConfFileValue "/etc/os-release" "NAME") + localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + fi + + echo "$localOsVar ($localOsName $localOsVer)" } GetOs @@ -1664,6 +1685,11 @@ function InitLocalOSDependingSettings { if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then FIND_CMD=$(dirname $BASH)/find PING_CMD='$SYSTEMROOT\system32\ping -n 2' + + # On BSD, when not root, min ping interval is 1s + elif [ "$LOCAL_OS" == "BSD" ] && [ "$LOCAL_USER" != "root" ]; then + FIND_CMD=find + PING_CMD="ping -c 2 -i 1" else FIND_CMD=find PING_CMD="ping -c 2 -i .2" @@ -1713,7 +1739,7 @@ function InitRemoteOSDependingSettings { fi ## Set rsync default arguments - RSYNC_ARGS="-rltD" + RSYNC_ARGS="-rltD -8" if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -1808,6 +1834,68 @@ function ParentPid { fi } +# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 +# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 +vercomp () { + if [[ $1 == $2 ]] + then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +function GetConfFileValue () { + local file="${1}" + local name="${2}" + local value + + value=$(grep "^$name=" "$file") + if [ $? == 0 ]; then + value="${value##*=}" + echo "$value" + else + Logger "Cannot get value for [$name] in config file [$file]." "ERROR" + fi +} + +function SetConfFileValue () { + local file="${1}" + local name="${2}" + local value="${3}" + + if grep "^$name=" "$file" > /dev/null; then + # Using -i.tmp for BSD compat + sed -i.tmp "s/^$name=.*/$name=$value/" "$file" + rm -f "$file.tmp" + Logger "Set [$name] to [$value] in config file [$file]." "DEBUG" + else + Logger "Cannot set value [$name] to [$value] in config file [$file]." "ERROR" + fi +} + # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap _OFUNCTIONS_BOOTSTRAP=true @@ -1907,6 +1995,13 @@ function CheckEnvironment { Logger "pgrep not present. Sync cannot start." "CRITICAL" exit 1 fi + + if [ "$SUDO_EXEC" == "yes" ]; then + if ! type sudo > /dev/null 2>&1 ; then + Logger "sudo not present. Sync cannot start." "CRITICAL" + exit 1 + fi + fi } # Only gets checked in config file mode where all values should be present @@ -2105,7 +2200,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -2114,7 +2209,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -2428,7 +2523,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -2437,7 +2532,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -2745,9 +2840,9 @@ function treeList { if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" else - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" fi Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" @@ -2766,6 +2861,7 @@ function treeList { return 0 else Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return $retval fi @@ -2802,12 +2898,13 @@ function deleteList { cmd="(grep -F -x -v -f \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeCurrentFile]}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeAfterFileNoSuffix]}\" || :) > \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}\"" fi - Logger "CMD: $cmd" "DEBUG" + Logger "Launching command [$cmd]." "DEBUG" eval "$cmd" 2>> "$LOG_FILE" retval=$? if [ $retval -ne 0 ]; then - Logger "Couldl not prepare $replicaType deletion list." "CRITICAL" $retval + Logger "Could not prepare $replicaType deletion list." "CRITICAL" $retval + Logger "Command was [$cmd]." "WARN" return $retval fi @@ -2860,11 +2957,12 @@ function _getFileCtimeMtimeRemote { local cmd cmd='cat "'$fileList'" | '$SSH_CMD' "cat > \".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP\""' - Logger "CMD: $cmd" "DEBUG" + Logger "Launching command [$cmd]." "DEBUG" eval "$cmd" retval=$? if [ $retval -ne 0 ]; then Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval + Logger "Command was [$cmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi @@ -2918,9 +3016,9 @@ function syncAttrs { if [ "$REMOTE_OPERATION" == "yes" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n -8 $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" $REMOTE_USER@$REMOTE_HOST:\"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" $REMOTE_USER@$REMOTE_HOST:\"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" else - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n -8 $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" \"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" \"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" fi Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" @@ -2929,6 +3027,7 @@ function syncAttrs { if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then Logger "Getting list of files that need updates failed [$retval]. Stopping execution." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -3015,6 +3114,7 @@ function syncAttrs { if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then Logger "Updating file attributes on $destReplica [$retval]. Stopping execution." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -3077,6 +3177,7 @@ function syncUpdate { if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then Logger "Updating $destinationReplica replica failed. Stopping execution." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -3207,6 +3308,7 @@ function _deleteRemote { retval=$? if [ $retval -ne 0 ]; then Logger "Cannot copy the deletion list to remote replica." "ERROR" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP" ]; then Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP)" "ERROR" fi @@ -3255,7 +3357,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -3264,7 +3366,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -3410,13 +3512,13 @@ ENDSSH fi ## Copy back the deleted failed file list - #rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP\"" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) -r --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" --include \"$(dirname ${TARGET[$__stateDir]})\" --include \"${TARGET[$__stateDir]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__failedDeletedListFile]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__successDeletedListFile]}\" --exclude='*' $REMOTE_USER@$REMOTE_HOST:\"$(EscapeSpaces ${TARGET[$__replicaDir]})\" \"${INITIATOR[$__replicaDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP\"" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" 2>> "$LOG_FILE" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot copy back the failed deletion list to initiator replica." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP" ]; then Logger "Comand output: $(cat $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -3893,7 +3995,7 @@ env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPat # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? @@ -4257,7 +4359,7 @@ function SyncOnChanges { else cmd='bash '$osync_cmd' '$opts fi - Logger "daemon cmd: $cmd" "DEBUG" + Logger "Daemon cmd: $cmd" "DEBUG" eval "$cmd" retval=$? if [ $retval -ne 0 ] && [ $retval != 2 ]; then diff --git a/install.sh b/install.sh index 7e4695a..6446aa4 100755 --- a/install.sh +++ b/install.sh @@ -3,12 +3,12 @@ _OFUNCTIONS_BOOTSTRAP=true PROGRAM=osync -PROGRAM_VERSION=1.2-RC1+dev +PROGRAM_VERSION=1.2-RC2 PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2017020701 +SCRIPT_BUILD=2017020704 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -86,6 +86,8 @@ function UrlEncode { } function GetLocalOS { local localOsVar + local localOsName + local localOsVer # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then @@ -146,9 +148,29 @@ function GetLocalOS { Logger "Local OS: [$localOsVar]." "DEBUG" fi + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName=$(GetConfFileValue "/etc/os-release" "NAME") + localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + fi + # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar" + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" } +function GetConfFileValue () { + local file="${1}" + local name="${2}" + local value + + value=$(grep "^$name=" "$file") + if [ $? == 0 ]; then + value="${value##*=}" + echo "$value" + else + Logger "Cannot get value for [$name] in config file [$file]." "ERROR" + fi +} + function SetLocalOSSettings { USER=root @@ -196,17 +218,17 @@ function GetInit { fi } -function CreateConfDir { - if [ ! -d "$CONF_DIR" ]; then - mkdir "$CONF_DIR" +function CreateDir { + local dir="${1}" + + if [ ! -d "$dir" ]; then + mkdir "$dir" if [ $? == 0 ]; then - QuickLogger "Created directory [$CONF_DIR]." + QuickLogger "Created directory [$dir]." else - QuickLogger "Cannot create directory [$CONF_DIR]." + QuickLogger "Cannot create directory [$dir]." exit 1 fi - else - QuickLogger "Config directory [$CONF_DIR] exists." fi } @@ -217,11 +239,12 @@ function CopyFile { local fileMod="${4}" local fileUser="${5}" local fileGroup="${6}" + local overwrite="${7:-false}" local userGroup="" local oldFileName - if [ -f "$destPath/$fileName" ]; then + if [ -f "$destPath/$fileName" ] && [ $overwrite == false ]; then oldFileName="$fileName" fileName="$oldFileName.new" cp "$sourcePath/$oldFileName" "$destPath/$fileName" @@ -273,7 +296,7 @@ function CopyExampleFiles { for file in "${exampleFiles[@]}"; do if [ -f "$SCRIPT_PATH/$file" ]; then - CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" + CopyFile "$SCRIPT_PATH" "$CONF_DIR" "$file" "" "" "" false fi done } @@ -295,25 +318,27 @@ function CopyProgram { fi for file in "${binFiles[@]}"; do - CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" + CopyFile "$SCRIPT_PATH" "$BIN_DIR" "$file" 755 "$user" "$group" true done } function CopyServiceFiles { if ([ "$init" == "systemd" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_SYSTEMD_SYSTEM" ]); then - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_SYSTEM" "$SERVICE_FILE_SYSTEMD_SYSTEM" "" "" "" true + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_SYSTEMD_USER" "$SERVICE_FILE_SYSTEMD_USER" "" "" "" true QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_SYSTEMD_SYSTEM] and [$SERVICE_DIR_SYSTEMD_USER]." QuickLogger "Can be activated with [systemctl start SERVICE_NAME@instance.conf] where instance.conf is the name of the config file in $CONF_DIR." QuickLogger "Can be enabled on boot with [systemctl enable $SERVICE_NAME@instance.conf]." QuickLogger "In userland, active with [systemctl --user start $SERVICE_NAME@instance.conf]." - elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ]); then - CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" + elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then + CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]." QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]." QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]." + else + QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation." fi } @@ -405,7 +430,8 @@ if [ "$ACTION" == "uninstall" ]; then RemoveAll QuickLogger "$PROGRAM uninstalled." else - CreateConfDir + CreateDir "$CONF_DIR" + CreateDir "$BIN_DIR" CopyExampleFiles CopyProgram CopyServiceFiles diff --git a/osync-batch.sh b/osync-batch.sh index 47a9c0e..46df1c8 100755 --- a/osync-batch.sh +++ b/osync-batch.sh @@ -119,15 +119,15 @@ function Batch { fi done runList=("${runAgainList[@]}") - runs=$(($runs + 1)) + runs=$((runs + 1)) done fi } function Usage { echo "$PROGRAM $PROGRAM_BUILD" - echo $AUTHOR - echo $CONTACT + echo "$AUTHOR" + echo "$CONTACT" echo "" echo "Batch script to sequentially run osync or obackup instances and rerun failed ones." echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]" diff --git a/osync.sh b/osync.sh index a23cfd3..39063e6 100755 --- a/osync.sh +++ b/osync.sh @@ -3,15 +3,15 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance AUTHOR="(C) 2013-2017 by Orsiris de Jong" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" -PROGRAM_VERSION=1.2-RC1+dev -PROGRAM_BUILD=2016121901 +PROGRAM_VERSION=1.2-RC2 +PROGRAM_BUILD=2017020702 IS_STABLE=no -_OFUNCTIONS_VERSION=2.1-RC1+dev -_OFUNCTIONS_BUILD=2017010401 +_OFUNCTIONS_VERSION=2.1-RC2 +_OFUNCTIONS_BUILD=2017020703 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -123,7 +123,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -132,7 +132,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -369,7 +369,12 @@ function SendAlert { attachment=true fi if [ -e "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" ]; then - body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)" + if [ "$MAIL_BODY_CHARSET" != "" ] && type iconv > /dev/null 2>&1; then + iconv -f UTF-8 -t $MAIL_BODY_CHARSET "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP" > "$RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP" + body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.iconv.$SCRIPT_PID.$TSTAMP)" + else + body="$MAIL_ALERT_MSG"$'\n\n'"$(cat $RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP)" + fi fi if [ $ERROR_ALERT == true ]; then @@ -1038,6 +1043,8 @@ function ArrayContains () { function GetLocalOS { local localOsVar + local localOsName + local localOsVer # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then @@ -1098,8 +1105,14 @@ function GetLocalOS { Logger "Local OS: [$localOsVar]." "DEBUG" fi + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName=$(GetConfFileValue "/etc/os-release" "NAME") + localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + fi + # Add a global variable for statistics in installer - LOCAL_OS_FULL="$localOsVar" + LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" } @@ -1115,6 +1128,8 @@ $SSH_CMD bash -s << 'ENDSSH' >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$T function GetOs { local localOsVar + local localOsName + local localOsVer # There's no good way to tell if currently running in BusyBox shell. Using sluggish way. if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then @@ -1133,7 +1148,13 @@ function GetOs { fi fi fi - echo "$localOsVar" + # Get linux versions + if [ -f "/etc/os-release" ]; then + localOsName=$(GetConfFileValue "/etc/os-release" "NAME") + localOsVer=$(GetConfFileValue "/etc/os-release" "VERSION") + fi + + echo "$localOsVar ($localOsName $localOsVer)" } GetOs @@ -1526,6 +1547,11 @@ function InitLocalOSDependingSettings { if [ "$LOCAL_OS" == "msys" ] || [ "$LOCAL_OS" == "Cygwin" ]; then FIND_CMD=$(dirname $BASH)/find PING_CMD='$SYSTEMROOT\system32\ping -n 2' + + # On BSD, when not root, min ping interval is 1s + elif [ "$LOCAL_OS" == "BSD" ] && [ "$LOCAL_USER" != "root" ]; then + FIND_CMD=find + PING_CMD="ping -c 2 -i 1" else FIND_CMD=find PING_CMD="ping -c 2 -i .2" @@ -1574,7 +1600,7 @@ function InitRemoteOSDependingSettings { fi ## Set rsync default arguments - RSYNC_ARGS="-rltD" + RSYNC_ARGS="-rltD -8" if [ "$_DRYRUN" == true ]; then RSYNC_DRY_ARG="-n" DRY_WARNING="/!\ DRY RUN " @@ -1669,6 +1695,68 @@ function ParentPid { fi } +# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 +# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 +vercomp () { + if [[ $1 == $2 ]] + then + return 0 + fi + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + return 1 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + return 2 + fi + done + return 0 +} + +function GetConfFileValue () { + local file="${1}" + local name="${2}" + local value + + value=$(grep "^$name=" "$file") + if [ $? == 0 ]; then + value="${value##*=}" + echo "$value" + else + Logger "Cannot get value for [$name] in config file [$file]." "ERROR" + fi +} + +function SetConfFileValue () { + local file="${1}" + local name="${2}" + local value="${3}" + + if grep "^$name=" "$file" > /dev/null; then + # Using -i.tmp for BSD compat + sed -i.tmp "s/^$name=.*/$name=$value/" "$file" + rm -f "$file.tmp" + Logger "Set [$name] to [$value] in config file [$file]." "DEBUG" + else + Logger "Cannot set value [$name] to [$value] in config file [$file]." "ERROR" + fi +} + # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap _OFUNCTIONS_BOOTSTRAP=true @@ -1767,6 +1855,13 @@ function CheckEnvironment { Logger "pgrep not present. Sync cannot start." "CRITICAL" exit 1 fi + + if [ "$SUDO_EXEC" == "yes" ]; then + if ! type sudo > /dev/null 2>&1 ; then + Logger "sudo not present. Sync cannot start." "CRITICAL" + exit 1 + fi + fi } # Only gets checked in config file mode where all values should be present @@ -1957,7 +2052,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -1966,7 +2061,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -2268,7 +2363,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -2277,7 +2372,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -2575,9 +2670,9 @@ function treeList { if [ "$REMOTE_OPERATION" == "yes" ] && [ "$replicaType" == "${TARGET[$__type]}" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --list-only $REMOTE_USER@$REMOTE_HOST:\"$escapedReplicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" else - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS -8 --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \"$replicaPath\" 2> \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP\" | (grep -E \"^-|^d|^l\" || :) | (awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)}' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.$SCRIPT_PID.$TSTAMP\"" fi Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" @@ -2596,6 +2691,7 @@ function treeList { return 0 else Logger "Cannot create replica file list in [$replicaPath]." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" Logger "Command output\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$replicaType.error.$SCRIPT_PID.$TSTAMP)" "WARN" return $retval fi @@ -2631,12 +2727,13 @@ function deleteList { cmd="(grep -F -x -v -f \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeCurrentFile]}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__treeAfterFileNoSuffix]}\" || :) > \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$replicaType${INITIATOR[$__deletedListFile]}\"" fi - Logger "CMD: $cmd" "DEBUG" + Logger "Launching command [$cmd]." "DEBUG" eval "$cmd" 2>> "$LOG_FILE" retval=$? if [ $retval -ne 0 ]; then - Logger "Couldl not prepare $replicaType deletion list." "CRITICAL" $retval + Logger "Could not prepare $replicaType deletion list." "CRITICAL" $retval + Logger "Command was [$cmd]." "WARN" return $retval fi @@ -2687,11 +2784,12 @@ function _getFileCtimeMtimeRemote { local cmd cmd='cat "'$fileList'" | '$SSH_CMD' "cat > \".$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP\""' - Logger "CMD: $cmd" "DEBUG" + Logger "Launching command [$cmd]." "DEBUG" eval "$cmd" retval=$? if [ $retval -ne 0 ]; then Logger "Sending ctime required file list failed with [$retval] on $replicaType. Stopping execution." "CRITICAL" $retval + Logger "Command was [$cmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP" ]; then Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replicaType.$SCRIPT_PID.$TSTAMP)" "WARN" fi @@ -2744,9 +2842,9 @@ function syncAttrs { if [ "$REMOTE_OPERATION" == "yes" ]; then CheckConnectivity3rdPartyHosts CheckConnectivityRemoteHost - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n -8 $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" $REMOTE_USER@$REMOTE_HOST:\"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE -e \"$RSYNC_SSH_CMD\" --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" $REMOTE_USER@$REMOTE_HOST:\"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" else - rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n -8 $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" \"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" + rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -i -n $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_PARTIAL_EXCLUDE --exclude \"$OSYNC_DIR\" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE \"$initiatorReplica\" \"$targetReplica\" >> $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP 2>&1 &" fi Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" @@ -2755,6 +2853,7 @@ function syncAttrs { if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then Logger "Getting list of files that need updates failed [$retval]. Stopping execution." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -2841,6 +2940,7 @@ function syncAttrs { if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then Logger "Updating file attributes on $destReplica [$retval]. Stopping execution." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$destReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -2902,6 +3002,7 @@ function syncUpdate { if [ $retval -ne 0 ] && [ $retval -ne 24 ]; then Logger "Updating $destinationReplica replica failed. Stopping execution." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP" ]; then Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -3030,6 +3131,7 @@ function _deleteRemote { retval=$? if [ $retval -ne 0 ]; then Logger "Cannot copy the deletion list to remote replica." "ERROR" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP" ]; then Logger "$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.precopy.$SCRIPT_PID.$TSTAMP)" "ERROR" fi @@ -3074,7 +3176,7 @@ function joinString { function _Logger { local logValue="${1}" # Log to file local stdValue="${2}" # Log to screeen - local toStderr="${3:-false}" # Log to stderr instead of stdout + local toStdErr="${3:-false}" # Log to stderr instead of stdout if [ "$logValue" != "" ]; then echo -e "$logValue" >> "$LOG_FILE" @@ -3083,7 +3185,7 @@ function _Logger { fi if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then - if [ $toStderr == true ]; then + if [ $toStdErr == true ]; then # Force stderr color in subshell (>&2 echo -e "$stdValue") @@ -3224,13 +3326,13 @@ ENDSSH fi ## Copy back the deleted failed file list - #rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP\"" rsyncCmd="$(type -p $RSYNC_EXECUTABLE) -r --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" --include \"$(dirname ${TARGET[$__stateDir]})\" --include \"${TARGET[$__stateDir]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__failedDeletedListFile]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__successDeletedListFile]}\" --exclude='*' $REMOTE_USER@$REMOTE_HOST:\"$(EscapeSpaces ${TARGET[$__replicaDir]})\" \"${INITIATOR[$__replicaDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP\"" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" eval "$rsyncCmd" 2>> "$LOG_FILE" retval=$? if [ $retval -ne 0 ]; then Logger "Cannot copy back the failed deletion list to initiator replica." "CRITICAL" $retval + Logger "Command was [$rsyncCmd]." "WARN" if [ -f "$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP" ]; then Logger "Comand output: $(cat $RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID.$TSTAMP)" "NOTICE" fi @@ -3703,7 +3805,7 @@ env _DRYRUN="'$_DRYRUN'" env replicaType="'$replicaType'" env replicaDeletionPat # Cannot launch log function from xargs, ugly hack if [ -d "$replicaDeletionPath" ]; then - $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" ill delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' + $REMOTE_FIND_CMD "$replicaDeletionPath" -type f -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete file {}"; fi; if [ '$_DRYRUN' == false ]; then rm -f "$file"; fi' retval1=$? $REMOTE_FIND_CMD "$replicaDeletionPath" -type d -empty -ctime +"$changeTime" -print0 | xargs -0 -I {} bash -c 'export file="{}"; if [ '$_LOGGER_VERBOSE' == true ]; then echo "On "'$replicaType'" will delete directory {}"; fi; if [ '$_DRYRUN' == false ]; then rm -rf "{}"; fi' retval2=$? @@ -4060,7 +4162,7 @@ function SyncOnChanges { else cmd='bash '$osync_cmd' '$opts fi - Logger "daemon cmd: $cmd" "DEBUG" + Logger "Daemon cmd: $cmd" "DEBUG" eval "$cmd" retval=$? if [ $retval -ne 0 ] && [ $retval != 2 ]; then