Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
pull/84/head
deajan 8 years ago
parent 42b0c8d406
commit e4026f537e
4 changed files with 571 additions and 333 deletions
Show Stats Download Patch File Download Diff File

417
dev/debug_osync.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta3 PROGRAM_VERSION=1.2-beta3
PROGRAM_BUILD=2016111902 PROGRAM_BUILD=2016112401
IS_STABLE=no IS_STABLE=no
# Execution order #__WITH_PARANOIA_DEBUG # Execution order #__WITH_PARANOIA_DEBUG
@ -45,7 +45,7 @@ IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111901 ## FUNC_BUILD=2016112401
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -60,8 +60,18 @@ IS_STABLE=no
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile
#TODO: Windows checks, check sendmail & mailsend
## META ISSUES
##
## Updated _LOGGER_STDERR
## Updated WaitForTaskCompletion syntax
## Updated ParallelExec syntax
## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2" echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -96,10 +106,10 @@ fi #__WITH_PARANOIA_DEBUG
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys SLEEP_TIME=.05
_LOGGER_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05 SLEEP_TIME=.05
fi fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
@ -281,7 +291,7 @@ function KillChilds {
done done
fi fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then if ( [ "$self" == true ] && kill -0 "$pid" > /dev/null 2>&1); then
Logger "Sending SIGTERM to process [$pid]." "DEBUG" Logger "Sending SIGTERM to process [$pid]." "DEBUG"
kill -s TERM "$pid" kill -s TERM "$pid"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -455,13 +465,18 @@ function SendEmail {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then # We need to detect which version of mail is installed
if ! $(type -p mail) -V > /dev/null 2>&1; then
# This may be MacOS mail program
attachment_command=""
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $attachment" attachment_command="-A $attachment"
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a$attachment" attachment_command="-a$attachment"
else else
attachment_command="" attachment_command=""
fi fi
echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
@ -573,30 +588,30 @@ function Spinner {
return 0 return 0
fi fi
case $toggle case $_OFUNCTIONS_SPINNER_TOGGLE
in in
1) 1)
echo -n " \ " echo -n " \ "
echo -ne "\r" echo -ne "\r"
toggle="2" _OFUNCTIONS_SPINNER_TOGGLE=2
;; ;;
2) 2)
echo -n " | " echo -n " | "
echo -ne "\r" echo -ne "\r"
toggle="3" _OFUNCTIONS_SPINNER_TOGGLE=3
;; ;;
3) 3)
echo -n " / " echo -n " / "
echo -ne "\r" echo -ne "\r"
toggle="4" _OFUNCTIONS_SPINNER_TOGGLE=4
;; ;;
*) *)
echo -n " - " echo -n " - "
echo -ne "\r" echo -ne "\r"
toggle="1" _OFUNCTIONS_SPINNER_TOGGLE=1
;; ;;
esac esac
} }
@ -611,16 +626,21 @@ function joinString {
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
# Warning: Don't imbricate this function into another run if you plan to use the global variable output # Warning: Don't imbricate this function into another run if you plan to use the global variable output
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true "${FUNCNAME[0]}"
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${4}" # Who called this function local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noError="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} called by [$callerName]." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
__CheckArguments 6 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 9 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison local log_ttime=0 # local time instance for comparaison
@ -648,7 +668,9 @@ function WaitForTaskCompletion {
while [ ${#pidsArray[@]} -gt 0 ]; do while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=() newPidsArray=()
Spinner if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin)) exec_time=$(($SECONDS - $seconds_begin))
else else
@ -665,14 +687,16 @@ function WaitForTaskCompletion {
fi fi
if [ $exec_time -gt $softMaxTime ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then if [ $soft_alert != true ] && [ $softMaxTime -ne 0 ] && [ $noError != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true soft_alert=true
SendAlert true SendAlert true
fi fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" if [ $noError != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
KillChilds $pid true KillChilds $pid true
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -681,7 +705,9 @@ function WaitForTaskCompletion {
Logger "Could not stop task with pid [$pid]." "ERROR" Logger "Could not stop task with pid [$pid]." "ERROR"
fi fi
done done
SendAlert true if [ $noError != true ]; then
SendAlert true
fi
fi fi
fi fi
@ -717,7 +743,7 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $SLEEP_TIME sleep $sleepTime
done done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
@ -733,17 +759,28 @@ function WaitForTaskCompletion {
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands # Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
# Only 2 first arguments are mandatory
function ParallelExec { function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ? local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${6}" # Who called this function local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noError="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
__CheckArguments 2-11 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
__CheckArguments 8 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount local commandCount
local command local command
@ -774,15 +811,60 @@ function ParallelExec {
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin))
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert != true ] && [ $softMaxTime -ne 0 ] && [ $noError != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true
SendAlert true
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noError != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noError != true ]; then
SendAlert true
fi
# Return the number of commands that haven't run / finished run
return $(($commandCount - $counter + ${#pidsArray[@]}))
fi
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then if [ $readFromFile == true ]; then
#TODO: Checked on FreeBSD 10, also check on Win
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else else
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
fi fi
Logger "Running command [$command]." "DEBUG" Logger "Running command [$command]." "DEBUG"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID" 2>&1 &
pid=$! pid=$!
pidsArray+=($pid) pidsArray+=($pid)
commandsArrayPid[$pid]="$command" commandsArrayPid[$pid]="$command"
@ -983,11 +1065,16 @@ function GetLocalOS {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
@ -1006,6 +1093,9 @@ function GetLocalOS {
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
LOCAL_OS="msys" LOCAL_OS="msys"
;; ;;
*"Microsoft"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
LOCAL_OS="MacOSX" LOCAL_OS="MacOSX"
;; ;;
@ -1044,15 +1134,19 @@ function GetOs {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
echo "$localOsVar" echo "$localOsVar"
} }
@ -1075,6 +1169,9 @@ ENDSSH
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
REMOTE_OS="msys" REMOTE_OS="msys"
;; ;;
*"Microsoft"*)
REMOTE_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
REMOTE_OS="MacOSX" REMOTE_OS="MacOSX"
;; ;;
@ -1112,7 +1209,8 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1147,7 +1245,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1181,7 +1279,7 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1200,7 +1298,7 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1209,18 +1307,18 @@ function CheckConnectivityRemoteHost {
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
return $retval return $retval
fi fi
fi fi
fi fi #__WITH_PARANOIA_DEBUG
} }
function CheckConnectivity3rdPartyHosts { function CheckConnectivity3rdPartyHosts {
@ -1229,14 +1327,14 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess local remote3rdPartySuccess
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug #__WITH_PARANOIA_DEBUG
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
remote3rdPartySuccess=false remote3rdPartySuccess=false
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1252,7 +1350,7 @@ function CheckConnectivity3rdPartyHosts {
return 0 return 0
fi fi
fi fi
fi fi #__WITH_PARANOIA_DEBUG
} }
#__BEGIN_WITH_PARANOIA_DEBUG #__BEGIN_WITH_PARANOIA_DEBUG
@ -1426,66 +1524,12 @@ function PreInit {
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set compression executable and extension
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
fi fi
#TODO: Remote OS isn't defined yet
## Busybox fix (Termux xz command doesn't support compression at all) ## Busybox fix (Termux xz command doesn't support compression at all)
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString="" compressionString=""
@ -1564,7 +1608,7 @@ function InitLocalOSSettings {
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then
PROCESS_STATE_CMD="echo none" PROCESS_STATE_CMD="echo none"
DF_CMD="df" DF_CMD="df"
else else
@ -1579,7 +1623,7 @@ function InitLocalOSSettings {
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
# Tested on GNU stat and busybox # Tested on GNU stat, busybox and Cygwin
STAT_CMD="stat -c %y" STAT_CMD="stat -c %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
@ -1588,13 +1632,6 @@ function InitLocalOSSettings {
function InitRemoteOSSettings { function InitRemoteOSSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
@ -1612,6 +1649,72 @@ function InitRemoteOSSettings {
} }
function InitRsyncSettings {
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
## Set rsync default arguments
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability"
fi
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
else
Logger "Disabling ACL and extended attributes synchronization on [$LOCAL_OS]." "NOTICE"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
}
## IFS debug function ## IFS debug function
function PrintIFS { function PrintIFS {
printf "IFS is: %q" "$IFS" printf "IFS is: %q" "$IFS"
@ -1856,7 +1959,7 @@ function CheckReplicaPaths {
_CheckReplicaPathsRemote "${TARGET[$__replicaDir]}" & _CheckReplicaPathsRemote "${TARGET[$__replicaDir]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" Logger "Cancelling task." "CRITICAL"
exit 1 exit 1
@ -1938,7 +2041,7 @@ function CheckDiskSpace {
_CheckDiskSpaceRemote "${TARGET[$__replicaDir]}" & _CheckDiskSpaceRemote "${TARGET[$__replicaDir]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
} }
@ -1989,7 +2092,7 @@ function CreateStateDirs {
_CreateStateDirsRemote "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" & _CreateStateDirsRemote "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" Logger "Cancelling task." "CRITICAL"
exit 1 exit 1
@ -2120,7 +2223,7 @@ function CheckLocks {
_CheckLocksRemote "${TARGET[$__lockFile]}" & _CheckLocksRemote "${TARGET[$__lockFile]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" Logger "Cancelling task." "CRITICAL"
exit 1 exit 1
@ -2198,7 +2301,7 @@ function WriteLockFiles {
INITIATOR_LOCK_FILE_EXISTS=true INITIATOR_LOCK_FILE_EXISTS=true
TARGET_LOCK_FILE_EXISTS=true TARGET_LOCK_FILE_EXISTS=true
WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
@ -2274,7 +2377,7 @@ function UnlockReplicas {
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -2444,7 +2547,7 @@ function syncAttrs {
fi fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
@ -2476,7 +2579,7 @@ function syncAttrs {
_getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" & _getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
# If target gets updated first, then sync_attr must update initiators attrs first # If target gets updated first, then sync_attr must update initiators attrs first
# For join, remove leading replica paths # For join, remove leading replica paths
@ -2524,7 +2627,7 @@ function syncAttrs {
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
@ -2696,6 +2799,8 @@ function _deleteRemote {
local deletionListFromReplica local deletionListFromReplica
local loggerPrefix
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
deletionListFromReplica="${TARGET[$__type]}" deletionListFromReplica="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
@ -2830,12 +2935,16 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
done < "$FILE_LIST" done < "$FILE_LIST"
ENDSSH ENDSSH
if [ -f "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then if [ -z "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then
Logger "Remote Deletion:\n$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "VERBOSE" loggerPrefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX=""
Logger "$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "ERROR"
_LOGGER_PREFIX="$loggerPrefix"
fi fi
## Copy back the deleted failed file list ## Copy back the deleted failed file list
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\"" #rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) -r --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" --include \"$(dirname ${TARGET[$__stateDir]})\" --include \"${TARGET[$__stateDir]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__failedDeletedListFile]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__successDeletedListFile]}\" --exclude='*' $REMOTE_USER@$REMOTE_HOST:\"$(EscapeSpaces ${TARGET[$__replicaDir]})\" \"${INITIATOR[$__replicaDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" 2>> "$LOG_FILE" eval "$rsyncCmd" 2>> "$LOG_FILE"
result=$? result=$?
@ -2985,7 +3094,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -3030,7 +3139,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -3067,7 +3176,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -3134,7 +3243,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -3180,7 +3289,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -3332,7 +3441,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
@ -3347,7 +3456,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -3368,9 +3477,9 @@ function _SummaryFromFile {
function Summary { function Summary {
__CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# "${FUNCNAME[0]}" "$@" #__WITH_PARANOIA_DEBUG
local prefix local loggerPrefix
prefix="$_LOGGER_PREFIX" loggerPrefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX="" _LOGGER_PREFIX=""
Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS" Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS"
@ -3391,7 +3500,7 @@ function Summary {
fi fi
_SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "- <<" _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "- <<"
_LOGGER_PREFIX="$prefix" _LOGGER_PREFIX="$loggerPrefix"
} }
function Init { function Init {
@ -3633,9 +3742,18 @@ function SyncOnChanges {
local cmd local cmd
local retval local retval
if ! type inotifywait > /dev/null 2>&1 ; then local sleepTime
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1 if [ "$LOCAL_OS" == "MacOSX" ]; then
if ! type fswatch > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1
fi
else
if ! type inotifywait > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1
fi
fi fi
Logger "#### Running osync in file monitor mode." "NOTICE" Logger "#### Running osync in file monitor mode." "NOTICE"
@ -3654,8 +3772,14 @@ function SyncOnChanges {
fi fi
Logger "#### Monitoring now." "NOTICE" Logger "#### Monitoring now." "NOTICE"
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & if [ "$LOCAL_OS" == "MacOSX" ]; then
wait $! fswatch --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -1 "$INITIATOR_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true ${FUNCNAME[0]}
else
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
wait $!
fi
retval=$? retval=$?
if [ $retval == 0 ]; then if [ $retval == 0 ]; then
Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE" Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE"
@ -3874,6 +3998,7 @@ if [ $sync_on_changes == true ]; then
else else
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
InitRsyncSettings
if [ $no_maxtime == true ]; then if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0

39
install.sh
Unescape Escape View File

@ -4,7 +4,7 @@ PROGRAM=osync
PROGRAM_VERSION=1.2-beta3 PROGRAM_VERSION=1.2-beta3
PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SCRIPT_BUILD=2016111201 SCRIPT_BUILD=2016112401
## osync / obackup / pmocr / zsnap install script ## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8 & 10
@ -79,24 +79,29 @@ function urlencode() {
} }
function SetOSSettings { function SetOSSettings {
USER=root local localOsVar
if type busybox > /dev/null 2>&1; then USER=root
QuickLogger "$0 won't work in busybox. Please use $PROGRAM_BINARY.sh directly."
exit 1
fi
local local_os_var
local_os_var="$(uname -spio 2>&1)" # There's no good way to tell if currently running in BusyBox shell. Using sluggish way.
if [ $? != 0 ]; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
local_os_var="$(uname -v 2>&1)" localOsVar="BusyBox"
if [ $? != 0 ]; then else
local_os_var="$(uname)" # Detecting the special ubuntu userland in Windows 10 bash
fi if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
fi localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi
fi
fi
case $local_os_var in case $localOsVar in
*"BSD"*) *"BSD"*)
GROUP=wheel GROUP=wheel
;; ;;
@ -117,7 +122,7 @@ function SetOSSettings {
exit 1 exit 1
fi fi
OS=$(urlencode "$local_os_var") OS=$(urlencode "$localOsVar")
} }
function GetInit { function GetInit {

38
osync-batch.sh
Unescape Escape View File

@ -3,7 +3,7 @@ SUBPROGRAM=osync
PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones PROGRAM="$SUBPROGRAM-batch" # Batch program to run osync / obackup instances sequentially and rerun failed ones
AUTHOR="(L) 2013-2016 by Orsiris de Jong" AUTHOR="(L) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr" CONTACT="http://www.netpower.fr - ozy@netpower.fr"
PROGRAM_BUILD=2016082901 PROGRAM_BUILD=2016112402
## Runs an osync /obackup instance for every conf file found ## Runs an osync /obackup instance for every conf file found
## If an instance fails, run it again if time permits ## If an instance fails, run it again if time permits
@ -13,9 +13,6 @@ if ! type "$BASH" > /dev/null; then
exit 127 exit 127
fi fi
## Configuration file path. The path where all the osync / obackup conf files are, usually /etc/osync or /etc/obackup
CONF_FILE_PATH=/etc/$SUBPROGRAM
## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours. ## If maximum execution time is not reached, failed instances will be rerun. Max exec time is in seconds. Example is set to 10 hours.
MAX_EXECUTION_TIME=36000 MAX_EXECUTION_TIME=36000
@ -74,6 +71,10 @@ function CheckEnvironment {
else else
SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh) SUBPROGRAM_EXECUTABLE=$(type -p $SUBPROGRAM.sh)
fi fi
if [ "$CONF_FILE_PATH" == "" ]; then
Usage
fi
} }
function Batch { function Batch {
@ -111,7 +112,7 @@ function Batch {
Logger "$SUBPROGRAM instances will be run for: $runList" "NOTICE" Logger "$SUBPROGRAM instances will be run for: $runList" "NOTICE"
for confFile in $runList for confFile in $runList
do do
$SUBPROGRAM_EXECUTABLE "$confFile" $opts & $SUBPROGRAM_EXECUTABLE "$confFile" --silent $opts &
wait $! wait $!
result=$? result=$?
if [ $result != 0 ]; then if [ $result != 0 ]; then
@ -141,16 +142,18 @@ function Usage {
echo $CONTACT echo $CONTACT
echo "" echo ""
echo "Batch script to sequentially run osync or obackup instances and rerun failed ones." echo "Batch script to sequentially run osync or obackup instances and rerun failed ones."
echo "Usage: $SUBPROGRAM-batch.sh [OPTIONS]" echo "Usage: $PROGRAM.sh [OPTIONS] [$SUBPROGRAM OPTIONS]"
echo "" echo ""
echo "[OPTIONS]" echo "[OPTIONS]"
echo "--path=/path/to/conf Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup" echo "--path=/path/to/conf Path to osync / obackup conf files, defaults to /etc/osync or /etc/obackup"
echo "--max-runs=X Number of max runs per instance, (defaults to 3)" echo "--max-runs=X Number of max runs per instance, (defaults to 3)"
echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check" echo "--max-exec-time=X Retry failed instances only if max execution time not reached (defaults to 36000 seconds). Set to 0 to bypass execution time check"
echo "--no-maxtime Run osync / obackup without honoring conf file defined timeouts" echo "[$SUBPROGRAM OPTIONS]"
echo "--dry Will run osync / obackup without actually doing anything; just testing" echo "Specify whatever options $PROGRAM accepts. Example"
echo "--silent Will run osync / obackup without any output to stdout, used for cron jobs" echo "$PROGRAM.sh --path=/etc/$SUBPROGRAM --no-maxtime"
echo "--verbose Increases output" echo ""
echo "No output will be written to stdout/stderr."
echo "Verify log file in [$LOG_FILE]."
exit 128 exit 128
} }
@ -158,18 +161,6 @@ opts=""
for i in "$@" for i in "$@"
do do
case $i in case $i in
--silent)
opts=$opts" --silent"
;;
--dry)
opts=$opts" --dry"
;;
--verbose)
opts=$opts" --verbose"
;;
--no-maxtime)
opts=$opts" --no-maxtime"
;;
--path=*) --path=*)
CONF_FILE_PATH=${i##*=} CONF_FILE_PATH=${i##*=}
;; ;;
@ -183,8 +174,7 @@ do
Usage Usage
;; ;;
*) *)
Logger "Unknown param '$i'" "CRITICAL" opts="$i "
Usage
;; ;;
esac esac
done done

410
osync.sh
Unescape Escape View File

@ -4,14 +4,14 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2016 by Orsiris de Jong" AUTHOR="(C) 2013-2016 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr" CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.2-beta3 PROGRAM_VERSION=1.2-beta3
PROGRAM_BUILD=2016111902 PROGRAM_BUILD=2016112401
IS_STABLE=no IS_STABLE=no
#### MINIMAL-FUNCTION-SET BEGIN #### #### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016111901 ## FUNC_BUILD=2016112401
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables: ## To use in a program, define the following variables:
@ -26,8 +26,18 @@ IS_STABLE=no
## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel ## Logger sets {ERROR|WARN}_ALERT variable when called with critical / error / warn loglevel
## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID ## When called from subprocesses, variable of main process can't be set. Status needs to be get via $RUN_DIR/$PROGRAM.Logger.{error|warn}.$SCRIPT_PID
#TODO: Rewrite Logger so we can decide what to send to stdout, stderr and logfile
#TODO: Windows checks, check sendmail & mailsend
## META ISSUES
##
## Updated _LOGGER_STDERR
## Updated WaitForTaskCompletion syntax
## Updated ParallelExec syntax
## SendEmail WinNT10 & msys are two totally different beasts. Document in sync.conf and host_backup.conf
if ! type "$BASH" > /dev/null; then if ! type "$BASH" > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2" echo "Please run this script only with bash shell. Tested on bash >= 3.2"
@ -58,10 +68,10 @@ WARN_ALERT=false
## allow debugging from command line with _DEBUG=yes ## allow debugging from command line with _DEBUG=yes
if [ ! "$_DEBUG" == "yes" ]; then if [ ! "$_DEBUG" == "yes" ]; then
_DEBUG=no _DEBUG=no
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys SLEEP_TIME=.05
_LOGGER_VERBOSE=false _LOGGER_VERBOSE=false
else else
if [ "$SLEEP_TIME" == "" ]; then # Set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
SLEEP_TIME=.05 SLEEP_TIME=.05
fi fi
trap 'TrapError ${LINENO} $?' ERR trap 'TrapError ${LINENO} $?' ERR
@ -234,7 +244,7 @@ function KillChilds {
done done
fi fi
# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing # Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
if ( [ "$self" == true ] && kill -0 $pid > /dev/null 2>&1); then if ( [ "$self" == true ] && kill -0 "$pid" > /dev/null 2>&1); then
Logger "Sending SIGTERM to process [$pid]." "DEBUG" Logger "Sending SIGTERM to process [$pid]." "DEBUG"
kill -s TERM "$pid" kill -s TERM "$pid"
if [ $? != 0 ]; then if [ $? != 0 ]; then
@ -405,13 +415,18 @@ function SendEmail {
fi fi
if type mail > /dev/null 2>&1 ; then if type mail > /dev/null 2>&1 ; then
if [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then # We need to detect which version of mail is installed
if ! $(type -p mail) -V > /dev/null 2>&1; then
# This may be MacOS mail program
attachment_command=""
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
attachment_command="-A $attachment" attachment_command="-A $attachment"
elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
attachment_command="-a$attachment" attachment_command="-a$attachment"
else else
attachment_command="" attachment_command=""
fi fi
echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails" echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
if [ $? != 0 ]; then if [ $? != 0 ]; then
Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN" Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
@ -522,30 +537,30 @@ function Spinner {
return 0 return 0
fi fi
case $toggle case $_OFUNCTIONS_SPINNER_TOGGLE
in in
1) 1)
echo -n " \ " echo -n " \ "
echo -ne "\r" echo -ne "\r"
toggle="2" _OFUNCTIONS_SPINNER_TOGGLE=2
;; ;;
2) 2)
echo -n " | " echo -n " | "
echo -ne "\r" echo -ne "\r"
toggle="3" _OFUNCTIONS_SPINNER_TOGGLE=3
;; ;;
3) 3)
echo -n " / " echo -n " / "
echo -ne "\r" echo -ne "\r"
toggle="4" _OFUNCTIONS_SPINNER_TOGGLE=4
;; ;;
*) *)
echo -n " - " echo -n " - "
echo -ne "\r" echo -ne "\r"
toggle="1" _OFUNCTIONS_SPINNER_TOGGLE=1
;; ;;
esac esac
} }
@ -560,13 +575,18 @@ function joinString {
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
# Warning: Don't imbricate this function into another run if you plan to use the global variable output # Warning: Don't imbricate this function into another run if you plan to use the global variable output
# Standard wait $! emulation would be WaitForTaskCompletion $! 0 0 1 0 true false true "${FUNCNAME[0]}"
function WaitForTaskCompletion { function WaitForTaskCompletion {
local pids="${1}" # pids to wait for, separated by semi-colon local pids="${1}" # pids to wait for, separated by semi-colon
local softMaxTime="${2}" # If program with pid $pid takes longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0. local softMaxTime="${2:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${3}" # If program with pid $pid takes longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0. local hardMaxTime="${3:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${4}" # Who called this function local sleepTime="${4:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${5:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${5:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${6:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${6:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${7:-true}" # Show spinner (true), don't show anything (false)
local noError="${8:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${9}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
@ -594,7 +614,9 @@ function WaitForTaskCompletion {
while [ ${#pidsArray[@]} -gt 0 ]; do while [ ${#pidsArray[@]} -gt 0 ]; do
newPidsArray=() newPidsArray=()
Spinner if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin)) exec_time=$(($SECONDS - $seconds_begin))
else else
@ -611,14 +633,16 @@ function WaitForTaskCompletion {
fi fi
if [ $exec_time -gt $softMaxTime ]; then if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert == true ] && [ $softMaxTime -ne 0 ]; then if [ $soft_alert != true ] && [ $softMaxTime -ne 0 ] && [ $noError != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN" Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true soft_alert=true
SendAlert true SendAlert true
fi fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR" if [ $noError != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do for pid in "${pidsArray[@]}"; do
KillChilds $pid true KillChilds $pid true
if [ $? == 0 ]; then if [ $? == 0 ]; then
@ -627,7 +651,9 @@ function WaitForTaskCompletion {
Logger "Could not stop task with pid [$pid]." "ERROR" Logger "Could not stop task with pid [$pid]." "ERROR"
fi fi
done done
SendAlert true if [ $noError != true ]; then
SendAlert true
fi
fi fi
fi fi
@ -659,7 +685,7 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $SLEEP_TIME sleep $sleepTime
done done
@ -674,16 +700,27 @@ function WaitForTaskCompletion {
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs # Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands # Returns the number of non zero exit codes from commands
# Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets # Use cmd1;cmd2;cmd3 syntax for small sets, use file for large command sets
# Only 2 first arguments are mandatory
function ParallelExec { function ParallelExec {
local numberOfProcesses="${1}" # Number of simultaneous commands to run local numberOfProcesses="${1}" # Number of simultaneous commands to run
local commandsArg="${2}" # Semi-colon separated list of commands, or file containing one command per line local commandsArg="${2}" # Semi-colon separated list of commands, or path to file containing one command per line
local readFromFile="${3:-false}" # Is commandsArg a file or a string ? local readFromFile="${3:-false}" # commandsArg is a file (true), or a string (false)
local softMaxTime="${4:-0}" local softMaxTime="${4:-0}" # If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
local hardMaxTime="${5:-0}" local hardMaxTime="${5:-0}" # If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
local callerName="${6}" # Who called this function local sleepTime="${6:-.05}" # Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
local counting="${7:-true}" # Count time since function has been launched if true, since script has been launched if false local keepLogging="${7:-0}" # Every keepLogging seconds, an alive log message is send. Setting this value to zero disables any alive logging.
local keepLogging="${8:-0}" # Log a standby message every X seconds. Set to zero to disable logging local counting="${8:-true}" # Count time since function has been launched (true), or since script has been launched (false)
local spinner="${9:-false}" # Show spinner (true), don't show spinner (false)
local noError="${10:-false}" # Log errors when reaching soft / hard max time (false), don't log errors on those triggers (true)
local callerName="${11:-false}" # Name of the function who called this function for debugging purposes, generally ${FUNCNAME[0]}
local soft_alert=false # Does a soft alert need to be triggered, if yes, send an alert once
local log_ttime=0 # local time instance for comparaison
local seconds_begin=$SECONDS # Seconds since the beginning of the script
local exec_time=0 # Seconds since the beginning of this function
local commandCount local commandCount
local command local command
@ -713,15 +750,60 @@ function ParallelExec {
while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do while [ $counter -lt "$commandCount" ] || [ ${#pidsArray[@]} -gt 0 ]; do
if [ $spinner == true ]; then
Spinner
fi
if [ $counting == true ]; then
exec_time=$(($SECONDS - $seconds_begin))
else
exec_time=$SECONDS
fi
if [ $keepLogging -ne 0 ]; then
if [ $((($exec_time + 1) % $keepLogging)) -eq 0 ]; then
if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1s
log_ttime=$exec_time
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
fi
fi
fi
if [ $exec_time -gt $softMaxTime ]; then
if [ $soft_alert != true ] && [ $softMaxTime -ne 0 ] && [ $noError != true ]; then
Logger "Max soft execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
soft_alert=true
SendAlert true
fi
if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
if [ $noError != true ]; then
Logger "Max hard execution time exceeded for task [$callerName] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
fi
for pid in "${pidsArray[@]}"; do
KillChilds $pid true
if [ $? == 0 ]; then
Logger "Task with pid [$pid] stopped successfully." "NOTICE"
else
Logger "Could not stop task with pid [$pid]." "ERROR"
fi
done
if [ $noError != true ]; then
SendAlert true
fi
# Return the number of commands that haven't run / finished run
return $(($commandCount - $counter + ${#pidsArray[@]}))
fi
fi
while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do while [ $counter -lt "$commandCount" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
if [ $readFromFile == true ]; then if [ $readFromFile == true ]; then
#TODO: Checked on FreeBSD 10, also check on Win
command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg") command=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$commandsArg")
else else
command="${commandsArray[$counter]}" command="${commandsArray[$counter]}"
fi fi
Logger "Running command [$command]." "DEBUG" Logger "Running command [$command]." "DEBUG"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$callerName.$SCRIPT_PID" 2>&1 &
pid=$! pid=$!
pidsArray+=($pid) pidsArray+=($pid)
commandsArrayPid[$pid]="$command" commandsArrayPid[$pid]="$command"
@ -916,11 +998,16 @@ function GetLocalOS {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
@ -939,6 +1026,9 @@ function GetLocalOS {
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
LOCAL_OS="msys" LOCAL_OS="msys"
;; ;;
*"Microsoft"*)
LOCAL_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
LOCAL_OS="MacOSX" LOCAL_OS="MacOSX"
;; ;;
@ -976,15 +1066,19 @@ function GetOs {
if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then if ls --help 2>&1 | grep -i "BusyBox" > /dev/null; then
localOsVar="BusyBox" localOsVar="BusyBox"
else else
localOsVar="$(uname -spio 2>&1)" # Detecting the special ubuntu userland in Windows 10 bash
if [ $? != 0 ]; then if grep -i Microsoft /proc/sys/kernel/osrelease > /dev/null 2>&1; then
localOsVar="$(uname -v 2>&1)" localOsVar="Microsoft"
else
localOsVar="$(uname -spio 2>&1)"
if [ $? != 0 ]; then if [ $? != 0 ]; then
localOsVar="$(uname)" localOsVar="$(uname -v 2>&1)"
if [ $? != 0 ]; then
localOsVar="$(uname)"
fi
fi fi
fi fi
fi fi
echo "$localOsVar" echo "$localOsVar"
} }
@ -1007,6 +1101,9 @@ ENDSSH
*"MINGW32"*|*"CYGWIN"*) *"MINGW32"*|*"CYGWIN"*)
REMOTE_OS="msys" REMOTE_OS="msys"
;; ;;
*"Microsoft"*)
REMOTE_OS="WinNT10"
;;
*"Darwin"*) *"Darwin"*)
REMOTE_OS="MacOSX" REMOTE_OS="MacOSX"
;; ;;
@ -1043,7 +1140,8 @@ function RunLocalCommand {
Logger "Running command [$command] on local host." "NOTICE" Logger "Running command [$command] on local host." "NOTICE"
eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 & eval "$command" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1 &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING
WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1077,7 +1175,7 @@ function RunRemoteCommand {
cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1' cmd=$SSH_CMD' "$command" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
Logger "cmd: $cmd" "DEBUG" Logger "cmd: $cmd" "DEBUG"
eval "$cmd" & eval "$cmd" &
WaitForTaskCompletion $! 0 $hardMaxTime ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 0 $hardMaxTime $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval -eq 0 ]; then if [ $retval -eq 0 ]; then
Logger "Command succeded." "NOTICE" Logger "Command succeded." "NOTICE"
@ -1110,7 +1208,7 @@ function RunBeforeHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1128,7 +1226,7 @@ function RunAfterHook {
pids="$pids;$!" pids="$pids;$!"
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 0 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 0 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -1136,18 +1234,16 @@ function CheckConnectivityRemoteHost {
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" & eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 60 180 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN" Logger "Cannot ping [$REMOTE_HOST]. Return code [$retval]." "WARN"
return $retval return $retval
fi fi
fi fi
fi
} }
function CheckConnectivity3rdPartyHosts { function CheckConnectivity3rdPartyHosts {
@ -1155,14 +1251,13 @@ function CheckConnectivity3rdPartyHosts {
local remote3rdPartySuccess local remote3rdPartySuccess
local retval local retval
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
remote3rdPartySuccess=false remote3rdPartySuccess=false
for i in $REMOTE_3RD_PARTY_HOSTS for i in $REMOTE_3RD_PARTY_HOSTS
do do
eval "$PING_CMD $i > /dev/null 2>&1" & eval "$PING_CMD $i > /dev/null 2>&1" &
WaitForTaskCompletion $! 180 360 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $! 180 360 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ]; then if [ $retval != 0 ]; then
Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE" Logger "Cannot ping 3rd party host [$i]. Return code [$retval]." "NOTICE"
@ -1178,11 +1273,8 @@ function CheckConnectivity3rdPartyHosts {
return 0 return 0
fi fi
fi fi
fi
} }
#__BEGIN_WITH_PARANOIA_DEBUG
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd { function RsyncPatternsAdd {
local patternType="${1}" # exclude or include local patternType="${1}" # exclude or include
@ -1299,66 +1391,12 @@ function PreInit {
COMMAND_SUDO="" COMMAND_SUDO=""
fi fi
## Set rsync default arguments ## Set compression executable and extension
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
## Set compression executable and extension
if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then if [ "$(IsInteger $COMPRESSION_LEVEL)" -eq 0 ]; then
COMPRESSION_LEVEL=3 COMPRESSION_LEVEL=3
fi fi
#TODO: Remote OS isn't defined yet
## Busybox fix (Termux xz command doesn't support compression at all) ## Busybox fix (Termux xz command doesn't support compression at all)
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$REMOTE_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$REMOTE_OS" == "Android" ]; then
compressionString="" compressionString=""
@ -1435,7 +1473,7 @@ function InitLocalOSSettings {
PING_CMD="ping -c 2 -i .2" PING_CMD="ping -c 2 -i .2"
fi fi
if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ]; then if [ "$LOCAL_OS" == "BusyBox" ] || [ "$LOCAL_OS" == "Android" ] || [ "$LOCAL_OS" == "msys" ]; then
PROCESS_STATE_CMD="echo none" PROCESS_STATE_CMD="echo none"
DF_CMD="df" DF_CMD="df"
else else
@ -1450,7 +1488,7 @@ function InitLocalOSSettings {
STAT_CMD="stat -f \"%Sm\"" STAT_CMD="stat -f \"%Sm\""
STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m" STAT_CTIME_MTIME_CMD="stat -f %N;%c;%m"
else else
# Tested on GNU stat and busybox # Tested on GNU stat, busybox and Cygwin
STAT_CMD="stat -c %y" STAT_CMD="stat -c %y"
STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y" STAT_CTIME_MTIME_CMD="stat -c %n;%Z;%Y"
fi fi
@ -1458,13 +1496,6 @@ function InitLocalOSSettings {
function InitRemoteOSSettings { function InitRemoteOSSettings {
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -E"
fi
fi
if [ "$REMOTE_OS" == "msys" ]; then if [ "$REMOTE_OS" == "msys" ]; then
REMOTE_FIND_CMD=$(dirname $BASH)/find REMOTE_FIND_CMD=$(dirname $BASH)/find
else else
@ -1482,6 +1513,71 @@ function InitRemoteOSSettings {
} }
function InitRsyncSettings {
## Set rsync default arguments
RSYNC_ARGS="-rltD"
if [ "$_DRYRUN" == true ]; then
RSYNC_DRY_ARG="-n"
DRY_WARNING="/!\ DRY RUN "
else
RSYNC_DRY_ARG=""
fi
RSYNC_ATTR_ARGS=""
if [ "$PRESERVE_PERMISSIONS" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -p"
fi
if [ "$PRESERVE_OWNER" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -o"
fi
if [ "$PRESERVE_GROUP" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -g"
fi
if [ "$PRESERVE_EXECUTABILITY" != "no" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" --executability"
fi
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ] && [ "$LOCAL_OS" != "msys" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
if [ "$PRESERVE_ACL" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -A"
fi
if [ "$PRESERVE_XATTR" == "yes" ]; then
RSYNC_ATTR_ARGS=$RSYNC_ATTR_ARGS" -X"
fi
else
Logger "Disabling ACL and extended attributes synchronization on [$LOCAL_OS]." "NOTICE"
fi
if [ "$RSYNC_COMPRESS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -z"
fi
if [ "$COPY_SYMLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -L"
fi
if [ "$KEEP_DIRLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -K"
fi
if [ "$PRESERVE_HARDLINKS" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" -H"
fi
if [ "$CHECKSUM" == "yes" ]; then
RSYNC_TYPE_ARGS=$RSYNC_TYPE_ARGS" --checksum"
fi
if [ "$BANDWIDTH" != "" ] && [ "$BANDWIDTH" != "0" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
fi
if [ "$PARTIAL" == "yes" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --partial --partial-dir=\"$PARTIAL_DIR\""
RSYNC_PARTIAL_EXCLUDE="--exclude=\"$PARTIAL_DIR\""
fi
if [ "$DELTA_COPIES" != "no" ]; then
RSYNC_ARGS=$RSYNC_ARGS" --no-whole-file"
else
RSYNC_ARGS=$RSYNC_ARGS" --whole-file"
fi
}
## IFS debug function ## IFS debug function
function PrintIFS { function PrintIFS {
printf "IFS is: %q" "$IFS" printf "IFS is: %q" "$IFS"
@ -1720,7 +1816,7 @@ function CheckReplicaPaths {
_CheckReplicaPathsRemote "${TARGET[$__replicaDir]}" & _CheckReplicaPathsRemote "${TARGET[$__replicaDir]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" Logger "Cancelling task." "CRITICAL"
exit 1 exit 1
@ -1799,7 +1895,7 @@ function CheckDiskSpace {
_CheckDiskSpaceRemote "${TARGET[$__replicaDir]}" & _CheckDiskSpaceRemote "${TARGET[$__replicaDir]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
} }
@ -1847,7 +1943,7 @@ function CreateStateDirs {
_CreateStateDirsRemote "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" & _CreateStateDirsRemote "${TARGET[$__replicaDir]}${TARGET[$__stateDir]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" Logger "Cancelling task." "CRITICAL"
exit 1 exit 1
@ -1975,7 +2071,7 @@ function CheckLocks {
_CheckLocksRemote "${TARGET[$__lockFile]}" & _CheckLocksRemote "${TARGET[$__lockFile]}" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
Logger "Cancelling task." "CRITICAL" Logger "Cancelling task." "CRITICAL"
exit 1 exit 1
@ -2050,7 +2146,7 @@ function WriteLockFiles {
INITIATOR_LOCK_FILE_EXISTS=true INITIATOR_LOCK_FILE_EXISTS=true
TARGET_LOCK_FILE_EXISTS=true TARGET_LOCK_FILE_EXISTS=true
WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
for pid in "${pidArray[@]}"; do for pid in "${pidArray[@]}"; do
@ -2123,7 +2219,7 @@ function UnlockReplicas {
fi fi
if [ "$pids" != "" ]; then if [ "$pids" != "" ]; then
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -2288,7 +2384,7 @@ function syncAttrs {
fi fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
@ -2320,7 +2416,7 @@ function syncAttrs {
_getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" & _getFileCtimeMtimeRemote "${TARGET[$__replicaDir]}" "${TARGET[$__type]}" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-cleaned.$SCRIPT_PID" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 1800 0 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 1800 0 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
# If target gets updated first, then sync_attr must update initiators attrs first # If target gets updated first, then sync_attr must update initiators attrs first
# For join, remove leading replica paths # For join, remove leading replica paths
@ -2368,7 +2464,7 @@ function syncAttrs {
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" eval "$rsyncCmd"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
retval=$? retval=$?
if [ $retval != 0 ] && [ $retval != 24 ]; then if [ $retval != 0 ] && [ $retval != 24 ]; then
@ -2537,6 +2633,8 @@ function _deleteRemote {
local deletionListFromReplica local deletionListFromReplica
local loggerPrefix
if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then if [ "$replicaType" == "${INITIATOR[$__type]}" ]; then
deletionListFromReplica="${TARGET[$__type]}" deletionListFromReplica="${TARGET[$__type]}"
elif [ "$replicaType" == "${TARGET[$__type]}" ]; then elif [ "$replicaType" == "${TARGET[$__type]}" ]; then
@ -2671,12 +2769,16 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _DEBUG=$_DEBUG _DRYRUN=$
done < "$FILE_LIST" done < "$FILE_LIST"
ENDSSH ENDSSH
if [ -f "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then if [ -z "$RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID" ]; then
Logger "Remote Deletion:\n$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "VERBOSE" loggerPrefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX=""
Logger "$(cat $RUN_DIR/$PROGRAM.remote_deletion.$SCRIPT_PID)" "ERROR"
_LOGGER_PREFIX="$loggerPrefix"
fi fi
## Copy back the deleted failed file list ## Copy back the deleted failed file list
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\"" #rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" $REMOTE_USER@$REMOTE_HOST:\"{$failedDeleteList,$successDeleteList}\" \"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) -r --rsync-path=\"$RSYNC_PATH\" -e \"$RSYNC_SSH_CMD\" --include \"$(dirname ${TARGET[$__stateDir]})\" --include \"${TARGET[$__stateDir]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__failedDeletedListFile]}\" --include \"${TARGET[$__stateDir]}/$replicaType${TARGET[$__successDeletedListFile]}\" --exclude='*' $REMOTE_USER@$REMOTE_HOST:\"$(EscapeSpaces ${TARGET[$__replicaDir]})\" \"${INITIATOR[$__replicaDir]}\" > \"$RUN_DIR/$PROGRAM.remote_failed_deletion_list_copy.$SCRIPT_PID\""
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG" Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd" 2>> "$LOG_FILE" eval "$rsyncCmd" 2>> "$LOG_FILE"
result=$? result=$?
@ -2824,7 +2926,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -2869,7 +2971,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -2906,7 +3008,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then if [ "$resumeInitiator" == "${SYNC_ACTION[2]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[2]}" ]; then
if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then if [[ "$RSYNC_ATTR_ARGS" == *"-X"* ]] || [[ "$RSYNC_ATTR_ARGS" == *"-A"* ]]; then
syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR" syncAttrs "${INITIATOR[$__replicaDir]}" "$TARGET_SYNC_DIR"
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}" echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}" echo "${SYNC_ACTION[2]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -2973,7 +3075,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -3019,7 +3121,7 @@ function Sync {
targetPid="$!" targetPid="$!"
fi fi
WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING WaitForTaskCompletion "$initiatorPid;$targetPid" $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME $SLEEP_TIME $KEEP_LOGGING false true false ${FUNCNAME[0]}
if [ $? != 0 ]; then if [ $? != 0 ]; then
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION" IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
initiatorFail=false initiatorFail=false
@ -3168,7 +3270,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__backupDir]}" $CONFLICT_BACKUP_DAYS "conflict backup" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then if [ "$SOFT_DELETE" != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ]; then
@ -3183,7 +3285,7 @@ function SoftDelete {
_SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" & _SoftDeleteRemote "${TARGET[$__type]}" "${TARGET[$__replicaDir]}${TARGET[$__deleteDir]}" $SOFT_DELETE_DAYS "softdelete" &
pids="$pids;$!" pids="$pids;$!"
fi fi
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING WaitForTaskCompletion $pids 720 1800 $SLEEP_TIME $KEEP_LOGGING true true false ${FUNCNAME[0]}
fi fi
} }
@ -3202,9 +3304,9 @@ function _SummaryFromFile {
function Summary { function Summary {
local prefix local loggerPrefix
prefix="$_LOGGER_PREFIX" loggerPrefix="$_LOGGER_PREFIX"
_LOGGER_PREFIX="" _LOGGER_PREFIX=""
Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS" Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS"
@ -3225,7 +3327,7 @@ function Summary {
fi fi
_SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "- <<" _SummaryFromFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID" "- <<"
_LOGGER_PREFIX="$prefix" _LOGGER_PREFIX="$loggerPrefix"
} }
function Init { function Init {
@ -3463,9 +3565,18 @@ function SyncOnChanges {
local cmd local cmd
local retval local retval
if ! type inotifywait > /dev/null 2>&1 ; then local sleepTime
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1 if [ "$LOCAL_OS" == "MacOSX" ]; then
if ! type fswatch > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1
fi
else
if ! type inotifywait > /dev/null 2>&1 ; then
Logger "No inotifywait command found. Cannot monitor changes." "CRITICAL"
exit 1
fi
fi fi
Logger "#### Running osync in file monitor mode." "NOTICE" Logger "#### Running osync in file monitor mode." "NOTICE"
@ -3484,8 +3595,14 @@ function SyncOnChanges {
fi fi
Logger "#### Monitoring now." "NOTICE" Logger "#### Monitoring now." "NOTICE"
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" & if [ "$LOCAL_OS" == "MacOSX" ]; then
wait $! fswatch --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -1 "$INITIATOR_SYNC_DIR" > /dev/null &
# Mac fswatch doesn't have timeout switch, replacing wait $! with WaitForTaskCompletion without warning nor spinner and increased SLEEP_TIME to avoid cpu hogging. This sims wait $! with timeout
WaitForTaskCompletion $! 0 $MAX_WAIT 1 0 true false true ${FUNCNAME[0]}
else
inotifywait --exclude $OSYNC_DIR $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -qq -r -e create -e modify -e delete -e move -e attrib --timeout "$MAX_WAIT" "$INITIATOR_SYNC_DIR" &
wait $!
fi
retval=$? retval=$?
if [ $retval == 0 ]; then if [ $retval == 0 ]; then
Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE" Logger "#### Changes detected, waiting $MIN_WAIT seconds before running next sync." "NOTICE"
@ -3704,6 +3821,7 @@ if [ $sync_on_changes == true ]; then
else else
GetRemoteOS GetRemoteOS
InitRemoteOSSettings InitRemoteOSSettings
InitRsyncSettings
if [ $no_maxtime == true ]; then if [ $no_maxtime == true ]; then
SOFT_MAX_EXEC_TIME=0 SOFT_MAX_EXEC_TIME=0
HARD_MAX_EXEC_TIME=0 HARD_MAX_EXEC_TIME=0

Write Preview
Loading…
Cancel
Save