@ -1,56 +1,60 @@
#!/usr/bin/env bash
#TODO(critical): handle conflict prevalance, especially in sync_attrs function
#TODO(critical): writelockfiles remote does not shut execution
PROGRAM = "osync" # Rsync based two way sync engine with fault tolerance
AUTHOR = "(C) 2013-2016 by Orsiris de Jong"
CONTACT = "http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION = 1.2-dev-parallel
PROGRAM_BUILD = 2016082 201
PROGRAM_BUILD = 2016082 803
IS_STABLE = no
# Function Name Is parallel #__WITH_PARANOIA_DEBUG
# GetLocalOS no #__WITH_PARANOIA_DEBUG
# InitLocalOSSettings no #__WITH_PARANOIA_DEBUG
# CheckEnvironment no #__WITH_PARANOIA_DEBUG
# PreInit no #__WITH_PARANOIA_DEBUG
# Init no #__WITH_PARANOIA_DEBUG
# PostInit no #__WITH_PARANOIA_DEBUG
# GetRemoteOS no #__WITH_PARANOIA_DEBUG
# InitRemoteOSSettings no #__WITH_PARANOIA_DEBUG
# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG
# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG
# RunBeforeHook yes #__WITH_PARANOIA_DEBUG
# Main no #__WITH_PARANOIA_DEBUG
# CreateStateDirs yes #__WITH_PARANOIA_DEBUG
# CheckLocks yes #__WITH_PARANOIA_DEBUG
# WriteLockFiles yes #__WITH_PARANOIA_DEBUG
# Sync no #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# delete_list yes #__WITH_PARANOIA_DEBUG
# delete_list yes #__WITH_PARANOIA_DEBUG
# sync_attrs no #__WITH_PARANOIA_DEBUG
# _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG
# sync_update yes #__WITH_PARANOIA_DEBUG
# sync_update yes #__WITH_PARANOIA_DEBUG
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# SoftDelete yes #__WITH_PARANOIA_DEBUG
# RunAfterHook yes #__WITH_PARANOIA_DEBUG
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
# CleanUp no #__WITH_PARANOIA_DEBUG
# Execution order
# Function Name Is parallel #__WITH_PARANOIA_DEBUG
# GetLocalOS no #__WITH_PARANOIA_DEBUG
# InitLocalOSSettings no #__WITH_PARANOIA_DEBUG
# CheckEnvironment no #__WITH_PARANOIA_DEBUG
# PreInit no #__WITH_PARANOIA_DEBUG
# Init no #__WITH_PARANOIA_DEBUG
# PostInit no #__WITH_PARANOIA_DEBUG
# GetRemoteOS no #__WITH_PARANOIA_DEBUG
# InitRemoteOSSettings no #__WITH_PARANOIA_DEBUG
# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG
# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG
# RunBeforeHook yes #__WITH_PARANOIA_DEBUG
# Main no #__WITH_PARANOIA_DEBUG
# CreateStateDirs yes #__WITH_PARANOIA_DEBUG
# CheckLocks yes #__WITH_PARANOIA_DEBUG
# WriteLockFiles yes #__WITH_PARANOIA_DEBUG
# Sync no #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# delete_list yes #__WITH_PARANOIA_DEBUG
# delete_list yes #__WITH_PARANOIA_DEBUG
# sync_attrs no #__WITH_PARANOIA_DEBUG
# _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG
# sync_update yes #__WITH_PARANOIA_DEBUG
# sync_update yes #__WITH_PARANOIA_DEBUG
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# tree_list yes #__WITH_PARANOIA_DEBUG
# SoftDelete yes #__WITH_PARANOIA_DEBUG
# RunAfterHook yes #__WITH_PARANOIA_DEBUG
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
# CleanUp no #__WITH_PARANOIA_DEBUG
#### MINIMAL-FUNCTION-SET BEGIN ####
## FUNC_BUILD=2016082204
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## FUNC_BUILD=2016082802
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
## To use in a program, define the following variables:
## PROGRAM=program-name
## INSTANCE_ID=program-instance-name
## _DEBUG=yes/no
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
if ! type " $BASH " > /dev/null; then
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
exit 127
@ -75,6 +79,9 @@ fi
ERROR_ALERT = 0
WARN_ALERT = 0
# Current log
CURRENT_LOG =
## allow function call checks #__WITH_PARANOIA_DEBUG
if [ " $_PARANOIA_DEBUG " = = "yes" ] ; then #__WITH_PARANOIA_DEBUG
_DEBUG = yes #__WITH_PARANOIA_DEBUG
@ -96,6 +103,10 @@ SCRIPT_PID=$$
LOCAL_USER = $( whoami)
LOCAL_HOST = $( hostname)
if [ " $PROGRAM " = = "" ] ; then
PROGRAM = "ofunctions"
fi
## Default log file until config file is loaded
if [ -w /var/log ] ; then
LOG_FILE = " /var/log/ $PROGRAM .log "
@ -136,6 +147,7 @@ function _Logger {
local evalue = " ${ 3 } " # What to log to stderr
echo -e " $lvalue " >> " $LOG_FILE "
CURRENT_LOG = " $CURRENT_LOG " $'\n' " $lvalue "
if [ " $_LOGGER_STDERR " -eq 1 ] ; then
cat <<< " $evalue " 1>& 2
@ -267,11 +279,14 @@ function KillAllChilds {
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
__CheckArguments 0 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
local runAlert = " ${ 1 :- false } " # Specifies if current message is sent while running or at the end of a run
__CheckArguments 0-1 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
local mail_no_attachment =
local attachment_command =
local subject =
local body =
# Windows specific settings
local encryption_string =
@ -300,7 +315,8 @@ function SendAlert {
else
mail_no_attachment = 0
fi
MAIL_ALERT_MSG = " $MAIL_ALERT_MSG " $'\n\n' $( tail -n 50 " $LOG_FILE " )
body = " $MAIL_ALERT_MSG " $'\n\n' " $CURRENT_LOG "
if [ $ERROR_ALERT -eq 1 ] ; then
subject = " Error alert for $INSTANCE_ID "
elif [ $WARN_ALERT -eq 1 ] ; then
@ -309,11 +325,17 @@ function SendAlert {
subject = " Alert for $INSTANCE_ID "
fi
if [ $runAlert = = true ] ; then
subject = " Currently runing - $subject "
else
subject = " Fnished run - $subject "
fi
if [ " $mail_no_attachment " -eq 0 ] ; then
attachment_command = " -a $ALERT_LOG_FILE "
fi
if type mutt > /dev/null 2>& 1 ; then
echo " $MAIL_ALERT_MSG " | $( type -p mutt) -x -s " $subject " $DESTINATION_MAILS $attachment_command
echo " $ body " | $( type -p mutt) -x -s " $subject " $DESTINATION_MAILS $attachment_command
if [ $? != 0 ] ; then
Logger " Cannot send alert mail via $( type -p mutt) !!! " "WARN"
else
@ -330,10 +352,10 @@ function SendAlert {
else
attachment_command = ""
fi
echo " $ MAIL_ALERT_MSG " | $( type -p mail) $attachment_command -s " $subject " $DESTINATION_MAILS
echo " $ body " | $( type -p mail) $attachment_command -s " $subject " $DESTINATION_MAILS
if [ $? != 0 ] ; then
Logger " Cannot send alert mail via $( type -p mail) with attachments !!! " "WARN"
echo " $ MAIL_ALERT_MSG " | $( type -p mail) -s " $subject " $DESTINATION_MAILS
echo " $ body " | $( type -p mail) -s " $subject " $DESTINATION_MAILS
if [ $? != 0 ] ; then
Logger " Cannot send alert mail via $( type -p mail) without attachments !!! " "WARN"
else
@ -347,7 +369,7 @@ function SendAlert {
fi
if type sendmail > /dev/null 2>& 1 ; then
echo -e " Subject: $subject \r\n $ MAIL_ALERT_MSG " | $( type -p sendmail) $DESTINATION_MAILS
echo -e " Subject: $subject \r\n $ body " | $( type -p sendmail) $DESTINATION_MAILS
if [ $? != 0 ] ; then
Logger " Cannot send alert mail via $( type -p sendmail) !!! " "WARN"
else
@ -370,7 +392,7 @@ function SendAlert {
if [ " $SMTP_USER " != "" ] && [ " $SMTP_USER " != "" ] ; then
auth_string = " -auth -user \" $SMTP_USER \" -pass \" $SMTP_PASSWORD \" "
fi
$( type mailsend.exe) -f $SENDER_MAIL -t " $DESTINATION_MAILS " -sub " $subject " -M " $ MAIL_ALERT_MSG " -attach " $attachment " -smtp " $SMTP_SERVER " -port " $SMTP_PORT " $encryption_string $auth_string
$( type mailsend.exe) -f $SENDER_MAIL -t " $DESTINATION_MAILS " -sub " $subject " -M " $ body " -attach " $attachment " -smtp " $SMTP_SERVER " -port " $SMTP_PORT " $encryption_string $auth_string
if [ $? != 0 ] ; then
Logger " Cannot send mail via $( type mailsend.exe) !!! " "WARN"
else
@ -386,7 +408,7 @@ function SendAlert {
else
SMTP_OPTIONS = ""
fi
$( type -p sendemail) -f $SENDER_MAIL -t " $DESTINATION_MAILS " -u " $subject " -m " $ MAIL_ALERT_MSG " -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>& 1
$( type -p sendemail) -f $SENDER_MAIL -t " $DESTINATION_MAILS " -u " $subject " -m " $ body " -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>& 1
if [ $? != 0 ] ; then
Logger " Cannot send alert mail via $( type -p sendemail) !!! " "WARN"
else
@ -397,7 +419,7 @@ function SendAlert {
# pfSense specific
if [ -f /usr/local/bin/mail.php ] ; then
echo " $ MAIL_ALERT_MSG " | /usr/local/bin/mail.php -s= " $subject "
echo " $ body " | /usr/local/bin/mail.php -s= " $subject "
if [ $? != 0 ] ; then
Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
else
@ -616,6 +638,7 @@ function joinString {
# Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
# Warning: Don't imbricate this function into another run if you plan to use the global variable output
function WaitForTaskCompletion {
local pids = " ${ 1 } " # pids to wait for, separated by semi-colon
local soft_max_time = " ${ 2 } " # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
@ -636,9 +659,13 @@ function WaitForTaskCompletion {
local retval = 0 # return value of monitored pid process
local errorcount = 0 # Number of pids that finished with errors
local pid # Current pid working on
local pidCount # number of given pids
local pidState # State of the process
local pidsArray # Array of currently running pids
local newPidsArray # New array of currently running pids
IFS = ';' read -a pidsArray <<< " $pids "
pidCount = ${# pidsArray [@] }
@ -667,7 +694,7 @@ function WaitForTaskCompletion {
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ] ; then
Logger " Max soft execution time exceeded for task [ $caller_name ] with pids [ $( joinString , ${ pidsArray [@] } ) ]. " "WARN"
soft_alert = 1
SendAlert
SendAlert true
fi
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ] ; then
@ -680,7 +707,7 @@ function WaitForTaskCompletion {
Logger " Could not stop task with pid [ $pid ]. " "ERROR"
fi
done
SendAlert
SendAlert true
errrorcount = $(( errorcount+1))
fi
fi
@ -699,17 +726,18 @@ function WaitForTaskCompletion {
retval = $?
if [ $retval -ne 0 ] ; then
errorcount = $(( errorcount+1))
Logger " ${ FUNCNAME [0] } called by [ $caller_name ] finished monitoring [ $pid ] with exitcode [ $re sul t]. " "DEBUG"
Logger " ${ FUNCNAME [0] } called by [ $caller_name ] finished monitoring [ $pid ] with exitcode [ $re tval ]. " "DEBUG"
if [ " $WAIT_FOR_TASK_COMPLETION " = = "" ] ; then
WAIT_FOR_TASK_COMPLETION = " $pid : $re sul t"
WAIT_FOR_TASK_COMPLETION = " $pid : $re tval "
else
WAIT_FOR_TASK_COMPLETION = " ; $pid : $re sul t"
WAIT_FOR_TASK_COMPLETION = " ; $pid : $re tval "
fi
fi
fi
done
pidsArray = ( " ${ newPidsArray [@] } " )
# Trivial wait time for bash to not eat up all CPU
sleep $SLEEP_TIME
done
@ -723,6 +751,66 @@ function WaitForTaskCompletion {
fi
}
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
# Returns the number of non zero exit codes from commands
function ParallelExec {
local numberOfProcesses = " ${ 1 } " # Number of simultaneous commands to run
local commandsArg = " ${ 2 } " # Semi-colon separated list of commands
local pid
local runningPids = 0
local counter = 0
local commandsArray
local pidsArray
local newPidsArray
local retval
local retvalAll = 0
local pidState
local commandsArrayPid
IFS = ';' read -r -a commandsArray <<< " $commandsArg "
Logger " Runnning ${# commandsArray [@] } commands in $numberOfProcesses simultaneous processes. " "DEBUG"
while [ $counter -lt " ${# commandsArray [@] } " ] || [ ${# pidsArray [@] } -gt 0 ] ; do
while [ $counter -lt " ${# commandsArray [@] } " ] && [ ${# pidsArray [@] } -lt $numberOfProcesses ] ; do
Logger " Running command [ ${ commandsArray [ $counter ] } ]. " "DEBUG"
eval " ${ commandsArray [ $counter ] } " &
pid = $!
pidsArray += ( $pid )
commandsArrayPid[ $pid ] = " ${ commandsArray [ $counter ] } "
counter = $(( counter+1))
done
newPidsArray = ( )
for pid in " ${ pidsArray [@] } " ; do
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
if kill -0 $pid > /dev/null 2>& 1; then
pidState = $( ps -p$pid -o state = 2 > /dev/null)
if [ " $pidState " != "D" ] && [ " $pidState " != "Z" ] ; then
newPidsArray += ( $pid )
fi
else
# pid is dead, get it's exit code from wait command
wait $pid
retval = $?
if [ $retval -ne 0 ] ; then
Logger " Command [ ${ commandsArrayPid [ $pid ] } ] failed with exit code [ $retval ]. " "ERROR"
retvalAll = $(( retvalAll+1))
fi
fi
done
pidsArray = ( " ${ newPidsArray [@] } " )
# Trivial wait time for bash to not eat up all CPU
sleep $SLEEP_TIME
done
return $retvalAll
}
function CleanUp {
__CheckArguments 0 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
@ -1016,7 +1104,7 @@ function CheckConnectivityRemoteHost {
if [ " $REMOTE_HOST_PING " != "no" ] && [ " $REMOTE_OPERATION " != "no" ] ; then
eval " $PING_CMD $REMOTE_HOST > /dev/null 2>&1 " &
WaitForTaskCompletion $! 1 0 180 ${ FUNCNAME [0] } true $KEEP_LOGGING
WaitForTaskCompletion $! 6 0 180 ${ FUNCNAME [0] } true $KEEP_LOGGING
if [ $? != 0 ] ; then
Logger " Cannot ping $REMOTE_HOST " "ERROR"
return 1
@ -1061,12 +1149,15 @@ function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ " $_DEBUG " = = "yes" ] ; then
local number_of_arguments = " ${ 1 } " # Number of arguments the tested function should have
local number_of_given_arguments = " ${ 2 } " # Number of arguments that have been passed
local function_name = " ${ 3 } " # Function name that called __CheckArguments
local numberOfArguments = " ${ 1 } " # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments = " ${ 2 } " # Number of arguments that have been passed
local functionName = " ${ 3 } " # Function name that called __CheckArguments
local minArgs
local maxArgs
if [ " $_PARANOIA_DEBUG " = = "yes" ] ; then
Logger " Entering function [ $function_name ]. " "DEBUG"
Logger " Entering function [ $function N ame]. " "DEBUG"
fi
# All arguments of the function to check are passed as array in ${4} (the function call waits for $@)
@ -1074,23 +1165,31 @@ function __CheckArguments {
# In order to avoid this, we need to iterate over ${4} and count
local iterate = 4
local fetch_arguments = 1
local arg_list = ""
while [ $fetch_arguments -eq 1 ] ; do
local fetchArguments = 1
local argList = ""
local countedArguments
while [ $fetchArguments -eq 1 ] ; do
cmd = 'argument=${' $iterate '}'
eval $cmd
if [ " $argument " = "" ] ; then
fetch _a rguments= 0
fetch A rguments= 0
else
arg _l ist= " $arg_list [Argument $(( $iterate - 3 )) : $argument ] "
arg L ist= " $arg_list [Argument $(( $iterate - 3 )) : $argument ] "
iterate = $(( $iterate + 1 ))
fi
done
local counted_arguments = $(( iterate-4))
countedArguments = $(( iterate-4))
if [ $( IsNumeric " $numberOfArguments " ) -eq 1 ] ; then
minArgs = $numberOfArguments
maxArgs = $numberOfArguments
else
IFS = '-' read minArgs maxArgs <<< " $numberOfArguments "
fi
if [ $counted_arguments -ne $number_of_arguments ] ; then
Logger " Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments , count: $counted_arguments , bash seen: $number_of_given_arguments . see log file. " "ERROR"
Logger " Arguments passed: $arg_list " "ERROR"
if ! ( [ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ] ) ; then
Logger " Function $function Name may have inconsistent number of arguments. Expected min: $minArgs , max: $maxArgs , count: $countedArguments , bash seen: $numberOfGivenA rguments. see log file. " "ERROR"
Logger " Arguments passed: $arg L ist" "ERROR"
fi
fi
}
@ -1334,7 +1433,6 @@ function InitLocalOSSettings {
function InitRemoteOSSettings {
__CheckArguments 0 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
#TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
if [ " $PRESERVE_EXECUTABILITY " != "no" ] ; then
if [ " $LOCAL_OS " != "MacOSX" ] && [ " $REMOTE_OS " != "MacOSX" ] ; then
@ -1448,6 +1546,24 @@ function CheckEnvironment {
function CheckCurrentConfig {
__CheckArguments 0 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars = ( CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
for i in " ${ yes_no_vars [@] } " ; do
test = " if [ \"\$ $i \" != \"yes\" ] && [ \"\$ $i \" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi "
eval " $test "
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars = ( MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in " ${ num_vars [@] } " ; do
test = " if [ $( IsNumeric \" \$ $i \" ) -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi "
eval " $test "
done
}
function CheckCurrentConfigAll {
__CheckArguments 0 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
if [ " $INSTANCE_ID " = = "" ] ; then
Logger "No INSTANCE_ID defined in config file." "CRITICAL"
exit 1
@ -1463,21 +1579,11 @@ function CheckCurrentConfig {
exit 1
fi
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars = ( CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
for i in " ${ yes_no_vars [@] } " ; do
test = " if [ \"\$ $i \" != \"yes\" ] && [ \"\$ $i \" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi "
eval " $test "
done
# Check all variables that should contain a numerical value >= 0
declare -a num_vars = ( MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
for i in " ${ num_vars [@] } " ; do
test = " if [ $( IsNumeric \" \$ $i \" ) -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi "
eval " $test "
done
#TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
if [ " $REMOTE_OPERATION " = = "yes" ] && [ ! -f " $SSH_RSA_PRIVATE_KEY " ] ; then
Logger " Cannot find rsa private key [ $SSH_RSA_PRIVATE_KEY ]. Cannot connect to remote system. " "CRITICAL"
exit 1
fi
}
###### Osync specific functions (non shared)
@ -1792,22 +1898,24 @@ function CheckLocks {
function _WriteLockFilesLocal {
local lockfile = " ${ 1 } "
__CheckArguments 1 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
local replica_type = " ${ 2 } "
__CheckArguments 2 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
$COMMAND_SUDO echo " $SCRIPT_PID @ $INSTANCE_ID " > " $lockfile "
if [ $? != 0 ] ; then
Logger " Could not create lock file [$lockfile ]. " "CRITICAL"
Logger " Could not create lock file on local $replica_type in [$lockfile ]. " "CRITICAL"
exit 1
else
Logger " Locked replica o n [$lockfile ]. " "DEBUG"
Logger " Locked local $replica_type replica i n [$lockfile ]. " "DEBUG"
fi
}
function _WriteLockFilesRemote {
local lockfile = " ${ 1 } "
__CheckArguments 1 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
local replica_type = " ${ 2 } "
__CheckArguments 2 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
local =
local cmd
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
@ -1816,33 +1924,49 @@ function _WriteLockFilesRemote {
Logger " cmd: $cmd " "DEBUG"
eval " $cmd "
if [ $? != 0 ] ; then
Logger " Could not set lock on remote $replica_type replica ." "CRITICAL"
Logger " Could not create lock file on remote $replica_type in [ $lockfile ] ." "CRITICAL"
exit 1
else
Logger " Locked remote $replica_type replica ." "DEBUG"
Logger " Locked remote $replica_type replica in [$lockfile ] ." "DEBUG"
fi
}
function WriteLockFiles {
__CheckArguments 0 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
local pids
local initiatorPid
local targetPid
local pidArray
local pid
_WriteLockFilesLocal " ${ INITIATOR [2] } " " ${ INITIATOR [0] } " &
initiatorPid = " $! "
_WriteLockFilesLocal " ${ INITIATOR [2] } " &
pids = " $! "
if [ " $REMOTE_OPERATION " != "yes" ] ; then
_WriteLockFilesLocal " ${ TARGET [2] } " &
pids = " $pids ; $! "
_WriteLockFilesLocal " ${ TARGET [2] } " " ${ TARGET [0] } " &
targetPid= " $! "
else
_WriteLockFilesRemote " ${ TARGET [2] } " &
pids= " $pids ; $! "
_WriteLockFilesRemote " ${ TARGET [2] } " " ${ TARGET [0] } " &
targetPid= " $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
INITIATOR_LOCK_FILE_EXISTS = true
TARGET_LOCK_FILE_EXISTS = true
WaitForTaskCompletion " $initiatorPid ; $targetPid " 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
if [ $? -ne 0 ] ; then
IFS = ';' read -r -a pidArray <<< " $WAIT_FOR_TASK_COMPLETION "
for pid in " ${ pidArray [@] } " ; do
pid = ${ pid % : * }
if [ $pid = = $initiatorPid ] ; then
INITIATOR_LOCK_FILE_EXISTS = false
elif [ $pid = = $targetPid ] ; then
TARGET_LOCK_FILE_EXISTS = false
fi
done
Logger "Cancelling task." "CRITICAL"
exit 1
fi
LOCK_FILES_EXIST = 1
}
function _UnlockReplicasLocal {
@ -1884,21 +2008,29 @@ function UnlockReplicas {
local pids
if [ $_NOLOCKS -eq 1 ] || [ $LOCK_FILES_EXIST -eq 0 ] ; then
if [ $_NOLOCKS -eq 1 ] ; then
return 0
fi
_UnlockReplicasLocal " ${ INITIATOR [2] } " &
pids = " $! "
if [ $INITIATOR_LOCK_FILE_EXISTS = = true ] ; then
_UnlockReplicasLocal " ${ INITIATOR [2] } " &
pids = " $! "
fi
if [ " $REMOTE_OPERATION " != "yes" ] ; then
_UnlockReplicasLocal " ${ TARGET [2] } " &
pids = " $pids ; $! "
else
_UnlockReplicasRemote " ${ TARGET [2] } " &
pids = " $pids ; $! "
#WIP check if WaitForTaskCompletion can handle emppty pids like ";pid"
if [ $TARGET_LOCK_FILE_EXISTS = = true ] ; then
if [ " $REMOTE_OPERATION " != "yes" ] ; then
_UnlockReplicasLocal " ${ TARGET [2] } " &
pids = " $pids ; $! "
else
_UnlockReplicasRemote " ${ TARGET [2] } " &
pids = " $pids ; $! "
fi
fi
if [ " $pids " != "" ] ; then
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
}
###### Sync core functions
@ -1918,11 +2050,6 @@ function tree_list {
__CheckArguments 3 $# ${ FUNCNAME [0] } " $@ " #__WITH_PARANOIA_DEBUG
#WIP
#if [ "$replica_type" == "${INITIATOR[0]}" ]; then
# exit 12
#fi
escaped_replica_path = $( EscapeSpaces " $replica_path " )
Logger " Creating $replica_type replica file list [ $replica_path ]. " "NOTICE"
@ -2039,7 +2166,6 @@ function sync_attrs {
Logger " RSYNC_CMD: $rsync_cmd " "DEBUG"
eval " $rsync_cmd "
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
#WIP: return retval from process instead of err count if only one pid is tested
retval = $?
if [ $_VERBOSE -eq 1 ] && [ -f " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $SCRIPT_PID " ] ; then
Logger " List:\n $( cat $RUN_DIR /$PROGRAM .${ FUNCNAME [0] } .$SCRIPT_PID ) " "NOTICE"
@ -2120,7 +2246,6 @@ function sync_attrs {
Logger " RSYNC_CMD: $rsync_cmd " "DEBUG"
eval " $rsync_cmd "
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
#WIP: the same
retval = $?
if [ $_VERBOSE -eq 1 ] && [ -f " $RUN_DIR / $PROGRAM .attr-update. $dest_replica . $SCRIPT_PID " ] ; then
Logger " List:\n $( cat $RUN_DIR /$PROGRAM .attr-update.$dest_replica .$SCRIPT_PID ) " "NOTICE"
@ -3133,7 +3258,9 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then
CONFLICT_PREVALANCE = initiator
fi
LOCK_FILES_EXIST = 0
INITIATOR_LOCK_FILE_EXISTS = false
TARGET_LOCK_FILE_EXISTS = false
FORCE_UNLOCK = 0
no_maxtime = 0
opts = ""
@ -3291,6 +3418,7 @@ opts="${opts# *}"
if [ $_QUICK_SYNC -lt 2 ] ; then
CheckCurrentConfig
fi
CheckCurrentConfigAll
DATE = $( date)
Logger "-------------------------------------------------------------" "NOTICE"
Shadowigor
8 years ago