mirror of
https://github.com/deajan/osync
synced 2024-11-03 15:40:14 +00:00
Updated auto-generated scripts
This commit is contained in:
Shadowigor
parent
d8cb176ec6
commit
b7f21f2d89
@ -1,56 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#TODO(critical): handle conflict prevalance, especially in sync_attrs function
|
||||
#TODO(critical): writelockfiles remote does not shut execution
|
||||
|
||||
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
|
||||
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
|
||||
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
|
||||
PROGRAM_VERSION=1.2-dev-parallel
|
||||
PROGRAM_BUILD=2016082201
|
||||
PROGRAM_BUILD=2016082803
|
||||
IS_STABLE=no
|
||||
|
||||
# Function Name Is parallel #__WITH_PARANOIA_DEBUG
|
||||
# Execution order
|
||||
# Function Name Is parallel #__WITH_PARANOIA_DEBUG
|
||||
|
||||
# GetLocalOS no #__WITH_PARANOIA_DEBUG
|
||||
# InitLocalOSSettings no #__WITH_PARANOIA_DEBUG
|
||||
# CheckEnvironment no #__WITH_PARANOIA_DEBUG
|
||||
# PreInit no #__WITH_PARANOIA_DEBUG
|
||||
# Init no #__WITH_PARANOIA_DEBUG
|
||||
# PostInit no #__WITH_PARANOIA_DEBUG
|
||||
# GetRemoteOS no #__WITH_PARANOIA_DEBUG
|
||||
# InitRemoteOSSettings no #__WITH_PARANOIA_DEBUG
|
||||
# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG
|
||||
# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG
|
||||
# RunBeforeHook yes #__WITH_PARANOIA_DEBUG
|
||||
# Main no #__WITH_PARANOIA_DEBUG
|
||||
# CreateStateDirs yes #__WITH_PARANOIA_DEBUG
|
||||
# CheckLocks yes #__WITH_PARANOIA_DEBUG
|
||||
# WriteLockFiles yes #__WITH_PARANOIA_DEBUG
|
||||
# Sync no #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# delete_list yes #__WITH_PARANOIA_DEBUG
|
||||
# delete_list yes #__WITH_PARANOIA_DEBUG
|
||||
# sync_attrs no #__WITH_PARANOIA_DEBUG
|
||||
# _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG
|
||||
# sync_update yes #__WITH_PARANOIA_DEBUG
|
||||
# sync_update yes #__WITH_PARANOIA_DEBUG
|
||||
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
|
||||
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# SoftDelete yes #__WITH_PARANOIA_DEBUG
|
||||
# RunAfterHook yes #__WITH_PARANOIA_DEBUG
|
||||
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
|
||||
# CleanUp no #__WITH_PARANOIA_DEBUG
|
||||
# GetLocalOS no #__WITH_PARANOIA_DEBUG
|
||||
# InitLocalOSSettings no #__WITH_PARANOIA_DEBUG
|
||||
# CheckEnvironment no #__WITH_PARANOIA_DEBUG
|
||||
# PreInit no #__WITH_PARANOIA_DEBUG
|
||||
# Init no #__WITH_PARANOIA_DEBUG
|
||||
# PostInit no #__WITH_PARANOIA_DEBUG
|
||||
# GetRemoteOS no #__WITH_PARANOIA_DEBUG
|
||||
# InitRemoteOSSettings no #__WITH_PARANOIA_DEBUG
|
||||
# CheckReplicaPaths yes #__WITH_PARANOIA_DEBUG
|
||||
# CheckDiskSpace yes #__WITH_PARANOIA_DEBUG
|
||||
# RunBeforeHook yes #__WITH_PARANOIA_DEBUG
|
||||
# Main no #__WITH_PARANOIA_DEBUG
|
||||
# CreateStateDirs yes #__WITH_PARANOIA_DEBUG
|
||||
# CheckLocks yes #__WITH_PARANOIA_DEBUG
|
||||
# WriteLockFiles yes #__WITH_PARANOIA_DEBUG
|
||||
# Sync no #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# delete_list yes #__WITH_PARANOIA_DEBUG
|
||||
# delete_list yes #__WITH_PARANOIA_DEBUG
|
||||
# sync_attrs no #__WITH_PARANOIA_DEBUG
|
||||
# _get_file_ctime_mtime yes #__WITH_PARANOIA_DEBUG
|
||||
# sync_update yes #__WITH_PARANOIA_DEBUG
|
||||
# sync_update yes #__WITH_PARANOIA_DEBUG
|
||||
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
|
||||
# deletion_propagation yes #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# tree_list yes #__WITH_PARANOIA_DEBUG
|
||||
# SoftDelete yes #__WITH_PARANOIA_DEBUG
|
||||
# RunAfterHook yes #__WITH_PARANOIA_DEBUG
|
||||
# UnlockReplicas yes #__WITH_PARANOIA_DEBUG
|
||||
# CleanUp no #__WITH_PARANOIA_DEBUG
|
||||
|
||||
#### MINIMAL-FUNCTION-SET BEGIN ####
|
||||
|
||||
## FUNC_BUILD=2016082204
|
||||
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
||||
## FUNC_BUILD=2016082802
|
||||
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
||||
|
||||
## To use in a program, define the following variables:
|
||||
## PROGRAM=program-name
|
||||
## INSTANCE_ID=program-instance-name
|
||||
## _DEBUG=yes/no
|
||||
|
||||
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
|
||||
if ! type "$BASH" > /dev/null; then
|
||||
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
|
||||
exit 127
|
||||
@ -75,6 +79,9 @@ fi
|
||||
ERROR_ALERT=0
|
||||
WARN_ALERT=0
|
||||
|
||||
# Current log
|
||||
CURRENT_LOG=
|
||||
|
||||
## allow function call checks #__WITH_PARANOIA_DEBUG
|
||||
if [ "$_PARANOIA_DEBUG" == "yes" ];then #__WITH_PARANOIA_DEBUG
|
||||
_DEBUG=yes #__WITH_PARANOIA_DEBUG
|
||||
@ -96,6 +103,10 @@ SCRIPT_PID=$$
|
||||
LOCAL_USER=$(whoami)
|
||||
LOCAL_HOST=$(hostname)
|
||||
|
||||
if [ "$PROGRAM" == "" ]; then
|
||||
PROGRAM="ofunctions"
|
||||
fi
|
||||
|
||||
## Default log file until config file is loaded
|
||||
if [ -w /var/log ]; then
|
||||
LOG_FILE="/var/log/$PROGRAM.log"
|
||||
@ -136,6 +147,7 @@ function _Logger {
|
||||
local evalue="${3}" # What to log to stderr
|
||||
|
||||
echo -e "$lvalue" >> "$LOG_FILE"
|
||||
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
|
||||
|
||||
if [ "$_LOGGER_STDERR" -eq 1 ]; then
|
||||
cat <<< "$evalue" 1>&2
|
||||
@ -267,11 +279,14 @@ function KillAllChilds {
|
||||
|
||||
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
|
||||
function SendAlert {
|
||||
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
|
||||
|
||||
__CheckArguments 0-1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
local mail_no_attachment=
|
||||
local attachment_command=
|
||||
local subject=
|
||||
local body=
|
||||
|
||||
# Windows specific settings
|
||||
local encryption_string=
|
||||
@ -300,7 +315,8 @@ function SendAlert {
|
||||
else
|
||||
mail_no_attachment=0
|
||||
fi
|
||||
MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
|
||||
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
|
||||
|
||||
if [ $ERROR_ALERT -eq 1 ]; then
|
||||
subject="Error alert for $INSTANCE_ID"
|
||||
elif [ $WARN_ALERT -eq 1 ]; then
|
||||
@ -309,11 +325,17 @@ function SendAlert {
|
||||
subject="Alert for $INSTANCE_ID"
|
||||
fi
|
||||
|
||||
if [ $runAlert == true ]; then
|
||||
subject="Currently runing - $subject"
|
||||
else
|
||||
subject="Fnished run - $subject"
|
||||
fi
|
||||
|
||||
if [ "$mail_no_attachment" -eq 0 ]; then
|
||||
attachment_command="-a $ALERT_LOG_FILE"
|
||||
fi
|
||||
if type mutt > /dev/null 2>&1 ; then
|
||||
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
|
||||
echo "$body" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN"
|
||||
else
|
||||
@ -330,10 +352,10 @@ function SendAlert {
|
||||
else
|
||||
attachment_command=""
|
||||
fi
|
||||
echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
|
||||
echo "$body" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN"
|
||||
echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
|
||||
echo "$body" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN"
|
||||
else
|
||||
@ -347,7 +369,7 @@ function SendAlert {
|
||||
fi
|
||||
|
||||
if type sendmail > /dev/null 2>&1 ; then
|
||||
echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
|
||||
echo -e "Subject:$subject\r\n$body" | $(type -p sendmail) $DESTINATION_MAILS
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
|
||||
else
|
||||
@ -370,7 +392,7 @@ function SendAlert {
|
||||
if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
|
||||
auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
|
||||
fi
|
||||
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
|
||||
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$body" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
|
||||
else
|
||||
@ -386,7 +408,7 @@ function SendAlert {
|
||||
else
|
||||
SMTP_OPTIONS=""
|
||||
fi
|
||||
$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
|
||||
$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$body" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN"
|
||||
else
|
||||
@ -397,7 +419,7 @@ function SendAlert {
|
||||
|
||||
# pfSense specific
|
||||
if [ -f /usr/local/bin/mail.php ]; then
|
||||
echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject"
|
||||
echo "$body" | /usr/local/bin/mail.php -s="$subject"
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
|
||||
else
|
||||
@ -616,6 +638,7 @@ function joinString {
|
||||
# Time control function for background processes, suitable for multiple synchronous processes
|
||||
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
|
||||
# Warning: Don't imbricate this function into another run if you plan to use the global variable output
|
||||
|
||||
function WaitForTaskCompletion {
|
||||
local pids="${1}" # pids to wait for, separated by semi-colon
|
||||
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
|
||||
@ -636,9 +659,13 @@ function WaitForTaskCompletion {
|
||||
local retval=0 # return value of monitored pid process
|
||||
local errorcount=0 # Number of pids that finished with errors
|
||||
|
||||
local pid # Current pid working on
|
||||
local pidCount # number of given pids
|
||||
local pidState # State of the process
|
||||
|
||||
local pidsArray # Array of currently running pids
|
||||
local newPidsArray # New array of currently running pids
|
||||
|
||||
IFS=';' read -a pidsArray <<< "$pids"
|
||||
pidCount=${#pidsArray[@]}
|
||||
|
||||
@ -667,7 +694,7 @@ function WaitForTaskCompletion {
|
||||
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
|
||||
Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
|
||||
soft_alert=1
|
||||
SendAlert
|
||||
SendAlert true
|
||||
|
||||
fi
|
||||
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
|
||||
@ -680,7 +707,7 @@ function WaitForTaskCompletion {
|
||||
Logger "Could not stop task with pid [$pid]." "ERROR"
|
||||
fi
|
||||
done
|
||||
SendAlert
|
||||
SendAlert true
|
||||
errrorcount=$((errorcount+1))
|
||||
fi
|
||||
fi
|
||||
@ -699,17 +726,18 @@ function WaitForTaskCompletion {
|
||||
retval=$?
|
||||
if [ $retval -ne 0 ]; then
|
||||
errorcount=$((errorcount+1))
|
||||
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
|
||||
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
|
||||
if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then
|
||||
WAIT_FOR_TASK_COMPLETION="$pid:$result"
|
||||
WAIT_FOR_TASK_COMPLETION="$pid:$retval"
|
||||
else
|
||||
WAIT_FOR_TASK_COMPLETION=";$pid:$result"
|
||||
WAIT_FOR_TASK_COMPLETION=";$pid:$retval"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
pidsArray=("${newPidsArray[@]}")
|
||||
# Trivial wait time for bash to not eat up all CPU
|
||||
sleep $SLEEP_TIME
|
||||
done
|
||||
|
||||
@ -723,6 +751,66 @@ function WaitForTaskCompletion {
|
||||
fi
|
||||
}
|
||||
|
||||
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
|
||||
# Returns the number of non zero exit codes from commands
|
||||
function ParallelExec {
|
||||
local numberOfProcesses="${1}" # Number of simultaneous commands to run
|
||||
local commandsArg="${2}" # Semi-colon separated list of commands
|
||||
|
||||
local pid
|
||||
local runningPids=0
|
||||
local counter=0
|
||||
local commandsArray
|
||||
local pidsArray
|
||||
local newPidsArray
|
||||
local retval
|
||||
local retvalAll=0
|
||||
local pidState
|
||||
local commandsArrayPid
|
||||
|
||||
IFS=';' read -r -a commandsArray <<< "$commandsArg"
|
||||
|
||||
Logger "Runnning ${#commandsArray[@]} commands in $numberOfProcesses simultaneous processes." "DEBUG"
|
||||
|
||||
while [ $counter -lt "${#commandsArray[@]}" ] || [ ${#pidsArray[@]} -gt 0 ]; do
|
||||
|
||||
while [ $counter -lt "${#commandsArray[@]}" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
|
||||
Logger "Running command [${commandsArray[$counter]}]." "DEBUG"
|
||||
eval "${commandsArray[$counter]}" &
|
||||
pid=$!
|
||||
pidsArray+=($pid)
|
||||
commandsArrayPid[$pid]="${commandsArray[$counter]}"
|
||||
counter=$((counter+1))
|
||||
done
|
||||
|
||||
|
||||
newPidsArray=()
|
||||
for pid in "${pidsArray[@]}"; do
|
||||
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
|
||||
if kill -0 $pid > /dev/null 2>&1; then
|
||||
pidState=$(ps -p$pid -o state= 2 > /dev/null)
|
||||
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
|
||||
newPidsArray+=($pid)
|
||||
fi
|
||||
else
|
||||
# pid is dead, get it's exit code from wait command
|
||||
wait $pid
|
||||
retval=$?
|
||||
if [ $retval -ne 0 ]; then
|
||||
Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
|
||||
retvalAll=$((retvalAll+1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
pidsArray=("${newPidsArray[@]}")
|
||||
|
||||
# Trivial wait time for bash to not eat up all CPU
|
||||
sleep $SLEEP_TIME
|
||||
done
|
||||
|
||||
return $retvalAll
|
||||
}
|
||||
|
||||
function CleanUp {
|
||||
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
@ -1016,7 +1104,7 @@ function CheckConnectivityRemoteHost {
|
||||
|
||||
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
|
||||
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
|
||||
WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot ping $REMOTE_HOST" "ERROR"
|
||||
return 1
|
||||
@ -1061,12 +1149,15 @@ function __CheckArguments {
|
||||
# Checks the number of arguments of a function and raises an error if some are missing
|
||||
|
||||
if [ "$_DEBUG" == "yes" ]; then
|
||||
local number_of_arguments="${1}" # Number of arguments the tested function should have
|
||||
local number_of_given_arguments="${2}" # Number of arguments that have been passed
|
||||
local function_name="${3}" # Function name that called __CheckArguments
|
||||
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
|
||||
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
|
||||
local functionName="${3}" # Function name that called __CheckArguments
|
||||
|
||||
local minArgs
|
||||
local maxArgs
|
||||
|
||||
if [ "$_PARANOIA_DEBUG" == "yes" ]; then
|
||||
Logger "Entering function [$function_name]." "DEBUG"
|
||||
Logger "Entering function [$functionName]." "DEBUG"
|
||||
fi
|
||||
|
||||
# All arguments of the function to check are passed as array in ${4} (the function call waits for $@)
|
||||
@ -1074,23 +1165,31 @@ function __CheckArguments {
|
||||
# In order to avoid this, we need to iterate over ${4} and count
|
||||
|
||||
local iterate=4
|
||||
local fetch_arguments=1
|
||||
local arg_list=""
|
||||
while [ $fetch_arguments -eq 1 ]; do
|
||||
local fetchArguments=1
|
||||
local argList=""
|
||||
local countedArguments
|
||||
while [ $fetchArguments -eq 1 ]; do
|
||||
cmd='argument=${'$iterate'}'
|
||||
eval $cmd
|
||||
if [ "$argument" = "" ]; then
|
||||
fetch_arguments=0
|
||||
fetchArguments=0
|
||||
else
|
||||
arg_list="$arg_list [Argument $(($iterate-3)): $argument]"
|
||||
argList="$arg_list [Argument $(($iterate-3)): $argument]"
|
||||
iterate=$(($iterate+1))
|
||||
fi
|
||||
done
|
||||
local counted_arguments=$((iterate-4))
|
||||
countedArguments=$((iterate-4))
|
||||
|
||||
if [ $counted_arguments -ne $number_of_arguments ]; then
|
||||
Logger "Function $function_name may have inconsistent number of arguments. Expected: $number_of_arguments, count: $counted_arguments, bash seen: $number_of_given_arguments. see log file." "ERROR"
|
||||
Logger "Arguments passed: $arg_list" "ERROR"
|
||||
if [ $(IsNumeric "$numberOfArguments") -eq 1 ]; then
|
||||
minArgs=$numberOfArguments
|
||||
maxArgs=$numberOfArguments
|
||||
else
|
||||
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
|
||||
fi
|
||||
|
||||
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
|
||||
Logger "Function $functionName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments. see log file." "ERROR"
|
||||
Logger "Arguments passed: $argList" "ERROR"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@ -1334,7 +1433,6 @@ function InitLocalOSSettings {
|
||||
function InitRemoteOSSettings {
|
||||
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
#TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes
|
||||
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
|
||||
if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
|
||||
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
|
||||
@ -1448,6 +1546,24 @@ function CheckEnvironment {
|
||||
function CheckCurrentConfig {
|
||||
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
# Check all variables that should contain "yes" or "no"
|
||||
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
|
||||
for i in "${yes_no_vars[@]}"; do
|
||||
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
|
||||
# Check all variables that should contain a numerical value >= 0
|
||||
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
|
||||
for i in "${num_vars[@]}"; do
|
||||
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
}
|
||||
|
||||
function CheckCurrentConfigAll {
|
||||
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
if [ "$INSTANCE_ID" == "" ]; then
|
||||
Logger "No INSTANCE_ID defined in config file." "CRITICAL"
|
||||
exit 1
|
||||
@ -1463,21 +1579,11 @@ function CheckCurrentConfig {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check all variables that should contain "yes" or "no"
|
||||
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
|
||||
for i in "${yes_no_vars[@]}"; do
|
||||
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
|
||||
# Check all variables that should contain a numerical value >= 0
|
||||
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
|
||||
for i in "${num_vars[@]}"; do
|
||||
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
|
||||
#TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
|
||||
if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then
|
||||
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
###### Osync specific functions (non shared)
|
||||
@ -1792,22 +1898,24 @@ function CheckLocks {
|
||||
|
||||
function _WriteLockFilesLocal {
|
||||
local lockfile="${1}"
|
||||
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
local replica_type="${2}"
|
||||
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile"
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Could not create lock file [$lockfile]." "CRITICAL"
|
||||
Logger "Could not create lock file on local $replica_type in [$lockfile]." "CRITICAL"
|
||||
exit 1
|
||||
else
|
||||
Logger "Locked replica on [$lockfile]." "DEBUG"
|
||||
Logger "Locked local $replica_type replica in [$lockfile]." "DEBUG"
|
||||
fi
|
||||
}
|
||||
|
||||
function _WriteLockFilesRemote {
|
||||
local lockfile="${1}"
|
||||
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
local replica_type="${2}"
|
||||
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
local cmd=
|
||||
local cmd
|
||||
|
||||
CheckConnectivity3rdPartyHosts
|
||||
CheckConnectivityRemoteHost
|
||||
@ -1816,33 +1924,49 @@ function _WriteLockFilesRemote {
|
||||
Logger "cmd: $cmd" "DEBUG"
|
||||
eval "$cmd"
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Could not set lock on remote $replica_type replica." "CRITICAL"
|
||||
Logger "Could not create lock file on remote $replica_type in [$lockfile]." "CRITICAL"
|
||||
exit 1
|
||||
else
|
||||
Logger "Locked remote $replica_type replica." "DEBUG"
|
||||
Logger "Locked remote $replica_type replica in [$lockfile]." "DEBUG"
|
||||
fi
|
||||
}
|
||||
|
||||
function WriteLockFiles {
|
||||
__CheckArguments 0 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
local pids
|
||||
local initiatorPid
|
||||
local targetPid
|
||||
local pidArray
|
||||
local pid
|
||||
|
||||
_WriteLockFilesLocal "${INITIATOR[2]}" "${INITIATOR[0]}"&
|
||||
initiatorPid="$!"
|
||||
|
||||
_WriteLockFilesLocal "${INITIATOR[2]}" &
|
||||
pids="$!"
|
||||
if [ "$REMOTE_OPERATION" != "yes" ]; then
|
||||
_WriteLockFilesLocal "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
_WriteLockFilesLocal "${TARGET[2]}" "${TARGET[0]}" &
|
||||
targetPid="$!"
|
||||
else
|
||||
_WriteLockFilesRemote "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
_WriteLockFilesRemote "${TARGET[2]}" "${TARGET[0]}" &
|
||||
targetPid="$!"
|
||||
fi
|
||||
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
|
||||
INITIATOR_LOCK_FILE_EXISTS=true
|
||||
TARGET_LOCK_FILE_EXISTS=true
|
||||
WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
if [ $? -ne 0 ]; then
|
||||
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
|
||||
for pid in "${pidArray[@]}"; do
|
||||
pid=${pid%:*}
|
||||
if [ $pid == $initiatorPid ]; then
|
||||
INITIATOR_LOCK_FILE_EXISTS=false
|
||||
elif [ $pid == $targetPid ]; then
|
||||
TARGET_LOCK_FILE_EXISTS=false
|
||||
fi
|
||||
done
|
||||
|
||||
Logger "Cancelling task." "CRITICAL"
|
||||
exit 1
|
||||
fi
|
||||
LOCK_FILES_EXIST=1
|
||||
}
|
||||
|
||||
function _UnlockReplicasLocal {
|
||||
@ -1884,21 +2008,29 @@ function UnlockReplicas {
|
||||
|
||||
local pids
|
||||
|
||||
if [ $_NOLOCKS -eq 1 ] || [ $LOCK_FILES_EXIST -eq 0 ]; then
|
||||
if [ $_NOLOCKS -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
_UnlockReplicasLocal "${INITIATOR[2]}" &
|
||||
pids="$!"
|
||||
|
||||
if [ "$REMOTE_OPERATION" != "yes" ]; then
|
||||
_UnlockReplicasLocal "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
else
|
||||
_UnlockReplicasRemote "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then
|
||||
_UnlockReplicasLocal "${INITIATOR[2]}" &
|
||||
pids="$!"
|
||||
fi
|
||||
|
||||
#WIP check if WaitForTaskCompletion can handle emppty pids like ";pid"
|
||||
if [ $TARGET_LOCK_FILE_EXISTS == true ]; then
|
||||
if [ "$REMOTE_OPERATION" != "yes" ]; then
|
||||
_UnlockReplicasLocal "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
else
|
||||
_UnlockReplicasRemote "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$pids" != "" ]; then
|
||||
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
fi
|
||||
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
}
|
||||
|
||||
###### Sync core functions
|
||||
@ -1918,11 +2050,6 @@ function tree_list {
|
||||
|
||||
__CheckArguments 3 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
||||
|
||||
#WIP
|
||||
#if [ "$replica_type" == "${INITIATOR[0]}" ]; then
|
||||
# exit 12
|
||||
#fi
|
||||
|
||||
escaped_replica_path=$(EscapeSpaces "$replica_path")
|
||||
|
||||
Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
|
||||
@ -2039,7 +2166,6 @@ function sync_attrs {
|
||||
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
||||
eval "$rsync_cmd"
|
||||
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
|
||||
#WIP: return retval from process instead of err count if only one pid is tested
|
||||
retval=$?
|
||||
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
||||
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
||||
@ -2120,7 +2246,6 @@ function sync_attrs {
|
||||
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
||||
eval "$rsync_cmd"
|
||||
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
|
||||
#WIP: the same
|
||||
retval=$?
|
||||
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
||||
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
|
||||
@ -3133,7 +3258,9 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then
|
||||
CONFLICT_PREVALANCE=initiator
|
||||
fi
|
||||
|
||||
LOCK_FILES_EXIST=0
|
||||
INITIATOR_LOCK_FILE_EXISTS=false
|
||||
TARGET_LOCK_FILE_EXISTS=false
|
||||
|
||||
FORCE_UNLOCK=0
|
||||
no_maxtime=0
|
||||
opts=""
|
||||
@ -3291,6 +3418,7 @@ opts="${opts# *}"
|
||||
if [ $_QUICK_SYNC -lt 2 ]; then
|
||||
CheckCurrentConfig
|
||||
fi
|
||||
CheckCurrentConfigAll
|
||||
|
||||
DATE=$(date)
|
||||
Logger "-------------------------------------------------------------" "NOTICE"
|
||||
|
||||
250
osync.sh
250
osync.sh
@ -1,23 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#TODO(critical): handle conflict prevalance, especially in sync_attrs function
|
||||
#TODO(critical): writelockfiles remote does not shut execution
|
||||
|
||||
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
|
||||
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
|
||||
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
|
||||
PROGRAM_VERSION=1.2-dev-parallel
|
||||
PROGRAM_BUILD=2016082201
|
||||
PROGRAM_BUILD=2016082803
|
||||
IS_STABLE=no
|
||||
|
||||
# Execution order
|
||||
|
||||
|
||||
#### MINIMAL-FUNCTION-SET BEGIN ####
|
||||
|
||||
## FUNC_BUILD=2016082204
|
||||
## BEGIN Generic functions for osync & obackup written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
||||
## FUNC_BUILD=2016082802
|
||||
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
||||
|
||||
## To use in a program, define the following variables:
|
||||
## PROGRAM=program-name
|
||||
## INSTANCE_ID=program-instance-name
|
||||
## _DEBUG=yes/no
|
||||
|
||||
## type -p does not work on platforms other than linux (bash). If if does not work, always assume output is not a zero exitcode
|
||||
if ! type "$BASH" > /dev/null; then
|
||||
echo "Please run this script only with bash shell. Tested on bash >= 3.2"
|
||||
exit 127
|
||||
@ -42,6 +46,9 @@ fi
|
||||
ERROR_ALERT=0
|
||||
WARN_ALERT=0
|
||||
|
||||
# Current log
|
||||
CURRENT_LOG=
|
||||
|
||||
|
||||
## allow debugging from command line with _DEBUG=yes
|
||||
if [ ! "$_DEBUG" == "yes" ]; then
|
||||
@ -59,6 +66,10 @@ SCRIPT_PID=$$
|
||||
LOCAL_USER=$(whoami)
|
||||
LOCAL_HOST=$(hostname)
|
||||
|
||||
if [ "$PROGRAM" == "" ]; then
|
||||
PROGRAM="ofunctions"
|
||||
fi
|
||||
|
||||
## Default log file until config file is loaded
|
||||
if [ -w /var/log ]; then
|
||||
LOG_FILE="/var/log/$PROGRAM.log"
|
||||
@ -98,6 +109,7 @@ function _Logger {
|
||||
local evalue="${3}" # What to log to stderr
|
||||
|
||||
echo -e "$lvalue" >> "$LOG_FILE"
|
||||
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
|
||||
|
||||
if [ "$_LOGGER_STDERR" -eq 1 ]; then
|
||||
cat <<< "$evalue" 1>&2
|
||||
@ -220,10 +232,13 @@ function KillAllChilds {
|
||||
|
||||
# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
|
||||
function SendAlert {
|
||||
local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
|
||||
|
||||
|
||||
local mail_no_attachment=
|
||||
local attachment_command=
|
||||
local subject=
|
||||
local body=
|
||||
|
||||
# Windows specific settings
|
||||
local encryption_string=
|
||||
@ -252,7 +267,8 @@ function SendAlert {
|
||||
else
|
||||
mail_no_attachment=0
|
||||
fi
|
||||
MAIL_ALERT_MSG="$MAIL_ALERT_MSG"$'\n\n'$(tail -n 50 "$LOG_FILE")
|
||||
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
|
||||
|
||||
if [ $ERROR_ALERT -eq 1 ]; then
|
||||
subject="Error alert for $INSTANCE_ID"
|
||||
elif [ $WARN_ALERT -eq 1 ]; then
|
||||
@ -261,11 +277,17 @@ function SendAlert {
|
||||
subject="Alert for $INSTANCE_ID"
|
||||
fi
|
||||
|
||||
if [ $runAlert == true ]; then
|
||||
subject="Currently runing - $subject"
|
||||
else
|
||||
subject="Fnished run - $subject"
|
||||
fi
|
||||
|
||||
if [ "$mail_no_attachment" -eq 0 ]; then
|
||||
attachment_command="-a $ALERT_LOG_FILE"
|
||||
fi
|
||||
if type mutt > /dev/null 2>&1 ; then
|
||||
echo "$MAIL_ALERT_MSG" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
|
||||
echo "$body" | $(type -p mutt) -x -s "$subject" $DESTINATION_MAILS $attachment_command
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p mutt) !!!" "WARN"
|
||||
else
|
||||
@ -282,10 +304,10 @@ function SendAlert {
|
||||
else
|
||||
attachment_command=""
|
||||
fi
|
||||
echo "$MAIL_ALERT_MSG" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
|
||||
echo "$body" | $(type -p mail) $attachment_command -s "$subject" $DESTINATION_MAILS
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p mail) with attachments !!!" "WARN"
|
||||
echo "$MAIL_ALERT_MSG" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
|
||||
echo "$body" | $(type -p mail) -s "$subject" $DESTINATION_MAILS
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p mail) without attachments !!!" "WARN"
|
||||
else
|
||||
@ -299,7 +321,7 @@ function SendAlert {
|
||||
fi
|
||||
|
||||
if type sendmail > /dev/null 2>&1 ; then
|
||||
echo -e "Subject:$subject\r\n$MAIL_ALERT_MSG" | $(type -p sendmail) $DESTINATION_MAILS
|
||||
echo -e "Subject:$subject\r\n$body" | $(type -p sendmail) $DESTINATION_MAILS
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
|
||||
else
|
||||
@ -322,7 +344,7 @@ function SendAlert {
|
||||
if [ "$SMTP_USER" != "" ] && [ "$SMTP_USER" != "" ]; then
|
||||
auth_string="-auth -user \"$SMTP_USER\" -pass \"$SMTP_PASSWORD\""
|
||||
fi
|
||||
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$MAIL_ALERT_MSG" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
|
||||
$(type mailsend.exe) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -sub "$subject" -M "$body" -attach "$attachment" -smtp "$SMTP_SERVER" -port "$SMTP_PORT" $encryption_string $auth_string
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
|
||||
else
|
||||
@ -338,7 +360,7 @@ function SendAlert {
|
||||
else
|
||||
SMTP_OPTIONS=""
|
||||
fi
|
||||
$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$MAIL_ALERT_MSG" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
|
||||
$(type -p sendemail) -f $SENDER_MAIL -t "$DESTINATION_MAILS" -u "$subject" -m "$body" -s $SMTP_SERVER $SMTP_OPTIONS > /dev/null 2>&1
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via $(type -p sendemail) !!!" "WARN"
|
||||
else
|
||||
@ -349,7 +371,7 @@ function SendAlert {
|
||||
|
||||
# pfSense specific
|
||||
if [ -f /usr/local/bin/mail.php ]; then
|
||||
echo "$MAIL_ALERT_MSG" | /usr/local/bin/mail.php -s="$subject"
|
||||
echo "$body" | /usr/local/bin/mail.php -s="$subject"
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot send alert mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
|
||||
else
|
||||
@ -566,6 +588,7 @@ function joinString {
|
||||
# Time control function for background processes, suitable for multiple synchronous processes
|
||||
# Fills a global variable called WAIT_FOR_TASK_COMPLETION that contains list of failed pids in format pid1:result1;pid2:result2
|
||||
# Warning: Don't imbricate this function into another run if you plan to use the global variable output
|
||||
|
||||
function WaitForTaskCompletion {
|
||||
local pids="${1}" # pids to wait for, separated by semi-colon
|
||||
local soft_max_time="${2}" # If program with pid $pid takes longer than $soft_max_time seconds, will log a warning, unless $soft_max_time equals 0.
|
||||
@ -584,9 +607,13 @@ function WaitForTaskCompletion {
|
||||
local retval=0 # return value of monitored pid process
|
||||
local errorcount=0 # Number of pids that finished with errors
|
||||
|
||||
local pid # Current pid working on
|
||||
local pidCount # number of given pids
|
||||
local pidState # State of the process
|
||||
|
||||
local pidsArray # Array of currently running pids
|
||||
local newPidsArray # New array of currently running pids
|
||||
|
||||
IFS=';' read -a pidsArray <<< "$pids"
|
||||
pidCount=${#pidsArray[@]}
|
||||
|
||||
@ -615,7 +642,7 @@ function WaitForTaskCompletion {
|
||||
if [ $soft_alert -eq 0 ] && [ $soft_max_time -ne 0 ]; then
|
||||
Logger "Max soft execution time exceeded for task [$caller_name] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
|
||||
soft_alert=1
|
||||
SendAlert
|
||||
SendAlert true
|
||||
|
||||
fi
|
||||
if [ $exec_time -gt $hard_max_time ] && [ $hard_max_time -ne 0 ]; then
|
||||
@ -628,7 +655,7 @@ function WaitForTaskCompletion {
|
||||
Logger "Could not stop task with pid [$pid]." "ERROR"
|
||||
fi
|
||||
done
|
||||
SendAlert
|
||||
SendAlert true
|
||||
errrorcount=$((errorcount+1))
|
||||
fi
|
||||
fi
|
||||
@ -647,17 +674,18 @@ function WaitForTaskCompletion {
|
||||
retval=$?
|
||||
if [ $retval -ne 0 ]; then
|
||||
errorcount=$((errorcount+1))
|
||||
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$result]." "DEBUG"
|
||||
Logger "${FUNCNAME[0]} called by [$caller_name] finished monitoring [$pid] with exitcode [$retval]." "DEBUG"
|
||||
if [ "$WAIT_FOR_TASK_COMPLETION" == "" ]; then
|
||||
WAIT_FOR_TASK_COMPLETION="$pid:$result"
|
||||
WAIT_FOR_TASK_COMPLETION="$pid:$retval"
|
||||
else
|
||||
WAIT_FOR_TASK_COMPLETION=";$pid:$result"
|
||||
WAIT_FOR_TASK_COMPLETION=";$pid:$retval"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
pidsArray=("${newPidsArray[@]}")
|
||||
# Trivial wait time for bash to not eat up all CPU
|
||||
sleep $SLEEP_TIME
|
||||
done
|
||||
|
||||
@ -670,6 +698,66 @@ function WaitForTaskCompletion {
|
||||
fi
|
||||
}
|
||||
|
||||
# Take a list of commands to run, runs them sequentially with numberOfProcesses commands simultaneously runs
|
||||
# Returns the number of non zero exit codes from commands
|
||||
function ParallelExec {
|
||||
local numberOfProcesses="${1}" # Number of simultaneous commands to run
|
||||
local commandsArg="${2}" # Semi-colon separated list of commands
|
||||
|
||||
local pid
|
||||
local runningPids=0
|
||||
local counter=0
|
||||
local commandsArray
|
||||
local pidsArray
|
||||
local newPidsArray
|
||||
local retval
|
||||
local retvalAll=0
|
||||
local pidState
|
||||
local commandsArrayPid
|
||||
|
||||
IFS=';' read -r -a commandsArray <<< "$commandsArg"
|
||||
|
||||
Logger "Runnning ${#commandsArray[@]} commands in $numberOfProcesses simultaneous processes." "DEBUG"
|
||||
|
||||
while [ $counter -lt "${#commandsArray[@]}" ] || [ ${#pidsArray[@]} -gt 0 ]; do
|
||||
|
||||
while [ $counter -lt "${#commandsArray[@]}" ] && [ ${#pidsArray[@]} -lt $numberOfProcesses ]; do
|
||||
Logger "Running command [${commandsArray[$counter]}]." "DEBUG"
|
||||
eval "${commandsArray[$counter]}" &
|
||||
pid=$!
|
||||
pidsArray+=($pid)
|
||||
commandsArrayPid[$pid]="${commandsArray[$counter]}"
|
||||
counter=$((counter+1))
|
||||
done
|
||||
|
||||
|
||||
newPidsArray=()
|
||||
for pid in "${pidsArray[@]}"; do
|
||||
# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
|
||||
if kill -0 $pid > /dev/null 2>&1; then
|
||||
pidState=$(ps -p$pid -o state= 2 > /dev/null)
|
||||
if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then
|
||||
newPidsArray+=($pid)
|
||||
fi
|
||||
else
|
||||
# pid is dead, get it's exit code from wait command
|
||||
wait $pid
|
||||
retval=$?
|
||||
if [ $retval -ne 0 ]; then
|
||||
Logger "Command [${commandsArrayPid[$pid]}] failed with exit code [$retval]." "ERROR"
|
||||
retvalAll=$((retvalAll+1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
pidsArray=("${newPidsArray[@]}")
|
||||
|
||||
# Trivial wait time for bash to not eat up all CPU
|
||||
sleep $SLEEP_TIME
|
||||
done
|
||||
|
||||
return $retvalAll
|
||||
}
|
||||
|
||||
function CleanUp {
|
||||
|
||||
if [ "$_DEBUG" != "yes" ]; then
|
||||
@ -955,7 +1043,7 @@ function CheckConnectivityRemoteHost {
|
||||
|
||||
if [ "$REMOTE_HOST_PING" != "no" ] && [ "$REMOTE_OPERATION" != "no" ]; then
|
||||
eval "$PING_CMD $REMOTE_HOST > /dev/null 2>&1" &
|
||||
WaitForTaskCompletion $! 10 180 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
WaitForTaskCompletion $! 60 180 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Cannot ping $REMOTE_HOST" "ERROR"
|
||||
return 1
|
||||
@ -1227,7 +1315,6 @@ function InitLocalOSSettings {
|
||||
|
||||
function InitRemoteOSSettings {
|
||||
|
||||
#TODO: fix add -E when both initiator and targets don't run MacOSX and PRESERVE_EXECUTABILITY=yes
|
||||
## MacOSX does not use the -E parameter like Linux or BSD does (-E is mapped to extended attrs instead of preserve executability)
|
||||
if [ "$PRESERVE_EXECUTABILITY" != "no" ];then
|
||||
if [ "$LOCAL_OS" != "MacOSX" ] && [ "$REMOTE_OS" != "MacOSX" ]; then
|
||||
@ -1339,6 +1426,23 @@ function CheckEnvironment {
|
||||
|
||||
function CheckCurrentConfig {
|
||||
|
||||
# Check all variables that should contain "yes" or "no"
|
||||
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
|
||||
for i in "${yes_no_vars[@]}"; do
|
||||
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
|
||||
# Check all variables that should contain a numerical value >= 0
|
||||
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
|
||||
for i in "${num_vars[@]}"; do
|
||||
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
}
|
||||
|
||||
function CheckCurrentConfigAll {
|
||||
|
||||
if [ "$INSTANCE_ID" == "" ]; then
|
||||
Logger "No INSTANCE_ID defined in config file." "CRITICAL"
|
||||
exit 1
|
||||
@ -1354,21 +1458,11 @@ function CheckCurrentConfig {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check all variables that should contain "yes" or "no"
|
||||
declare -a yes_no_vars=(CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
|
||||
for i in "${yes_no_vars[@]}"; do
|
||||
test="if [ \"\$$i\" != \"yes\" ] && [ \"\$$i\" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
|
||||
# Check all variables that should contain a numerical value >= 0
|
||||
declare -a num_vars=(MINIMUM_SPACE BANDWIDTH SOFT_MAX_EXEC_TIME HARD_MAX_EXEC_TIME KEEP_LOGGING MIN_WAIT MAX_WAIT CONFLICT_BACKUP_DAYS SOFT_DELETE_DAYS RESUME_TRY MAX_EXEC_TIME_PER_CMD_BEFORE MAX_EXEC_TIME_PER_CMD_AFTER)
|
||||
for i in "${num_vars[@]}"; do
|
||||
test="if [ $(IsNumeric \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi"
|
||||
eval "$test"
|
||||
done
|
||||
|
||||
#TODO(low): Add runtime variable tests (RSYNC_ARGS etc)
|
||||
if [ "$REMOTE_OPERATION" == "yes" ] && [ ! -f "$SSH_RSA_PRIVATE_KEY" ]; then
|
||||
Logger "Cannot find rsa private key [$SSH_RSA_PRIVATE_KEY]. Cannot connect to remote system." "CRITICAL"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
###### Osync specific functions (non shared)
|
||||
@ -1671,20 +1765,22 @@ function CheckLocks {
|
||||
|
||||
function _WriteLockFilesLocal {
|
||||
local lockfile="${1}"
|
||||
local replica_type="${2}"
|
||||
|
||||
$COMMAND_SUDO echo "$SCRIPT_PID@$INSTANCE_ID" > "$lockfile"
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Could not create lock file [$lockfile]." "CRITICAL"
|
||||
Logger "Could not create lock file on local $replica_type in [$lockfile]." "CRITICAL"
|
||||
exit 1
|
||||
else
|
||||
Logger "Locked replica on [$lockfile]." "DEBUG"
|
||||
Logger "Locked local $replica_type replica in [$lockfile]." "DEBUG"
|
||||
fi
|
||||
}
|
||||
|
||||
function _WriteLockFilesRemote {
|
||||
local lockfile="${1}"
|
||||
local replica_type="${2}"
|
||||
|
||||
local cmd=
|
||||
local cmd
|
||||
|
||||
CheckConnectivity3rdPartyHosts
|
||||
CheckConnectivityRemoteHost
|
||||
@ -1693,32 +1789,48 @@ function _WriteLockFilesRemote {
|
||||
Logger "cmd: $cmd" "DEBUG"
|
||||
eval "$cmd"
|
||||
if [ $? != 0 ]; then
|
||||
Logger "Could not set lock on remote $replica_type replica." "CRITICAL"
|
||||
Logger "Could not create lock file on remote $replica_type in [$lockfile]." "CRITICAL"
|
||||
exit 1
|
||||
else
|
||||
Logger "Locked remote $replica_type replica." "DEBUG"
|
||||
Logger "Locked remote $replica_type replica in [$lockfile]." "DEBUG"
|
||||
fi
|
||||
}
|
||||
|
||||
function WriteLockFiles {
|
||||
|
||||
local pids
|
||||
local initiatorPid
|
||||
local targetPid
|
||||
local pidArray
|
||||
local pid
|
||||
|
||||
_WriteLockFilesLocal "${INITIATOR[2]}" "${INITIATOR[0]}"&
|
||||
initiatorPid="$!"
|
||||
|
||||
_WriteLockFilesLocal "${INITIATOR[2]}" &
|
||||
pids="$!"
|
||||
if [ "$REMOTE_OPERATION" != "yes" ]; then
|
||||
_WriteLockFilesLocal "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
_WriteLockFilesLocal "${TARGET[2]}" "${TARGET[0]}" &
|
||||
targetPid="$!"
|
||||
else
|
||||
_WriteLockFilesRemote "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
_WriteLockFilesRemote "${TARGET[2]}" "${TARGET[0]}" &
|
||||
targetPid="$!"
|
||||
fi
|
||||
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
|
||||
INITIATOR_LOCK_FILE_EXISTS=true
|
||||
TARGET_LOCK_FILE_EXISTS=true
|
||||
WaitForTaskCompletion "$initiatorPid;$targetPid" 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
if [ $? -ne 0 ]; then
|
||||
IFS=';' read -r -a pidArray <<< "$WAIT_FOR_TASK_COMPLETION"
|
||||
for pid in "${pidArray[@]}"; do
|
||||
pid=${pid%:*}
|
||||
if [ $pid == $initiatorPid ]; then
|
||||
INITIATOR_LOCK_FILE_EXISTS=false
|
||||
elif [ $pid == $targetPid ]; then
|
||||
TARGET_LOCK_FILE_EXISTS=false
|
||||
fi
|
||||
done
|
||||
|
||||
Logger "Cancelling task." "CRITICAL"
|
||||
exit 1
|
||||
fi
|
||||
LOCK_FILES_EXIST=1
|
||||
}
|
||||
|
||||
function _UnlockReplicasLocal {
|
||||
@ -1757,21 +1869,29 @@ function UnlockReplicas {
|
||||
|
||||
local pids
|
||||
|
||||
if [ $_NOLOCKS -eq 1 ] || [ $LOCK_FILES_EXIST -eq 0 ]; then
|
||||
if [ $_NOLOCKS -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
_UnlockReplicasLocal "${INITIATOR[2]}" &
|
||||
pids="$!"
|
||||
|
||||
if [ "$REMOTE_OPERATION" != "yes" ]; then
|
||||
_UnlockReplicasLocal "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
else
|
||||
_UnlockReplicasRemote "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
if [ $INITIATOR_LOCK_FILE_EXISTS == true ]; then
|
||||
_UnlockReplicasLocal "${INITIATOR[2]}" &
|
||||
pids="$!"
|
||||
fi
|
||||
|
||||
#WIP check if WaitForTaskCompletion can handle emppty pids like ";pid"
|
||||
if [ $TARGET_LOCK_FILE_EXISTS == true ]; then
|
||||
if [ "$REMOTE_OPERATION" != "yes" ]; then
|
||||
_UnlockReplicasLocal "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
else
|
||||
_UnlockReplicasRemote "${TARGET[2]}" &
|
||||
pids="$pids;$!"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$pids" != "" ]; then
|
||||
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
fi
|
||||
WaitForTaskCompletion $pids 720 1800 ${FUNCNAME[0]} true $KEEP_LOGGING
|
||||
}
|
||||
|
||||
###### Sync core functions
|
||||
@ -1790,11 +1910,6 @@ function tree_list {
|
||||
local rsync_cmd
|
||||
|
||||
|
||||
#WIP
|
||||
#if [ "$replica_type" == "${INITIATOR[0]}" ]; then
|
||||
# exit 12
|
||||
#fi
|
||||
|
||||
escaped_replica_path=$(EscapeSpaces "$replica_path")
|
||||
|
||||
Logger "Creating $replica_type replica file list [$replica_path]." "NOTICE"
|
||||
@ -1907,7 +2022,6 @@ function sync_attrs {
|
||||
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
||||
eval "$rsync_cmd"
|
||||
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
|
||||
#WIP: return retval from process instead of err count if only one pid is tested
|
||||
retval=$?
|
||||
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
||||
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
||||
@ -1988,7 +2102,6 @@ function sync_attrs {
|
||||
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
||||
eval "$rsync_cmd"
|
||||
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
|
||||
#WIP: the same
|
||||
retval=$?
|
||||
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
||||
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
|
||||
@ -2989,7 +3102,9 @@ if [ "$CONFLICT_PREVALANCE" == "" ]; then
|
||||
CONFLICT_PREVALANCE=initiator
|
||||
fi
|
||||
|
||||
LOCK_FILES_EXIST=0
|
||||
INITIATOR_LOCK_FILE_EXISTS=false
|
||||
TARGET_LOCK_FILE_EXISTS=false
|
||||
|
||||
FORCE_UNLOCK=0
|
||||
no_maxtime=0
|
||||
opts=""
|
||||
@ -3147,6 +3262,7 @@ opts="${opts# *}"
|
||||
if [ $_QUICK_SYNC -lt 2 ]; then
|
||||
CheckCurrentConfig
|
||||
fi
|
||||
CheckCurrentConfigAll
|
||||
|
||||
DATE=$(date)
|
||||
Logger "-------------------------------------------------------------" "NOTICE"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user