|
|
|
|
@ -1,12 +1,14 @@
|
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
|
|
|
|
#TODO(critical): handle conflict prevalance, especially in sync_attrs function
|
|
|
|
|
#TODO(high): verbose mode doesn't show files to be softdeleted
|
|
|
|
|
#TODO(medium): No remote deletion dir when del dir is present
|
|
|
|
|
|
|
|
|
|
PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
|
|
|
|
|
AUTHOR="(C) 2013-2016 by Orsiris de Jong"
|
|
|
|
|
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
|
|
|
|
|
PROGRAM_VERSION=1.2-dev-parallel
|
|
|
|
|
PROGRAM_BUILD=2016082803
|
|
|
|
|
PROGRAM_BUILD=2016082901
|
|
|
|
|
IS_STABLE=no
|
|
|
|
|
|
|
|
|
|
# Execution order
|
|
|
|
|
@ -47,7 +49,7 @@ IS_STABLE=no
|
|
|
|
|
|
|
|
|
|
#### MINIMAL-FUNCTION-SET BEGIN ####
|
|
|
|
|
|
|
|
|
|
## FUNC_BUILD=2016082802
|
|
|
|
|
## FUNC_BUILD=2016082901
|
|
|
|
|
## BEGIN Generic bash functions written in 2013-2016 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
|
|
|
|
|
|
|
|
|
|
## To use in a program, define the following variables:
|
|
|
|
|
@ -67,19 +69,20 @@ export LC_ALL=C
|
|
|
|
|
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
|
|
|
|
|
|
|
|
|
|
# Environment variables that can be overriden by programs
|
|
|
|
|
_DRYRUN=0
|
|
|
|
|
_SILENT=0
|
|
|
|
|
_DRYRUN=false
|
|
|
|
|
_SILENT=false
|
|
|
|
|
_VERBOSE=false
|
|
|
|
|
_LOGGER_PREFIX="date"
|
|
|
|
|
_LOGGER_STDERR=0
|
|
|
|
|
_LOGGER_STDERR=false
|
|
|
|
|
if [ "$KEEP_LOGGING" == "" ]; then
|
|
|
|
|
KEEP_LOGGING=1801
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
|
|
|
|
|
ERROR_ALERT=0
|
|
|
|
|
WARN_ALERT=0
|
|
|
|
|
ERROR_ALERT=false
|
|
|
|
|
WARN_ALERT=false
|
|
|
|
|
|
|
|
|
|
# Current log
|
|
|
|
|
# Log from current run
|
|
|
|
|
CURRENT_LOG=
|
|
|
|
|
|
|
|
|
|
## allow function call checks #__WITH_PARANOIA_DEBUG
|
|
|
|
|
@ -91,11 +94,11 @@ fi #__WITH_PARANOIA_DEBUG
|
|
|
|
|
if [ ! "$_DEBUG" == "yes" ]; then
|
|
|
|
|
_DEBUG=no
|
|
|
|
|
SLEEP_TIME=.05 # Tested under linux and FreeBSD bash, #TODO tests on cygwin / msys
|
|
|
|
|
_VERBOSE=0
|
|
|
|
|
_VERBOSE=false
|
|
|
|
|
else
|
|
|
|
|
SLEEP_TIME=1
|
|
|
|
|
trap 'TrapError ${LINENO} $?' ERR
|
|
|
|
|
_VERBOSE=1
|
|
|
|
|
_VERBOSE=true
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
SCRIPT_PID=$$
|
|
|
|
|
@ -149,17 +152,21 @@ function _Logger {
|
|
|
|
|
echo -e "$lvalue" >> "$LOG_FILE"
|
|
|
|
|
CURRENT_LOG="$CURRENT_LOG"$'\n'"$lvalue"
|
|
|
|
|
|
|
|
|
|
if [ "$_LOGGER_STDERR" -eq 1 ]; then
|
|
|
|
|
if [ $_LOGGER_STDERR == true ]; then
|
|
|
|
|
cat <<< "$evalue" 1>&2
|
|
|
|
|
elif [ "$_SILENT" -eq 0 ]; then
|
|
|
|
|
elif [ "$_SILENT" == false ]; then
|
|
|
|
|
echo -e "$svalue"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# General log function with log levels
|
|
|
|
|
# General log function with log levels:
|
|
|
|
|
# CRITICAL, ERROR, WARN are colored in stdout, prefixed in stderr
|
|
|
|
|
# NOTICE is standard level
|
|
|
|
|
# VERBOSE is only sent to stdout / stderr if _VERBOSE=true
|
|
|
|
|
# DEBUG & PARANOIA_DEBUG are only sent if _DEBUG=yes
|
|
|
|
|
function Logger {
|
|
|
|
|
local value="${1}" # Sentence to log (in double quotes)
|
|
|
|
|
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, NOTICE, WARN, ERROR, CRITIAL
|
|
|
|
|
local level="${2}" # Log level: PARANOIA_DEBUG, DEBUG, VERBOSE, NOTICE, WARN, ERROR, CRITIAL
|
|
|
|
|
|
|
|
|
|
if [ "$_LOGGER_PREFIX" == "time" ]; then
|
|
|
|
|
prefix="TIME: $SECONDS - "
|
|
|
|
|
@ -171,19 +178,22 @@ function Logger {
|
|
|
|
|
|
|
|
|
|
if [ "$level" == "CRITICAL" ]; then
|
|
|
|
|
_Logger "$prefix\e[41m$value\e[0m" "$prefix$level:$value" "$level:$value"
|
|
|
|
|
ERROR_ALERT=1
|
|
|
|
|
ERROR_ALERT=true
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "ERROR" ]; then
|
|
|
|
|
_Logger "$prefix\e[91m$value\e[0m" "$prefix$level:$value" "$level:$value"
|
|
|
|
|
ERROR_ALERT=1
|
|
|
|
|
ERROR_ALERT=true
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "WARN" ]; then
|
|
|
|
|
_Logger "$prefix\e[93m$value\e[0m" "$prefix$level:$value" "$level:$value"
|
|
|
|
|
WARN_ALERT=1
|
|
|
|
|
WARN_ALERT=true
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "NOTICE" ]; then
|
|
|
|
|
_Logger "$prefix$value"
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "VERBOSE" ] && [ $_VERBOSE == true ]; then
|
|
|
|
|
_Logger "$prefix$value"
|
|
|
|
|
return
|
|
|
|
|
elif [ "$level" == "DEBUG" ]; then
|
|
|
|
|
if [ "$_DEBUG" == "yes" ]; then
|
|
|
|
|
_Logger "$prefix$value"
|
|
|
|
|
@ -220,7 +230,7 @@ function QuickLogger {
|
|
|
|
|
|
|
|
|
|
__CheckArguments 1 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
|
|
|
|
|
|
|
|
|
if [ "$_SILENT" -eq 1 ]; then
|
|
|
|
|
if [ $_SILENT == true ]; then
|
|
|
|
|
_QuickLogger "$value" "log"
|
|
|
|
|
else
|
|
|
|
|
_QuickLogger "$value" "stdout"
|
|
|
|
|
@ -317,9 +327,9 @@ function SendAlert {
|
|
|
|
|
fi
|
|
|
|
|
body="$MAIL_ALERT_MSG"$'\n\n'"$CURRENT_LOG"
|
|
|
|
|
|
|
|
|
|
if [ $ERROR_ALERT -eq 1 ]; then
|
|
|
|
|
if [ $ERROR_ALERT == true ]; then
|
|
|
|
|
subject="Error alert for $INSTANCE_ID"
|
|
|
|
|
elif [ $WARN_ALERT -eq 1 ]; then
|
|
|
|
|
elif [ $WARN_ALERT == true ]; then
|
|
|
|
|
subject="Warning alert for $INSTANCE_ID"
|
|
|
|
|
else
|
|
|
|
|
subject="Alert for $INSTANCE_ID"
|
|
|
|
|
@ -571,7 +581,7 @@ function TrapError {
|
|
|
|
|
local job="$0"
|
|
|
|
|
local line="$1"
|
|
|
|
|
local code="${2:-1}"
|
|
|
|
|
if [ $_SILENT -eq 0 ]; then
|
|
|
|
|
if [ $_SILENT == false ]; then
|
|
|
|
|
echo -e " /!\ ERROR in ${job}: Near line ${line}, exit code ${code}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
@ -597,7 +607,7 @@ function LoadConfigFile {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Spinner {
|
|
|
|
|
if [ $_SILENT -eq 1 ]; then
|
|
|
|
|
if [ $_SILENT == true ]; then
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
@ -998,7 +1008,7 @@ function RunLocalCommand {
|
|
|
|
|
local hard_max_time="${2}" # Max time to wait for command to compleet
|
|
|
|
|
__CheckArguments 2 $# ${FUNCNAME[0]} "$@" #__WITH_PARANOIA_DEBUG
|
|
|
|
|
|
|
|
|
|
if [ $_DRYRUN -ne 0 ]; then
|
|
|
|
|
if [ $_DRYRUN == true ]; then
|
|
|
|
|
Logger "Dryrun: Local command [$command] not run." "NOTICE"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
@ -1013,7 +1023,7 @@ function RunLocalCommand {
|
|
|
|
|
Logger "Command failed." "ERROR"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ]; then
|
|
|
|
|
if [ $_VERBOSE == true ] || [ $retval -ne 0 ]; then
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
@ -1031,7 +1041,7 @@ function RunRemoteCommand {
|
|
|
|
|
|
|
|
|
|
CheckConnectivity3rdPartyHosts
|
|
|
|
|
CheckConnectivityRemoteHost
|
|
|
|
|
if [ $_DRYRUN -ne 0 ]; then
|
|
|
|
|
if [ $_DRYRUN == true ]; then
|
|
|
|
|
Logger "Dryrun: Local command [$command] not run." "NOTICE"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
@ -1048,7 +1058,7 @@ function RunRemoteCommand {
|
|
|
|
|
Logger "Command failed." "ERROR"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE -eq 1 ] || [ $retval -ne 0 ])
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ] && ([ $_VERBOSE == true ] || [ $retval -ne 0 ])
|
|
|
|
|
then
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
@ -1122,7 +1132,7 @@ function CheckConnectivity3rdPartyHosts {
|
|
|
|
|
if [ "$_PARANOIA_DEBUG" != "yes" ]; then # Do not loose time in paranoia debug
|
|
|
|
|
|
|
|
|
|
if [ "$REMOTE_3RD_PARTY_HOSTS" != "" ]; then
|
|
|
|
|
remote_3rd_party_success=0
|
|
|
|
|
remote_3rd_party_success=false
|
|
|
|
|
for i in $REMOTE_3RD_PARTY_HOSTS
|
|
|
|
|
do
|
|
|
|
|
eval "$PING_CMD $i > /dev/null 2>&1" &
|
|
|
|
|
@ -1130,11 +1140,11 @@ function CheckConnectivity3rdPartyHosts {
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot ping 3rd party host $i" "NOTICE"
|
|
|
|
|
else
|
|
|
|
|
remote_3rd_party_success=1
|
|
|
|
|
remote_3rd_party_success=true
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
if [ $remote_3rd_party_success -ne 1 ]; then
|
|
|
|
|
if [ $remote_3rd_party_success == false ]; then
|
|
|
|
|
Logger "No remote 3rd party host responded to ping. No internet ?" "ERROR"
|
|
|
|
|
return 1
|
|
|
|
|
else
|
|
|
|
|
@ -1165,14 +1175,14 @@ function __CheckArguments {
|
|
|
|
|
# In order to avoid this, we need to iterate over ${4} and count
|
|
|
|
|
|
|
|
|
|
local iterate=4
|
|
|
|
|
local fetchArguments=1
|
|
|
|
|
local fetchArguments=true
|
|
|
|
|
local argList=""
|
|
|
|
|
local countedArguments
|
|
|
|
|
while [ $fetchArguments -eq 1 ]; do
|
|
|
|
|
while [ $fetchArguments == true ]; do
|
|
|
|
|
cmd='argument=${'$iterate'}'
|
|
|
|
|
eval $cmd
|
|
|
|
|
if [ "$argument" = "" ]; then
|
|
|
|
|
fetchArguments=0
|
|
|
|
|
fetchArguments=false
|
|
|
|
|
else
|
|
|
|
|
argList="$arg_list [Argument $(($iterate-3)): $argument]"
|
|
|
|
|
iterate=$(($iterate+1))
|
|
|
|
|
@ -1314,7 +1324,7 @@ function PreInit {
|
|
|
|
|
|
|
|
|
|
## Set rsync default arguments
|
|
|
|
|
RSYNC_ARGS="-rltD"
|
|
|
|
|
if [ "$_DRYRUN" -eq 1 ]; then
|
|
|
|
|
if [ "$_DRYRUN" == true ]; then
|
|
|
|
|
RSYNC_DRY_ARG="-n"
|
|
|
|
|
DRY_WARNING="/!\ DRY RUN"
|
|
|
|
|
else
|
|
|
|
|
@ -1486,7 +1496,7 @@ function TrapStop {
|
|
|
|
|
function TrapQuit {
|
|
|
|
|
local exitcode
|
|
|
|
|
|
|
|
|
|
if [ $ERROR_ALERT -ne 0 ]; then
|
|
|
|
|
if [ $ERROR_ALERT == true ]; then
|
|
|
|
|
UnlockReplicas
|
|
|
|
|
if [ "$_DEBUG" != "yes" ]
|
|
|
|
|
then
|
|
|
|
|
@ -1500,7 +1510,7 @@ function TrapQuit {
|
|
|
|
|
CleanUp
|
|
|
|
|
Logger "$PROGRAM finished with errors." "ERROR"
|
|
|
|
|
exitcode=1
|
|
|
|
|
elif [ $WARN_ALERT -ne 0 ]; then
|
|
|
|
|
elif [ $WARN_ALERT == true ]; then
|
|
|
|
|
UnlockReplicas
|
|
|
|
|
if [ "$_DEBUG" != "yes" ]
|
|
|
|
|
then
|
|
|
|
|
@ -1867,12 +1877,12 @@ function CheckLocks {
|
|
|
|
|
|
|
|
|
|
local pids
|
|
|
|
|
|
|
|
|
|
if [ $_NOLOCKS -eq 1 ]; then
|
|
|
|
|
if [ $_NOLOCKS == true ]; then
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Do not bother checking for locks when FORCE_UNLOCK is set
|
|
|
|
|
if [ $FORCE_UNLOCK -eq 1 ]; then
|
|
|
|
|
if [ $FORCE_UNLOCK == true ]; then
|
|
|
|
|
WriteLockFiles
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
exit 1
|
|
|
|
|
@ -2008,7 +2018,7 @@ function UnlockReplicas {
|
|
|
|
|
|
|
|
|
|
local pids
|
|
|
|
|
|
|
|
|
|
if [ $_NOLOCKS -eq 1 ]; then
|
|
|
|
|
if [ $_NOLOCKS == true ]; then
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
@ -2121,6 +2131,14 @@ function _get_file_ctime_mtime_local {
|
|
|
|
|
|
|
|
|
|
echo -n "" > "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"
|
|
|
|
|
while read -r file; do $STAT_CTIME_MTIME_CMD "$replica_path$file" | sort >> "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID"; done < "$file_list"
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL"
|
|
|
|
|
if [ $_VERBOSE == true ] && [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function _get_file_ctime_mtime_remote {
|
|
|
|
|
@ -2136,7 +2154,7 @@ function _get_file_ctime_mtime_remote {
|
|
|
|
|
eval "$cmd"
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Getting file attributes failed [$retval] on $replica_type. Stopping execution." "CRITICAL"
|
|
|
|
|
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == true ] && [ -f "$RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.ctime_mtime.$replica_type.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
exit 1
|
|
|
|
|
@ -2167,13 +2185,13 @@ function sync_attrs {
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == true ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $retval != 0 ] && [ $retval != 24 ]; then
|
|
|
|
|
Logger "Getting list of files that need updates failed [$retval]. Stopping execution." "CRITICAL"
|
|
|
|
|
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == false ] && [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
exit 1 #WIP: check why exit $retval was used
|
|
|
|
|
@ -2247,13 +2265,13 @@ function sync_attrs {
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${FUNCNAME[0]} false $KEEP_LOGGING
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == true ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $retval != 0 ] && [ $retval != 24 ]; then
|
|
|
|
|
Logger "Updating file attributes on $dest_replica [$retval]. Stopping execution." "CRITICAL"
|
|
|
|
|
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == false ] && [ -f "$RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.attr-update.$dest_replica.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
exit 1 #WIP check why exit $retval was used
|
|
|
|
|
@ -2301,13 +2319,13 @@ function sync_update {
|
|
|
|
|
Logger "RSYNC_CMD: $rsync_cmd" "DEBUG"
|
|
|
|
|
eval "$rsync_cmd"
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $_VERBOSE -eq 1 ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == true ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "List:\n$(cat $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $retval != 0 ] && [ $retval != 24 ]; then
|
|
|
|
|
Logger "Updating $destination_replica replica failed. Stopping execution." "CRITICAL"
|
|
|
|
|
if [ $_VERBOSE -eq 0 ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
if [ $_VERBOSE == false ] && [ -f "$RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID" ]; then
|
|
|
|
|
Logger "Rsync output:\n$(cat $RUN_DIR/$PROGRAM.update.$destination_replica.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
exit 1 #WIP exit $retval
|
|
|
|
|
@ -2328,21 +2346,24 @@ function _delete_local {
|
|
|
|
|
local parentdir
|
|
|
|
|
local previous_file=""
|
|
|
|
|
|
|
|
|
|
if [ -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ ! -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN == false ]; then
|
|
|
|
|
$COMMAND_SUDO mkdir -p "$replica_dir$deletion_dir"
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot create local replica deletion directory in [$replica_dir$deletion_dir]." "ERROR"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
while read -r files; do
|
|
|
|
|
## On every run, check wheter the next item is already deleted because it is included in a directory already deleted
|
|
|
|
|
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
|
|
|
|
|
if [ $_VERBOSE -eq 1 ]; then
|
|
|
|
|
Logger "Soft deleting $replica_dir$files" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$SOFT_DELETE" != "no" ]; then
|
|
|
|
|
if [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == false ]; then
|
|
|
|
|
if [ -e "$replica_dir$deletion_dir/$files" ]; then
|
|
|
|
|
rm -rf "${replica_dir:?}$deletion_dir/$files"
|
|
|
|
|
Logger "Deleting file [$replica_dir$files]." "VERBOSE"
|
|
|
|
|
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ -e "$replica_dir$files" ]; then
|
|
|
|
|
@ -2350,18 +2371,20 @@ function _delete_local {
|
|
|
|
|
parentdir="$(dirname "$files")"
|
|
|
|
|
if [ "$parentdir" != "." ]; then
|
|
|
|
|
mkdir -p "$replica_dir$deletion_dir/$parentdir"
|
|
|
|
|
Logger "Moving deleted file [$replica_dir$files] to [$replica_dir$deletion_dir/$parentdir]." "VERBOSE"
|
|
|
|
|
mv -f "$replica_dir$files" "$replica_dir$deletion_dir/$parentdir"
|
|
|
|
|
else
|
|
|
|
|
Logger "Moving deleted file [$replica_dir$files] to [$replica_dir$deletion_dir]." "VERBOSE"
|
|
|
|
|
mv -f "$replica_dir$files" "$replica_dir$deletion_dir"
|
|
|
|
|
fi
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot move $replica_dir$files to deletion directory." "ERROR"
|
|
|
|
|
Logger "Cannot move [$replica_dir$files] to deletion directory." "ERROR"
|
|
|
|
|
echo "$files" >> "${INITIATOR[1]}${INITIATOR[3]}/$deleted_failed_list_file"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
if [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == false ]; then
|
|
|
|
|
if [ -e "$replica_dir$files" ]; then
|
|
|
|
|
rm -rf "$replica_dir$files"
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
@ -2410,7 +2433,7 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
|
|
|
|
|
local value="${1}" # What to log
|
|
|
|
|
echo -e "$value" >&2 #TODO(high): logfile output missing
|
|
|
|
|
|
|
|
|
|
if [ $_SILENT -eq 0 ]; then
|
|
|
|
|
if [ $_SILENT == false ]; then
|
|
|
|
|
echo -e "$value"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
@ -2447,21 +2470,23 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
|
|
|
|
|
parentdir=
|
|
|
|
|
previous_file=""
|
|
|
|
|
|
|
|
|
|
if [ -d "$replica_dir$deletion_dir" ] && [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
$COMMAND_SUDO mkdir -p "$replica_dir$deletion_dir"
|
|
|
|
|
if [ ! -d "$REPLICA_DIR$DELETE_DIR" ] && [ $_DRYRUN == false ]; then
|
|
|
|
|
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR"
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot create remote replica deletion directory in [$REPLICA_DIR$DELETE_DIR]." "ERROR"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
while read -r files; do
|
|
|
|
|
## On every run, check wheter the next item is already deleted because it is included in a directory already deleted
|
|
|
|
|
if [[ "$files" != "$previous_file/"* ]] && [ "$files" != "" ]; then
|
|
|
|
|
if [ $_VERBOSE -eq 1 ]; then
|
|
|
|
|
Logger "Soft deleting $REPLICA_DIR$files" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ "$SOFT_DELETE" != "no" ]; then
|
|
|
|
|
if [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == false ]; then
|
|
|
|
|
if [ -e "$REPLICA_DIR$DELETE_DIR/$files" ]; then
|
|
|
|
|
$COMMAND_SUDO rm -rf "$REPLICA_DIR$DELETE_DIR/$files"
|
|
|
|
|
Logger "Deleting file [$REPLICA_DIR$files]." "VERBOSE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ -e "$REPLICA_DIR$files" ]; then
|
|
|
|
|
@ -2470,17 +2495,19 @@ $SSH_CMD ERROR_ALERT=0 sync_on_changes=$sync_on_changes _SILENT=$_SILENT _DEBUG=
|
|
|
|
|
if [ "$parentdir" != "." ]; then
|
|
|
|
|
$COMMAND_SUDO mkdir -p "$REPLICA_DIR$DELETE_DIR/$parentdir"
|
|
|
|
|
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR/$parentdir"
|
|
|
|
|
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETE_DIR/$parentdir]." "VERBOSE"
|
|
|
|
|
else
|
|
|
|
|
$COMMAND_SUDO mv -f "$REPLICA_DIR$files" "$REPLICA_DIR$DELETE_DIR"
|
|
|
|
|
Logger "Moving deleted file [$REPLICA_DIR$files] to [$REPLICA_DIR$DELETE_DIR]." "VERBOSE"
|
|
|
|
|
fi
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
Logger "Cannot move $REPLICA_DIR$files to deletion directory." "ERROR"
|
|
|
|
|
Logger "Cannot move [$REPLICA_DIR$files] to deletion directory." "ERROR"
|
|
|
|
|
echo "$files" >> "$FAILED_DELETE_LIST"
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
else
|
|
|
|
|
if [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == false ]; then
|
|
|
|
|
if [ -e "$REPLICA_DIR$files" ]; then
|
|
|
|
|
$COMMAND_SUDO rm -rf "$REPLICA_DIR$files"
|
|
|
|
|
if [ $? != 0 ]; then
|
|
|
|
|
@ -2544,7 +2571,7 @@ function deletion_propagation {
|
|
|
|
|
fi
|
|
|
|
|
retval=$?
|
|
|
|
|
if [ $retval == 0 ]; then
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE -eq 1 ]; then
|
|
|
|
|
if [ -f "$RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID" ] && [ $_VERBOSE == true ]; then
|
|
|
|
|
Logger "Remote:\n$(cat $RUN_DIR/$PROGRAM._delete_remote.$SCRIPT_PID)" "DEBUG"
|
|
|
|
|
fi
|
|
|
|
|
return $retval
|
|
|
|
|
@ -2894,13 +2921,13 @@ function _SoftDeleteLocal {
|
|
|
|
|
local retval
|
|
|
|
|
|
|
|
|
|
if [ -d "$replica_deletion_path" ]; then
|
|
|
|
|
if [ $_DRYRUN -eq 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == true ]; then
|
|
|
|
|
Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
|
|
|
|
|
else
|
|
|
|
|
Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $_VERBOSE -eq 1 ]; then
|
|
|
|
|
if [ $_VERBOSE == true ]; then
|
|
|
|
|
# Cannot launch log function from xargs, ugly hack
|
|
|
|
|
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} echo "Will delete file {}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID"
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
@ -2908,7 +2935,7 @@ function _SoftDeleteLocal {
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == false ]; then
|
|
|
|
|
$COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type f -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -f "{}" && $COMMAND_SUDO $FIND_CMD "$replica_deletion_path/" -type d -empty -ctime +$change_time -print0 | xargs -0 -I {} $COMMAND_SUDO rm -rf "{}" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID" 2>&1
|
|
|
|
|
else
|
|
|
|
|
Dummy
|
|
|
|
|
@ -2936,13 +2963,13 @@ function _SoftDeleteRemote {
|
|
|
|
|
CheckConnectivity3rdPartyHosts
|
|
|
|
|
CheckConnectivityRemoteHost
|
|
|
|
|
|
|
|
|
|
if [ $_DRYRUN -eq 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == true ]; then
|
|
|
|
|
Logger "Listing files older than $change_time days on $replica_type replica. Does not remove anything." "NOTICE"
|
|
|
|
|
else
|
|
|
|
|
Logger "Removing files older than $change_time days on $replica_type replica." "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $_VERBOSE -eq 1 ]; then
|
|
|
|
|
if [ $_VERBOSE == true ]; then
|
|
|
|
|
# Cannot launch log function from xargs, ugly hack
|
|
|
|
|
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} echo Will delete file {} && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} echo Will delete directory {}; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
@ -2950,7 +2977,7 @@ function _SoftDeleteRemote {
|
|
|
|
|
Logger "Command output:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID)" "NOTICE"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $_DRYRUN -ne 1 ]; then
|
|
|
|
|
if [ $_DRYRUN == false ]; then
|
|
|
|
|
cmd=$SSH_CMD' "if [ -d \"'$replica_deletion_path'\" ]; then '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type f -ctime +'$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -f \"{}\" && '$COMMAND_SUDO' '$REMOTE_FIND_CMD' \"'$replica_deletion_path'/\" -type d -empty -ctime '$change_time' -print0 | xargs -0 -I {} '$COMMAND_SUDO' rm -rf \"{}\"; else echo \"No remote backup/deletion directory.\"; fi" > "'$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID'" 2>&1'
|
|
|
|
|
|
|
|
|
|
Logger "cmd: $cmd" "DEBUG"
|
|
|
|
|
@ -3011,7 +3038,7 @@ function Init {
|
|
|
|
|
set -o errtrace
|
|
|
|
|
|
|
|
|
|
# Do not use exit and quit traps if osync runs in monitor mode
|
|
|
|
|
if [ $sync_on_changes -eq 0 ]; then
|
|
|
|
|
if [ $sync_on_changes == false ]; then
|
|
|
|
|
trap TrapStop INT HUP TERM QUIT
|
|
|
|
|
trap TrapQuit EXIT
|
|
|
|
|
else
|
|
|
|
|
@ -3081,7 +3108,7 @@ function Init {
|
|
|
|
|
local partial_dir="_partial"
|
|
|
|
|
local last_action="last-action"
|
|
|
|
|
local resume_count="resume-count"
|
|
|
|
|
if [ "$_DRYRUN" -eq 1 ]; then
|
|
|
|
|
if [ "$_DRYRUN" == true ]; then
|
|
|
|
|
local dry_suffix="-dry"
|
|
|
|
|
else
|
|
|
|
|
local dry_suffix=
|
|
|
|
|
@ -3125,11 +3152,11 @@ function Init {
|
|
|
|
|
## Set sync only function arguments for rsync
|
|
|
|
|
SYNC_OPTS="-u"
|
|
|
|
|
|
|
|
|
|
if [ $_VERBOSE -eq 1 ]; then
|
|
|
|
|
if [ $_VERBOSE == true ]; then
|
|
|
|
|
SYNC_OPTS=$SYNC_OPTS" -i"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [ $STATS -eq 1 ]; then
|
|
|
|
|
if [ $STATS == true ]; then
|
|
|
|
|
SYNC_OPTS=$SYNC_OPTS" --stats"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
@ -3252,8 +3279,8 @@ function SyncOnChanges {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# quicksync mode settings, overriden by config file
|
|
|
|
|
STATS=0
|
|
|
|
|
PARTIAL=0
|
|
|
|
|
STATS=false
|
|
|
|
|
PARTIAL=no
|
|
|
|
|
if [ "$CONFLICT_PREVALANCE" == "" ]; then
|
|
|
|
|
CONFLICT_PREVALANCE=initiator
|
|
|
|
|
fi
|
|
|
|
|
@ -3261,14 +3288,17 @@ fi
|
|
|
|
|
INITIATOR_LOCK_FILE_EXISTS=false
|
|
|
|
|
TARGET_LOCK_FILE_EXISTS=false
|
|
|
|
|
|
|
|
|
|
FORCE_UNLOCK=0
|
|
|
|
|
no_maxtime=0
|
|
|
|
|
FORCE_UNLOCK=false
|
|
|
|
|
no_maxtime=false
|
|
|
|
|
opts=""
|
|
|
|
|
ERROR_ALERT=0
|
|
|
|
|
ERROR_ALERT=false
|
|
|
|
|
WARN_ALERT=false
|
|
|
|
|
# Number of CTRL+C
|
|
|
|
|
SOFT_STOP=0
|
|
|
|
|
# Number of given replicas in command line
|
|
|
|
|
_QUICK_SYNC=0
|
|
|
|
|
sync_on_changes=0
|
|
|
|
|
_NOLOCKS=0
|
|
|
|
|
sync_on_changes=false
|
|
|
|
|
_NOLOCKS=false
|
|
|
|
|
osync_cmd=$0
|
|
|
|
|
|
|
|
|
|
if [ $# -eq 0 ]
|
|
|
|
|
@ -3280,7 +3310,7 @@ first=1
|
|
|
|
|
for i in "$@"; do
|
|
|
|
|
case $i in
|
|
|
|
|
--dry)
|
|
|
|
|
_DRYRUN=1
|
|
|
|
|
_DRYRUN=true
|
|
|
|
|
opts=$opts" --dry"
|
|
|
|
|
;;
|
|
|
|
|
--silent)
|
|
|
|
|
@ -3288,11 +3318,11 @@ for i in "$@"; do
|
|
|
|
|
opts=$opts" --silent"
|
|
|
|
|
;;
|
|
|
|
|
--verbose)
|
|
|
|
|
_VERBOSE=1
|
|
|
|
|
_VERBOSE=true
|
|
|
|
|
opts=$opts" --verbose"
|
|
|
|
|
;;
|
|
|
|
|
--stats)
|
|
|
|
|
STATS=1
|
|
|
|
|
STATS=true
|
|
|
|
|
opts=$opts" --stats"
|
|
|
|
|
;;
|
|
|
|
|
--partial)
|
|
|
|
|
@ -3300,11 +3330,11 @@ for i in "$@"; do
|
|
|
|
|
opts=$opts" --partial"
|
|
|
|
|
;;
|
|
|
|
|
--force-unlock)
|
|
|
|
|
FORCE_UNLOCK=1
|
|
|
|
|
FORCE_UNLOCK=true
|
|
|
|
|
opts=$opts" --force-unlock"
|
|
|
|
|
;;
|
|
|
|
|
--no-maxtime)
|
|
|
|
|
no_maxtime=1
|
|
|
|
|
no_maxtime=true
|
|
|
|
|
opts=$opts" --no-maxtime"
|
|
|
|
|
;;
|
|
|
|
|
--help|-h|--version|-v)
|
|
|
|
|
@ -3312,15 +3342,15 @@ for i in "$@"; do
|
|
|
|
|
;;
|
|
|
|
|
--initiator=*)
|
|
|
|
|
_QUICK_SYNC=$(($_QUICK_SYNC + 1))
|
|
|
|
|
no_maxtime=1
|
|
|
|
|
INITIATOR_SYNC_DIR=${i##*=}
|
|
|
|
|
opts=$opts" --initiator=\"$INITIATOR_SYNC_DIR\""
|
|
|
|
|
no_maxtime=true
|
|
|
|
|
;;
|
|
|
|
|
--target=*)
|
|
|
|
|
_QUICK_SYNC=$(($_QUICK_SYNC + 1))
|
|
|
|
|
TARGET_SYNC_DIR=${i##*=}
|
|
|
|
|
opts=$opts" --target=\"$TARGET_SYNC_DIR\""
|
|
|
|
|
no_maxtime=1
|
|
|
|
|
no_maxtime=true
|
|
|
|
|
;;
|
|
|
|
|
--rsakey=*)
|
|
|
|
|
SSH_RSA_PRIVATE_KEY=${i##*=}
|
|
|
|
|
@ -3331,13 +3361,13 @@ for i in "$@"; do
|
|
|
|
|
opts=$opts" --instance-id=\"$INSTANCE_ID\""
|
|
|
|
|
;;
|
|
|
|
|
--on-changes)
|
|
|
|
|
sync_on_changes=1
|
|
|
|
|
_NOLOCKS=1
|
|
|
|
|
sync_on_changes=true
|
|
|
|
|
_NOLOCKS=true
|
|
|
|
|
_LOGGER_PREFIX="date"
|
|
|
|
|
_LOGGER_STDERR=1
|
|
|
|
|
_LOGGER_STDERR=true
|
|
|
|
|
;;
|
|
|
|
|
--no-locks)
|
|
|
|
|
_NOLOCKS=1
|
|
|
|
|
_NOLOCKS=true
|
|
|
|
|
;;
|
|
|
|
|
*)
|
|
|
|
|
if [ $first == "0" ]; then
|
|
|
|
|
@ -3426,13 +3456,13 @@ opts="${opts# *}"
|
|
|
|
|
Logger "-------------------------------------------------------------" "NOTICE"
|
|
|
|
|
Logger "Sync task [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"
|
|
|
|
|
|
|
|
|
|
if [ $sync_on_changes -eq 1 ]; then
|
|
|
|
|
if [ $sync_on_changes == true ]; then
|
|
|
|
|
SyncOnChanges
|
|
|
|
|
else
|
|
|
|
|
GetRemoteOS
|
|
|
|
|
InitRemoteOSSettings
|
|
|
|
|
|
|
|
|
|
if [ $no_maxtime -eq 1 ]; then
|
|
|
|
|
if [ $no_maxtime == true ]; then
|
|
|
|
|
SOFT_MAX_EXEC_TIME=0
|
|
|
|
|
HARD_MAX_EXEC_TIME=0
|
|
|
|
|
fi
|
|
|
|
|
|
Shadowigor
8 years ago