@ -1,12 +1,16 @@
#!/usr/bin/env bash
#TODO(critical): handle conflict prevalance, especially in sync_attrs function
#TODO(high): test resume system
#TODO(critical): test new WaitForTaskCompletion behavior with self=true
#TODO(low): add keep logging & preserve attr conf entries to preflight check
#TODO(low): test failed _CreateStateDirsRemote should stop osync
PROGRAM = "osync" # Rsync based two way sync engine with fault tolerance
AUTHOR = "(C) 2013-2016 by Orsiris de Jong"
CONTACT = "http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION = 1.2-dev-parallel
PROGRAM_BUILD = 2016081 301
PROGRAM_BUILD = 2016081 802
IS_STABLE = no
# Function Name Is parallel #__WITH_PARANOIA_DEBUG
@ -61,7 +65,7 @@ function TrapStop {
if [ $SOFT_STOP -eq 1 ] ; then
Logger " /!\ WARNING: CTRL+C hit twice. Exiting osync. Please wait while replicas get unlocked..." "WARN"
SOFT_STOP = 2
exit 1
exit 2
fi
}
@ -95,7 +99,7 @@ function TrapQuit {
fi
CleanUp
Logger " $PROGRAM finished with warnings. " "WARN"
exitcode = 240 # Special exit code for daemon mode not stopping on warnings
exitcode = 2 # Warning exit code must not force daemon mode to quit
else
UnlockReplicas
if [ " $RUN_AFTER_CMD_ON_ERROR " = = "yes" ] ; then
@ -146,7 +150,7 @@ function CheckCurrentConfig {
fi
# Check all variables that should contain "yes" or "no"
declare -a yes_no_vars = ( CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
declare -a yes_no_vars = ( CREATE_DIRS SUDO_EXEC SSH_COMPRESSION SSH_IGNORE_KNOWN_HOSTS REMOTE_HOST_PING PRESERVE_PERMISSIONS PRESERVE_OWNER PRESERVE_GROUP PRESERVE_EXECUTABILITY PRESERVE_ ACL PRESERVE_XATTR COPY_SYMLINKS KEEP_DIRLINKS PRESERVE_HARDLINKS CHECKSUM RSYNC_COMPRESS CONFLICT_BACKUP CONFLICT_BACKUP_MULTIPLE SOFT_DELETE RESUME_SYNC FORCE_STRANGER_LOCK_RESUME PARTIAL DELTA_COPIES STOP_ON_CMD_ERROR RUN_AFTER_CMD_ON_ERROR)
for i in " ${ yes_no_vars [@] } " ; do
test = " if [ \"\$ $i \" != \"yes\" ] && [ \"\$ $i \" != \"no\" ]; then Logger \"Bogus $i value defined in config file. Correct your config file or update it using the update script if using and old version.\" \"CRITICAL\"; exit 1; fi "
eval " $test "
@ -241,7 +245,11 @@ function CheckReplicaPaths {
_CheckReplicaPathsRemote " ${ TARGET [1] } " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? -ne 0 ] ; then
Logger "Cancelling task." "CRITICAL"
exit 1
fi
}
function _CheckDiskSpaceLocal {
@ -303,7 +311,7 @@ function CheckDiskSpace {
_CheckDiskSpaceRemote " ${ TARGET [1] } " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } false true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
}
@ -354,7 +362,11 @@ function CreateStateDirs {
_CreateStateDirsRemote " ${ TARGET [1] } ${ TARGET [3] } " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
if [ $? -ne 0 ] ; then
Logger "Cancelling task." "CRITICAL"
exit 1
fi
}
function _CheckLocksLocal {
@ -455,7 +467,11 @@ function CheckLocks {
_CheckLocksRemote " ${ TARGET [2] } " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
if [ $? -ne 0 ] ; then
Logger "Cancelling task." "CRITICAL"
exit 1
fi
WriteLockFiles
}
@ -506,7 +522,11 @@ function WriteLockFiles {
_WriteLockFilesRemote " ${ TARGET [2] } " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGINg
if [ $? -ne 0 ] ; then
Logger "Cancelling task." "CRITICAL"
exit 1
fi
LOCK_FILES_EXIST = 1
}
@ -563,7 +583,7 @@ function UnlockReplicas {
_UnlockReplicasRemote " ${ TARGET [2] } " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } false true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
}
###### Sync core functions
@ -698,7 +718,8 @@ function sync_attrs {
fi
Logger " RSYNC_CMD: $rsync_cmd " "DEBUG"
eval " $rsync_cmd "
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
#WIP: return retval from process instead of err count if only one pid is tested
retval = $?
if [ $_VERBOSE -eq 1 ] && [ -f " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $SCRIPT_PID " ] ; then
Logger " List:\n $( cat $RUN_DIR /$PROGRAM .${ FUNCNAME [0] } .$SCRIPT_PID ) " "NOTICE"
@ -709,7 +730,7 @@ function sync_attrs {
if [ $_VERBOSE -eq 0 ] && [ -f " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $SCRIPT_PID " ] ; then
Logger " Rsync output:\n $( cat $RUN_DIR /$PROGRAM .${ FUNCNAME [0] } .$SCRIPT_PID ) " "NOTICE"
fi
exit $retval
exit 1 #WIP: check why exit $retval was used
else
cat " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $SCRIPT_PID " | ( grep -Ev "^[^ ]*(c|s|t)[^ ]* " || :) | ( grep -E "^[^ ]*(p|o|g|a)[^ ]* " || :) | sed -e 's/^[^ ]* //' >> " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } -cleaned. $SCRIPT_PID "
if [ $? != 0 ] ; then
@ -730,7 +751,7 @@ function sync_attrs {
_get_file_ctime_mtime_remote " ${ TARGET [1] } " " ${ TARGET [0] } " " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } -cleaned. $SCRIPT_PID " &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 1800 0 ${ FUNCNAME [0] } false true
WaitForTaskCompletion $pids 1800 0 ${ FUNCNAME [0] } true $KEEP_LOGGING
# If target gets updated first, then sync_attr must update initiator's attrs first
# For join, remove leading replica paths
@ -778,7 +799,8 @@ function sync_attrs {
Logger " RSYNC_CMD: $rsync_cmd " "DEBUG"
eval " $rsync_cmd "
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
#WIP: the same
retval = $?
if [ $_VERBOSE -eq 1 ] && [ -f " $RUN_DIR / $PROGRAM .attr-update. $dest_replica . $SCRIPT_PID " ] ; then
Logger " List:\n $( cat $RUN_DIR /$PROGRAM .attr-update.$dest_replica .$SCRIPT_PID ) " "NOTICE"
@ -789,7 +811,7 @@ function sync_attrs {
if [ $_VERBOSE -eq 0 ] && [ -f " $RUN_DIR / $PROGRAM .attr-update. $dest_replica . $SCRIPT_PID " ] ; then
Logger " Rsync output:\n $( cat $RUN_DIR /$PROGRAM .attr-update.$dest_replica .$SCRIPT_PID ) " "NOTICE"
fi
exit $retval
exit 1 #WIP check why exit $retval was used
else
Logger " Successfully updated file attributes on $dest_replica replica. " "NOTICE"
fi
@ -843,7 +865,7 @@ function sync_update {
if [ $_VERBOSE -eq 0 ] && [ -f " $RUN_DIR / $PROGRAM .update. $destination_replica . $SCRIPT_PID " ] ; then
Logger " Rsync output:\n $( cat $RUN_DIR /$PROGRAM .update.$destination_replica .$SCRIPT_PID ) " "NOTICE"
fi
exit $retval
exit 1 #WIP exit $retval
else
Logger " Updating $destination_replica replica succeded. " "NOTICE"
return 0
@ -1039,7 +1061,7 @@ ENDSSH
if [ -f " $RUN_DIR / $PROGRAM .remote_failed_deletion_list_copy. $SCRIPT_PID " ] ; then
Logger " Comand output: $( cat $RUN_DIR /$PROGRAM .remote_failed_deletion_list_copy.$SCRIPT_PID ) " "NOTICE"
fi
exit $result
exit 1 #WIP exit $retval
fi
return 0
}
@ -1174,7 +1196,7 @@ function Sync {
targetPid = " $! "
fi
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? != 0 ] ; then
IFS = ';' read -r -a pidArray <<< " $WAIT_FOR_TASK_COMPLETION "
for pid in " ${ pidArray [@] } " ; do
@ -1206,7 +1228,7 @@ function Sync {
targetPid = " $! "
fi
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? != 0 ] ; then
IFS = ';' read -r -a pidArray <<< " $WAIT_FOR_TASK_COMPLETION "
for pid in " ${ pidArray [@] } " ; do
@ -1230,7 +1252,7 @@ function Sync {
if [ " $resumeInitiator " = = " ${ SYNC_ACTION [2] } " ] || [ " $resumeTarget " = = " ${ SYNC_ACTION [2] } " ] ; then
if [ " $RSYNC_ATTR_ARGS " != "" ] ; then
sync_attrs " ${ INITIATOR [1] } " " $TARGET_SYNC_DIR "
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion $! $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? != 0 ] ; then
echo " ${ SYNC_ACTION [2] } " > " ${ INITIATOR [7] } "
echo " ${ SYNC_ACTION [2] } " > " ${ INITIATOR [8] } "
@ -1262,7 +1284,7 @@ function Sync {
targetPid = " $! "
fi
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? != 0 ] ; then
IFS = ';' read -r -a pidArray <<< " $WAIT_FOR_TASK_COMPLETION "
for pid in " ${ pidArray [@] } " ; do
@ -1294,7 +1316,7 @@ function Sync {
targetPid = " $! "
fi
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? != 0 ] ; then
IFS = ';' read -r -a pidArray <<< " $WAIT_FOR_TASK_COMPLETION "
for pid in " ${ pidArray [@] } " ; do
@ -1327,7 +1349,7 @@ function Sync {
targetPid = " $! "
fi
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false false
WaitForTaskCompletion " $initiatorPid ; $targetPid " $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME ${ FUNCNAME [0] } false $KEEP_LOGGING
if [ $? != 0 ] ; then
IFS = ';' read -r -a pidArray <<< " $WAIT_FOR_TASK_COMPLETION "
for pid in " ${ pidArray [@] } " ; do
@ -1450,7 +1472,7 @@ function SoftDelete {
_SoftDeleteRemote " ${ TARGET [0] } " " ${ TARGET [1] } ${ TARGET [4] } " $CONFLICT_BACKUP_DAYS &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } false true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
fi
if [ " $SOFT_DELETE " != "no" ] && [ $SOFT_DELETE_DAYS -ne 0 ] ; then
@ -1465,7 +1487,7 @@ function SoftDelete {
_SoftDeleteRemote " ${ TARGET [0] } " " ${ TARGET [1] } ${ TARGET [5] } " $SOFT_DELETE_DAYS &
pids = " $pids ; $! "
fi
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } false true
WaitForTaskCompletion $pids 720 1800 ${ FUNCNAME [0] } true $KEEP_LOGGING
fi
}
@ -1696,7 +1718,7 @@ function SyncOnChanges {
Logger " daemon cmd: $cmd " "DEBUG"
eval " $cmd "
retval = $?
if [ $retval != 0 ] && [ $retval != 2 40 ] ; then
if [ $retval != 0 ] && [ $retval != 2 ] ; then
Logger "osync child exited with error." "ERROR"
fi
@ -1906,5 +1928,4 @@ opts="${opts# *}"
if [ $? = = 0 ] ; then
SoftDelete
fi
RunAfterHook
fi