@ -3,12 +3,13 @@
#TODO treeList, deleteList, _getFileCtimeMtime, conflictList should be called without having statedir informed. Just give the full path ?
#TODO add error handling to new functions
#TODO check if _getCtimeMtime | sort removal needs to be backported
#TODO backport treeList sed -r 's/^.* [0-9/]{10} [0-9:]{8} //' fix && _getFileCtimeMtime* IFS read fix
PROGRAM = "osync" # Rsync based two way sync engine with fault tolerance
AUTHOR = "(C) 2013-2017 by Orsiris de Jong"
CONTACT = "http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION = 1.2.2-dev
PROGRAM_BUILD = 2017060 303
PROGRAM_BUILD = 2017060 401
IS_STABLE = no
@ -691,19 +692,17 @@ function treeList {
# operation explanation
# (command || :) = Return code 0 regardless of command return code
# (grep -E \"^-|^d|^l\" || :) = Be sure line begins with '-' or 'd' or 'l' (rsync semantics for file, directory or symlink)
# (awk '{\$1=\$2=\$3=\$4=\"\" ;print}' || :) = Remove the first four columns of rsync output
# (awk '{\$1=\$1 ;print}' || :) = Removes leading spaces
# (awk '{$1=$2=$3=$4="" ;print substr(\$0,5)}' || :) = Same the two lines above, replaces them
# (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) = Only show output before ' -> ' in order to remove symlink destionations
# (sed -r 's/^.* [0-9/]{10} [0-9:]{8} //' || :) = Remove everything before timestamps
# (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) = Only show output before ' -> ' in order to remove symlink destinations
# (grep -v \"^\.$\" || :) = Removes line containing current directory sign '.'
Logger " Creating $replicaType replica file list [ $replicaPath ]. " "NOTICE"
if [ " $REMOTE_OPERATION " = = "yes" ] && [ " $replicaType " = = " ${ TARGET [ $__type ] } " ] ; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
rsyncCmd = " $( type -p $RSYNC_EXECUTABLE ) --rsync-path=\"env _REMOTE_TOKEN= $_REMOTE_TOKEN $RSYNC_PATH \" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \" $OSYNC_DIR \" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \" $RSYNC_SSH_CMD \" --list-only $REMOTE_USER @ $REMOTE_HOST :\" $escapedReplicaPath \" 2> \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType .error. $SCRIPT_PID . $TSTAMP \" | (grep -E \"^-|^d|^l\" || :) | ( awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)} ' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType . $SCRIPT_PID . $TSTAMP \" "
rsyncCmd = " $( type -p $RSYNC_EXECUTABLE ) --rsync-path=\"env _REMOTE_TOKEN= $_REMOTE_TOKEN $RSYNC_PATH \" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \" $OSYNC_DIR \" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE -e \" $RSYNC_SSH_CMD \" --list-only $REMOTE_USER @ $REMOTE_HOST :\" $escapedReplicaPath \" 2> \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType .error. $SCRIPT_PID . $TSTAMP \" | (grep -E \"^-|^d|^l\" || :) | ( sed -r 's/^.* [0-9/]{10} [0-9:]{8} // ' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType . $SCRIPT_PID . $TSTAMP \" "
else
rsyncCmd = " $( type -p $RSYNC_EXECUTABLE ) --rsync-path=\"env _REMOTE_TOKEN= $_REMOTE_TOKEN $RSYNC_PATH \" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \" $OSYNC_DIR \" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \" $replicaPath \" 2> \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType .error. $SCRIPT_PID . $TSTAMP \" | (grep -E \"^-|^d|^l\" || :) | ( awk '{\$1=\$2=\$3=\$4=\"\" ;print substr(\$0,5)} ' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType . $SCRIPT_PID . $TSTAMP \" "
rsyncCmd = " $( type -p $RSYNC_EXECUTABLE ) --rsync-path=\"env _REMOTE_TOKEN= $_REMOTE_TOKEN $RSYNC_PATH \" $RSYNC_ARGS $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS --exclude \" $OSYNC_DIR \" $RSYNC_PATTERNS $RSYNC_PARTIAL_EXCLUDE --list-only \" $replicaPath \" 2> \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType .error. $SCRIPT_PID . $TSTAMP \" | (grep -E \"^-|^d|^l\" || :) | ( sed -r 's/^.* [0-9/]{10} [0-9:]{8} // ' || :) | (awk 'BEGIN { FS=\" -> \" } ; { print \$1 }' || :) | (grep -v \"^\.$\" || :) | sort > \" $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . $replicaType . $SCRIPT_PID . $TSTAMP \" "
fi
Logger " RSYNC_CMD: $rsyncCmd " "DEBUG"
eval " $rsyncCmd "
@ -797,7 +796,7 @@ function _getFileCtimeMtimeLocal {
local retval
echo -n "" > " $RUN_DIR / $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP "
while read -r file; do $STAT_CTIME_MTIME_CMD " $replicaPath $file " >> " $RUN_DIR / $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP " ; done < " $fileList "
while IFS = '' read -r file; do $STAT_CTIME_MTIME_CMD " $replicaPath $file " >> " $RUN_DIR / $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP " ; done < " $fileList "
retval = $?
if [ $retval -ne 0 ] ; then
Logger " Getting file attributes failed [ $retval ] on $replicaType . Stopping execution. " "CRITICAL" $retval
@ -843,7 +842,7 @@ env _DEBUG="'$_DEBUG'" env _PARANOIA_DEBUG="'$_PARANOIA_DEBUG'" env _LOGGER_SILE
env PROGRAM = " ' $PROGRAM ' " env SCRIPT_PID = " ' $SCRIPT_PID ' " TSTAMP = " ' $TSTAMP ' " \
env replicaPath = " ' $replicaPath ' " env replicaType = " ' $replicaType ' " env REMOTE_STAT_CTIME_MTIME_CMD = " ' $REMOTE_STAT_CTIME_MTIME_CMD ' " \
env LANG = C $COMMAND_SUDO ' bash -s' << 'ENDSSH' > " $RUN_DIR / $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP "
while read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD " $replicaPath $file " ; done < " . $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP "
while IFS = '' read -r file; do $REMOTE_STAT_CTIME_MTIME_CMD " $replicaPath $file " ; done < " . $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP "
if [ -f " . $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP " ] ; then
rm -f " . $PROGRAM .ctime_mtime. $replicaType . $SCRIPT_PID . $TSTAMP "
fi
@ -916,8 +915,8 @@ function conflictList {
Logger "Creating conflictual file list." "NOTICE"
comm -23 " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ INITIATOR [ $__type ] } $timestampCurrentFilename " " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ INITIATOR [ $__type ] } $timestampAfterFilename " > " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . ${ INITIATOR [ $__type ] } . $SCRIPT_PID . $TSTAMP "
comm -23 " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ TARGET [ $__type ] } $timestampCurrentFilename " " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ TARGET [ $__type ] } $timestampAfterFilename " > " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . ${ TARGET [ $__type ] } . $SCRIPT_PID . $TSTAMP "
comm -23 " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ INITIATOR [ $__type ] } $timestampCurrentFilename " " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ INITIATOR [ $__type ] } $timestampAfterFilename " | sort > " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . ${ INITIATOR [ $__type ] } . $SCRIPT_PID . $TSTAMP "
comm -23 " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ TARGET [ $__type ] } $timestampCurrentFilename " " ${ INITIATOR [ $__replicaDir ] } ${ INITIATOR [ $__stateDir ] } / ${ TARGET [ $__type ] } $timestampAfterFilename " | sort > " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . ${ TARGET [ $__type ] } . $SCRIPT_PID . $TSTAMP "
join -j 1 -t ';' -o 1.1,1.2,1.3,2.2,2.3 " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . ${ INITIATOR [ $__type ] } . $SCRIPT_PID . $TSTAMP " " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } . ${ TARGET [ $__type ] } . $SCRIPT_PID . $TSTAMP " > " $RUN_DIR / $PROGRAM . ${ FUNCNAME [0] } .comapre. $SCRIPT_PID . $TSTAMP "