Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Added --syncÃ-type for unidirectional sync

Browse Source
pull/167/head
deajan 5 years ago
parent de99f0efdf
commit 71f857f9c8
2 changed files with 3247 additions and 23 deletions
Show Stats Download Patch File Download Diff File

70
dev/n_osync.sh
Unescape Escape View File

@ -7,7 +7,7 @@ PROGRAM="osync" # Rsync based two way sync engine with fault tolerance
AUTHOR="(C) 2013-2019 by Orsiris de Jong"
CONTACT="http://www.netpower.fr/osync - ozy@netpower.fr"
PROGRAM_VERSION=1.3.0-pre-rc1
PROGRAM_BUILD=2019052006
PROGRAM_BUILD=2019052007
IS_STABLE=false
CONFIG_FILE_REVISION_REQUIRED=1.3.0
@ -1251,6 +1251,8 @@ function syncUpdate {
local backupArgs
local deleteArgs
local exclude_list_initiator
local exclude_list_target
Logger "Updating $destinationReplica replica." "NOTICE"
if [ "$sourceReplica" == "${INITIATOR[$__type]}" ]; then
@ -1273,16 +1275,23 @@ function syncUpdate {
deleteArgs="--delete-after"
fi
if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}" ]; then
exclude_list_initiator="--exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\""
fi
if [ -f "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}" ]; then
exclude_list_target="--exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\""
fi
if [ "$REMOTE_OPERATION" == true ]; then
CheckConnectivity3rdPartyHosts
CheckConnectivityRemoteHost
if [ "$sourceReplica" == "${INITIATOR[$__type]}" ]; then
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs $deleteArgs --exclude \"$OSYNC_DIR\" $RSYNC_FULL_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" \"$sourceDir\" $REMOTE_USER@$REMOTE_HOST:\"$escDestDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs $deleteArgs --exclude \"$OSYNC_DIR\" $RSYNC_FULL_PATTERNS $RSYNC_PARTIAL_EXCLUDE $exclude_list_initiator $exclude_list_target \"$sourceDir\" $REMOTE_USER@$REMOTE_HOST:\"$escDestDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
else
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs $deleteArgs --exclude \"$OSYNC_DIR\" $RSYNC_FULL_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" $REMOTE_USER@$REMOTE_HOST:\"$escSourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS -e \"$RSYNC_SSH_CMD\" $backupArgs $deleteArgs --exclude \"$OSYNC_DIR\" $RSYNC_FULL_PATTERNS $RSYNC_PARTIAL_EXCLUDE $exclude_list_initiator $exclude_list_target $REMOTE_USER@$REMOTE_HOST:\"$escSourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
fi
else
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS $backupArgs $deleteArgs --exclude \"$OSYNC_DIR\" $RSYNC_FULL_PATTERNS $RSYNC_PARTIAL_EXCLUDE --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$sourceReplica${INITIATOR[$__deletedListFile]}\" --exclude-from=\"${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/$destinationReplica${INITIATOR[$__deletedListFile]}\" \"$sourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
rsyncCmd="$(type -p $RSYNC_EXECUTABLE) --rsync-path=\"env LC_ALL=C env _REMOTE_TOKEN=$_REMOTE_TOKEN $RSYNC_PATH\" $RSYNC_DEFAULT_ARGS $RSYNC_DRY_ARG $RSYNC_ATTR_ARGS $RSYNC_TYPE_ARGS $SYNC_OPTS $backupArgs $deleteArgs --exclude \"$OSYNC_DIR\" $RSYNC_FULL_PATTERNS $RSYNC_PARTIAL_EXCLUDE $exclude_list_initiator $exclude_list_target \"$sourceDir\" \"$destDir\" >> $RUN_DIR/$PROGRAM.update.$destinationReplica.$SCRIPT_PID.$TSTAMP 2>&1"
fi
Logger "RSYNC_CMD: $rsyncCmd" "DEBUG"
eval "$rsyncCmd"
@ -1677,6 +1686,7 @@ function Sync {
local resumeCount
local resumeInitiator
local resumeTarget
local rsyncRemoteDelete=false
local initiatorPid
local targetPid
@ -1738,9 +1748,11 @@ function Sync {
if [ "$SYNC_TYPE" == "initiator2target" ]; then
resumeInitiator="${SYNC_ACTION[5]}"
resumeTarget="${SYNC_ACTION[6]}"
rsyncRemoteDelete=true
elif [ "$SYNC_TYPE" == "target2initiator" ]; then
resumeInitiator="${SYNC_ACTION[6]}"
resumeTarget="${SYNC_ACTION[5]}"
rsyncRemoteDelete=true
fi
################################################################################################################################################# Actual sync begins here
@ -1942,7 +1954,7 @@ function Sync {
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ] || [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
if [ "$CONFLICT_PREVALANCE" == "${TARGET[$__type]}" ]; then
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" $rsyncRemoteDelete &
ExecTasks $! "${FUNCNAME[0]}_syncUpdate_initiator" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -1954,7 +1966,7 @@ function Sync {
fi
fi
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" $rsyncRemoteDelete &
ExecTasks $! "${FUNCNAME[0]}_syncUpdate_target" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
@ -1967,7 +1979,7 @@ function Sync {
fi
else
if [ "$resumeInitiator" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" &
syncUpdate "${INITIATOR[$__type]}" "${TARGET[$__type]}" $rsyncRemoteDelete &
ExecTasks $! "${FUNCNAME[0]}_syncUpdate_target" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__initiatorLastActionFile]}"
@ -1979,7 +1991,7 @@ function Sync {
fi
fi
if [ "$resumeTarget" == "${SYNC_ACTION[5]}" ]; then
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" &
syncUpdate "${TARGET[$__type]}" "${INITIATOR[$__type]}" $rsyncRemoteDelete &
ExecTasks $! "${FUNCNAME[0]}_syncUpdate_initiator" false 0 0 $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME false $SLEEP_TIME $KEEP_LOGGING
if [ $? -ne 0 ]; then
echo "${SYNC_ACTION[5]}" > "${INITIATOR[$__targetLastActionFile]}"
@ -1993,7 +2005,10 @@ function Sync {
fi
fi
# If SYNC_TYPE is not bidirectional, skip all other steps
if [ "$SYNC_TYPE" == "initiator2target" ] || [ "$SYNC_TYPE" == "target2initiator" ]; then
echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__initiatorLastActionFile]}"
echo "${SYNC_ACTION[9]}" > "${INITIATOR[$__targetLastActionFile]}"
resumeInitiator="${SYNC_ACTION[9]}"
resumeTarget="${SYNC_ACTION[9]}"
fi
@ -2416,16 +2431,25 @@ function _SummaryFromRsyncFile {
if [ -f "$summaryFile" ]; then
while read -r file; do
# grep -E "^<|^>|^\." = Remove all lines that do not begin with <, > or . to deal with a bizarre bug involving rsync 3.0.6 / CentOS 6 and --skip-compress showing 'adding zip' line for every skipped compressed extension
# grep -E "^<|^>|^\." = Remove all lines that do not begin with '<', '>' or '.' to deal with a bizarre bug involving rsync 3.0.6 / CentOS 6 and --skip-compress showing 'adding zip' line for every skipped compressed extension
if echo "$file" | grep -E "^<|^>|^\." > /dev/null 2>&1; then
# awk removes first part of line until space, then show all others
Logger "$direction $replicaPath$(echo $file | awk '{for (i=2; i<NF; i++) printf $i " "; print $NF}')" "ALWAYS"
if [ "$direction" == "+ >>" ]; then
Logger "+ $direction $replicaPath$(echo $file | awk '{for (i=2; i<NF; i++) printf $i " "; print $NF}')" "ALWAYS"
if [ "$direction" == ">>" ]; then
TARGET_UPDATES_COUNT=$((TARGET_UPDATES_COUNT+1))
elif [ "$direction" == "+ <<" ]; then
elif [ "$direction" == "<<" ]; then
INITIATOR_UPDATES_COUNT=$((INITIATOR_UPDATES_COUNT+1))
fi
fi
if echo "$file" | grep -E "\*deleting" > /dev/null 2>&1; then
if [ "$direction" == ">>" ]; then
Logger "- $direction $replicaPath$(echo $file | awk '{print $2}')" "ALWAYS"
TARGET_DELETES_COUNT=$((TARGET_DELETES_COUNT+1))
elif [ "$direction" == "<<" ]; then
Logger "- $direction $replicaPath$(echo $file | awk '{print $2}')" "ALWAYS"
INITIATOR_DELETES_COUNT=$((INITIATOR_DELETES_COUNT+1))
fi
fi
done < "$summaryFile"
fi
}
@ -2439,10 +2463,10 @@ function _SummaryFromDeleteFile {
if [ -f "$summaryFile" ]; then
while read -r file; do
Logger "$direction $replicaPath$file" "ALWAYS"
if [ "$direction" == "- >>" ]; then
Logger "- $direction $replicaPath$file" "ALWAYS"
if [ "$direction" == ">>" ]; then
TARGET_DELETES_COUNT=$((TARGET_DELETES_COUNT+1))
elif [ "$direction" == "- <<" ]; then
elif [ "$direction" == "<<" ]; then
INITIATOR_DELETES_COUNT=$((INITIATOR_DELETES_COUNT+1))
fi
done < "$summaryFile"
@ -2455,22 +2479,22 @@ function Summary {
(
_LOGGER_PREFIX=""
Logger "Attrib updates: INITIATOR << >> TARGET" "ALWAYS"
Logger "Attributes only updates: INITIATOR << >> TARGET" "ALWAYS"
_SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID.$TSTAMP" "~ >>"
_SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID.$TSTAMP" "~ <<"
_SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.target.$SCRIPT_PID.$TSTAMP" ">>"
_SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.attr-update.initiator.$SCRIPT_PID.$TSTAMP" "<<"
Logger "File transfers: INITIATOR << >> TARGET (may include file ownership and timestamp attributes)" "ALWAYS"
_SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID.$TSTAMP" "+ >>"
_SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID.$TSTAMP" "+ <<"
Logger "File transfers and simple deletions: INITIATOR << >> TARGET (may include file ownership and timestamp attributes)" "ALWAYS"
_SummaryFromRsyncFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.target.$SCRIPT_PID.$TSTAMP" ">>"
_SummaryFromRsyncFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.update.initiator.$SCRIPT_PID.$TSTAMP" "<<"
Logger "File deletions: INITIATOR << >> TARGET" "ALWAYS"
if [ "$REMOTE_OPERATION" == true ]; then
_SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "${INITIATOR[$__replicaDir]}${INITIATOR[$__stateDir]}/target${TARGET[$__successDeletedListFile]}" "- >>"
else
_SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID.$TSTAMP" "- >>"
_SummaryFromDeleteFile "${TARGET[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.target.$SCRIPT_PID.$TSTAMP" ">>"
fi
_SummaryFromDeleteFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID.$TSTAMP" "- <<"
_SummaryFromDeleteFile "${INITIATOR[$__replicaDir]}" "$RUN_DIR/$PROGRAM.delete.initiator.$SCRIPT_PID.$TSTAMP" "<<"
Logger "Initiator has $INITIATOR_UPDATES_COUNT updates." "ALWAYS"
Logger "Target has $TARGET_UPDATES_COUNT updates." "ALWAYS"

3200
dev/n_osync.sh.o
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save