Lot of bugfixes... Time for the last beta before going release candidate.

CHANGELOG.md

@@ -1,22 +1,24 @@
 FUTURE IMPROVEMENTS
 -------------------
 
-- Merge master and slave functions
+- Sync function merge (master and slave functions are more more or less the same)
-- Merge tree current and after functions
+- Tree function merge (current and after tree functions are the same except for output filename and logging)
-- Tree functions execute piped commands (grep, awk) on master when launched on remote slave, can cause more bandwith usage
+- Tree functions execute piped commands (grep, awk) on master when launched on remote slave which can cause more bandwith usage
 
 KNOWN ISSUES
 ------------
 
-(v?)- Cannot write pidlock on remote slave with SUDO_EXEC=yes but insufficient rights (sudo does not work for command echo)
+- If master and remote slave systems don't have rsync in the same path, execution may fail (RSYNC_PATH is configured on master but also executed on slave)
-- If master and remote slave aren't the same distros and rsync binary isn't in the same path, execution may fail (RSYNC_PATH should be configurable)
-- Possible non deleted file with space in name on master replica from slave remote replica
-- can load configuration files that don't have .conf extension...
-- Softdelete functions do not honor maximum execution time
 
 RECENT CHANGES
 --------------
 
+- Fixed a possible error upon master replica lock check
+- Fixed exclude directorires with spaces in names generate errros on master replica tree functions
+- Dryruns won't create after run tree lists and therefore not prevent building real run delete lists
+- Softdelete and conflict backup functions are now time controlled
+- Added bandwidth limit
+- Update and delete functions now run rsync with --stats parameter
 - Fixed LoadConfigFile function will not warn on wrong config file
 - Without --verbose parameter, last sync details are still logged to /tmp/osync_(pid)
 - Added --no-maxtime parameter for sync big changes without enforcing execution time checks

osync.sh

@@ -1,9 +1,9 @@
-#!/bin/bash
+*#!/bin/bash
 
 ###### Osync - Rsync based two way sync engine with fault tolerance
 ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr) 
 OSYNC_VERSION=0.98
-OSYNC_BUILD=0408201302
+OSYNC_BUILD=0408201305
 
 DEBUG=no
 SCRIPT_PID=$$
@@ -914,6 +914,11 @@ function delete_on_master
 
 function master_tree_after
 {
+	if [ $dryrun -eq 1 ]
+	then
+		Log "No need to create after run master replica file list, nothing should have changed."
+		return 0
+	fi
         Log "Creating after run master replica file list."
         $(which $RSYNC_EXECUTABLE) --rsync-path="$RSYNC_PATH" -rlptgoDE8 $RSYNC_ARGS --exclude "$OSYNC_DIR" $RSYNC_EXCLUDE --list-only "$MASTER_SYNC_DIR/" | grep "^-\|^d" | awk '{$1=$2=$3=$4="" ;print}' | awk '{$1=$1 ;print}' | (grep -v "^\.$" || :) | sort > /dev/shm/osync_master-tree-after_$SCRIPT_PID &
 	child_pid=$!
@@ -932,6 +937,11 @@ function master_tree_after
 
 function slave_tree_after
 {
+	if [ $dryrun -eq 1 ]
+	then
+		Log "No need to create after frun slave replica file list, nothing should have changed."
+		return 0
+	fi
         Log "Creating after run slave replica file list."
 	if [ "$REMOTE_SYNC" == "yes" ]
 	then
@@ -1134,9 +1144,18 @@ function SoftDelete
 			Log "Removing backups older than $CONFLICT_BACKUP_DAYS days on master replica."
 			if [ $dryrun -eq 1 ]
 			then
-				find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS
+				find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS &
 			else
-				find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf
+				find "$MASTER_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf &
+			fi
+			child_pid=$!
+        		WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
+        		retval=$?
+			if [ $retval -ne 0 ]
+			then
+				LogError "Error while executing conflict backup cleanup on master replica."
+			else
+				Log "Conflict backup cleanup complete on master replica."
 			fi
 		fi
 		
@@ -1149,6 +1168,15 @@ function SoftDelete
 			else
 				eval "$SSH_CMD \"if [ -d \\\"$SLAVE_BACKUP_DIR\\\" ]; then $COMMAND_SUDO find \\\"$SLAVE_BACKUP_DIR/\\\" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf; fi\""
 			fi
+			child_pid=$!
+                        WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
+                        retval=$?
+                        if [ $retval -ne 0 ]
+                        then
+                                LogError "Error while executing conflict backup cleanup on slave replica."
+                        else
+                                Log "Conflict backup cleanup complete on slave replica."
+                        fi
 		else
 			if [ -d "$SLAVE_BACKUP_DIR" ]
 			then
@@ -1159,6 +1187,15 @@ function SoftDelete
 				else
 					find "$SLAVE_BACKUP_DIR/" -ctime +$CONFLICT_BACKUP_DAYS | xargs rm -rf
 				fi
+				child_pid=$!
+	                        WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
+        	                retval=$?
+                	        if [ $retval -ne 0 ]
+                        	then
+                                	LogError "Error while executing conflict backup cleanup on slave replica."
+                        	else
+                                	Log "Conflict backup cleanup complete on slave replica."
+                        	fi
 			fi
 		fi
 	fi
@@ -1174,6 +1211,15 @@ function SoftDelete
 			else
 				find "$MASTER_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf
 			fi
+			child_pid=$!
+                        WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
+                        retval=$?
+                        if [ $retval -ne 0 ]
+                        then
+                                LogError "Error while executing soft delete cleanup on master replica."
+                        else
+                                Log "Soft delete cleanup complete on master replica."
+                        fi
 		fi
 
 		if [ "$REMOTE_SYNC" == "yes" ]
@@ -1185,6 +1231,16 @@ function SoftDelete
 			else
 				eval "$SSH_CMD \"if [ -d \\\"$SLAVE_DELETE_DIR\\\" ]; then $COMMAND_SUDO find \\\"$SLAVE_DELETE_DIR/\\\" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf; fi\""
 			fi
+			child_pid=$!
+                        WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
+                        retval=$?
+                        if [ $retval -ne 0 ]
+                        then
+                                LogError "Error while executing soft delete cleanup on slave replica."
+                        else
+                                Log "Soft delete cleanup complete on slave replica."
+                        fi
+
 		else
 			if [ -d "$SLAVE_DELETE_DIR" ]
 			then
@@ -1195,6 +1251,15 @@ function SoftDelete
 				else
 					find "$SLAVE_DELETE_DIR/" -ctime +$SOFT_DELETE_DAYS | xargs rm -rf
 				fi
+				child_pid=$!
+                       		WaitForCompletion $child_pid $SOFT_MAX_EXEC_TIME 0
+                        	retval=$?
+                        	if [ $retval -ne 0 ]
+                        	then
+                               		LogError "Error while executing soft delete cleanup on slave replica."
+                        	else
+                                	Log "Soft delete cleanup complete on slave replica."
+                        	fi
 			fi
 		fi
 	fi
@@ -1287,6 +1352,11 @@ function Init
 		RSYNC_ARGS=""
 	fi
 
+	if [ "$BANDWIDTH" != "0" ]
+	then
+		RSYNC_ARGS=$RSYNC_ARGS" --bwlimit=$BANDWIDTH"
+	fi
+
 	## Conflict options
 	if [ "$CONFLICT_BACKUP" != "no" ]
 	then

sync.conf

@@ -2,17 +2,17 @@
 
 ###### Osync - Rsync based two way sync engine with fault tolerance
 ###### (L) 2013 by Orsiris "Ozy" de Jong (www.netpower.fr)
-#### Config file rev 2207201302
+#### Config file rev 0408201301
 
 ## Sync job identification, any string you want, no spaces
 SYNC_ID="sync_test"
 
 ## Directories to synchronize
-MASTER_SYNC_DIR="/home/git/osync/test/the separate dir1"
+MASTER_SYNC_DIR="/home/git/osync/test/dir1"
-SLAVE_SYNC_DIR="/home/git/osync/test/the separate dir2"
+SLAVE_SYNC_DIR="/home/git/osync/test/dir2"
 
 ## Create sync directories if they do not exist
-CREATE_DIRS=yes
+CREATE_DIRS=no
 
 ## List of directories to exclude in sync on both sides (rsync patterns, wildcards work). Must be relative paths. List is separated by PATH SEPARATOR CHAR defined below (semicolon by default).
 RSYNC_EXCLUDE_PATTERN="tmp;archives"
@@ -22,6 +22,9 @@ PATH_SEPARATOR_CHAR=";"
 ## Generate an alert if master or slave have lass space than given value in KB.
 MINIMUM_SPACE=10240
 
+## Bandwidth limit Kbytes / second. Leave 0 to disable limitation
+BANDWIDTH=0
+
 ## If enabled, synchronization will be processed with sudo command. See documentation
 SUDO_EXEC=no
 ## Paranoia option. Don't change this unless you read the documentation and know what you are doing.
@@ -48,8 +51,8 @@ PRESERVE_XATTR=no
 RSYNC_COMPRESS=yes
 
 ## Maximum execution time (in seconds) for sync process. Soft exec time only generates warning. Hard exec time will generate warning and stop sync process.
-SOFT_MAX_EXEC_TIME=18000
+SOFT_MAX_EXEC_TIME=7200
-HARD_MAX_EXEC_TIME=43200
+HARD_MAX_EXEC_TIME=10600
 
 ## If the same file exists on both sides, newer version will be used. If both files have the same timestamp but differ, CONFILCT_PREVALANCE sets winner
 CONFLICT_PREVALANCE=master