diff --git a/dev/debug_osync.sh b/dev/debug_osync.sh index f91244d..fffea82 100755 --- a/dev/debug_osync.sh +++ b/dev/debug_osync.sh @@ -40,8 +40,8 @@ IS_STABLE=yes # CleanUp no #__WITH_PARANOIA_DEBUG -_OFUNCTIONS_VERSION=2.1 -_OFUNCTIONS_BUILD=2017032301 +_OFUNCTIONS_VERSION=2.1.2 +_OFUNCTIONS_BUILD=2017052601 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -66,6 +66,9 @@ fi ## Correct output of sort command (language agnostic sorting) export LC_ALL=C +## Default umask for file creation +umask 0077 + # Standard alert mail body MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." @@ -293,7 +296,7 @@ function Logger { return elif [ "$level" == "VERBOSE" ]; then if [ $_LOGGER_VERBOSE == true ]; then - _Logger "$prefix:$value" "$prefix$value" + _Logger "$prefix($level):$value" "$prefix$value" fi return elif [ "$level" == "ALWAYS" ]; then @@ -662,6 +665,18 @@ function Spinner { fi } +function _PerfProfiler { #__WITH_PARANOIA_DEBUG + local perfString #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + perfString=$(ps -p $$ -o %cpu,%mem,cmd,time) #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG + perfString="$perfString\n"$(ps -p $i -o %cpu,%mem,cmd,time | tail -1) #__WITH_PARANOIA_DEBUG + done #__WITH_PARANOIA_DEBUG + #__WITH_PARANOIA_DEBUG + Logger "PerfProfiler: $perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG +} #__WITH_PARANOIA_DEBUG + # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 @@ -794,6 +809,11 @@ function WaitForTaskCompletion { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + + if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG + _PerfProfiler ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG + done Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG @@ -959,6 +979,10 @@ function ParallelExec { # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + + if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG + _PerfProfiler ##__WITH_PARANOIA_DEBUG + fi ##__WITH_PARANOIA_DEBUG done return $errorCount @@ -1184,6 +1208,62 @@ function GetLocalOS { LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" } +#__BEGIN_WITH_PARANOIA_DEBUG +function __CheckArguments { + # Checks the number of arguments of a function and raises an error if some are missing + + if [ "$_DEBUG" == "yes" ]; then + local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments + local numberOfGivenArguments="${2}" # Number of arguments that have been passed + + local minArgs + local maxArgs + + # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) + # If any of the arguments contains spaces, bash things there are two aguments + # In order to avoid this, we need to iterate over ${3} and count + + callerName="${FUNCNAME[1]}" + + local iterate=3 + local fetchArguments=true + local argList="" + local countedArguments + while [ $fetchArguments == true ]; do + cmd='argument=${'$iterate'}' + eval $cmd + if [ "$argument" == "" ]; then + fetchArguments=false + else + argList="$argList[Argument $((iterate-2)): $argument] " + iterate=$((iterate+1)) + fi + done + + countedArguments=$((iterate-3)) + + if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then + minArgs=$numberOfArguments + maxArgs=$numberOfArguments + else + IFS='-' read minArgs maxArgs <<< "$numberOfArguments" + fi + + Logger "Entering function [$callerName]." "PARANOIA_DEBUG" + + if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then + Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" + Logger "$callerName arguments: $argList" "ERROR" + else + if [ ! -z "$argList" ]; then + Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" + fi + fi + fi +} + +#__END_WITH_PARANOIA_DEBUG + function GetRemoteOS { __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG @@ -1439,62 +1519,6 @@ function CheckConnectivity3rdPartyHosts { fi #__WITH_PARANOIA_DEBUG } -#__BEGIN_WITH_PARANOIA_DEBUG -function __CheckArguments { - # Checks the number of arguments of a function and raises an error if some are missing - - if [ "$_DEBUG" == "yes" ]; then - local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments - local numberOfGivenArguments="${2}" # Number of arguments that have been passed - - local minArgs - local maxArgs - - # All arguments of the function to check are passed as array in ${3} (the function call waits for $@) - # If any of the arguments contains spaces, bash things there are two aguments - # In order to avoid this, we need to iterate over ${3} and count - - callerName="${FUNCNAME[1]}" - - local iterate=3 - local fetchArguments=true - local argList="" - local countedArguments - while [ $fetchArguments == true ]; do - cmd='argument=${'$iterate'}' - eval $cmd - if [ "$argument" == "" ]; then - fetchArguments=false - else - argList="$argList[Argument $((iterate-2)): $argument] " - iterate=$((iterate+1)) - fi - done - - countedArguments=$((iterate-3)) - - if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then - minArgs=$numberOfArguments - maxArgs=$numberOfArguments - else - IFS='-' read minArgs maxArgs <<< "$numberOfArguments" - fi - - Logger "Entering function [$callerName]." "PARANOIA_DEBUG" - - if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then - Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR" - Logger "$callerName arguments: $argList" "ERROR" - else - if [ ! -z "$argList" ]; then - Logger "$callerName arguments: $argList" "PARANOIA_DEBUG" - fi - fi - fi -} - -#__END_WITH_PARANOIA_DEBUG - function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" @@ -1845,35 +1869,46 @@ function ParentPid { # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 -vercomp () { - if [[ $1 == $2 ]] - then - return 0 - fi - local IFS=. - local i ver1=($1) ver2=($2) - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) - do - ver1[i]=0 - done - for ((i=0; i<${#ver1[@]}; i++)) - do - if [[ -z ${ver2[i]} ]] - then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})) - then - return 1 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})) - then - return 2 - fi - done - return 0 +function VerComp () { + if [ "$1" == "" ] || [ "$2" == "" ]; then + Logger "Bogus Vercomp values [$1] and [$2]." "WARN" + return 1 + fi + + if [[ $1 == $2 ]] + then + echo 0 + return + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + echo 1 + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + echo 2 + return + fi + done + + echo 0 + return } function GetConfFileValue () { @@ -1906,6 +1941,29 @@ function SetConfFileValue () { fi } +# Function can replace [ -f /some/file* ] tests +# Modified version of http://stackoverflow.com/a/6364244/2635443 +function WildcardFileExists () { + local file="${1}" + local exists=0 + + for f in $file; do + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then + exists=1 + break + fi + done + + if [ $exists -eq 1 ]; then + echo 1 + else + echo 0 + fi +} + # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 diff --git a/install.sh b/install.sh index ee428b0..b650720 100755 --- a/install.sh +++ b/install.sh @@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh" PROGRAM_BATCH=$PROGRAM"-batch.sh" SSH_FILTER="ssh_filter.sh" -SCRIPT_BUILD=2017032101 +SCRIPT_BUILD=2017041701 ## osync / obackup / pmocr / zsnap install script ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 @@ -342,9 +342,9 @@ function CopyServiceFiles { elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true - QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]." - QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]." - QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]." + QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]." + QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]." + QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]." else QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation." fi @@ -386,7 +386,11 @@ function RemoveFile { function RemoveAll { RemoveFile "$BIN_DIR/$PROGRAM_BINARY" - RemoveFile "$BIN_DIR/$PROGRAM_BATCH" + + if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then + RemoveFile "$BIN_DIR/$PROGRAM_BATCH" + fi + if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal RemoveFile "$BIN_DIR/$SSH_FILTER" else @@ -447,7 +451,9 @@ else CreateDir "$BIN_DIR" CopyExampleFiles CopyProgram - CopyServiceFiles + if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then + CopyServiceFiles + fi QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM" if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then QuickLogger "" diff --git a/osync.sh b/osync.sh index 7ac85bf..9738dbf 100755 --- a/osync.sh +++ b/osync.sh @@ -9,8 +9,8 @@ IS_STABLE=yes -_OFUNCTIONS_VERSION=2.1 -_OFUNCTIONS_BUILD=2017032301 +_OFUNCTIONS_VERSION=2.1.2 +_OFUNCTIONS_BUILD=2017052601 _OFUNCTIONS_BOOTSTRAP=true ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr @@ -35,6 +35,9 @@ fi ## Correct output of sort command (language agnostic sorting) export LC_ALL=C +## Default umask for file creation +umask 0077 + # Standard alert mail body MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." @@ -252,7 +255,7 @@ function Logger { return elif [ "$level" == "VERBOSE" ]; then if [ $_LOGGER_VERBOSE == true ]; then - _Logger "$prefix:$value" "$prefix$value" + _Logger "$prefix($level):$value" "$prefix$value" fi return elif [ "$level" == "ALWAYS" ]; then @@ -612,6 +615,7 @@ function Spinner { } + # Time control function for background processes, suitable for multiple synchronous processes # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 # Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached @@ -736,6 +740,8 @@ function WaitForTaskCompletion { pidsArray=("${newPidsArray[@]}") # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + + done @@ -894,6 +900,7 @@ function ParallelExec { # Trivial wait time for bash to not eat up all CPU sleep $sleepTime + done return $errorCount @@ -1119,6 +1126,7 @@ function GetLocalOS { } + function GetRemoteOS { if [ "$REMOTE_OPERATION" != "yes" ]; then @@ -1362,7 +1370,6 @@ function CheckConnectivity3rdPartyHosts { fi } - function RsyncPatternsAdd { local patternType="${1}" # exclude or include local pattern="${2}" @@ -1706,35 +1713,46 @@ function ParentPid { # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 -vercomp () { - if [[ $1 == $2 ]] - then - return 0 - fi - local IFS=. - local i ver1=($1) ver2=($2) - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) - do - ver1[i]=0 - done - for ((i=0; i<${#ver1[@]}; i++)) - do - if [[ -z ${ver2[i]} ]] - then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})) - then - return 1 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})) - then - return 2 - fi - done - return 0 +function VerComp () { + if [ "$1" == "" ] || [ "$2" == "" ]; then + Logger "Bogus Vercomp values [$1] and [$2]." "WARN" + return 1 + fi + + if [[ $1 == $2 ]] + then + echo 0 + return + fi + + local IFS=. + local i ver1=($1) ver2=($2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) + do + ver1[i]=0 + done + for ((i=0; i<${#ver1[@]}; i++)) + do + if [[ -z ${ver2[i]} ]] + then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})) + then + echo 1 + return + fi + if ((10#${ver1[i]} < 10#${ver2[i]})) + then + echo 2 + return + fi + done + + echo 0 + return } function GetConfFileValue () { @@ -1767,6 +1785,29 @@ function SetConfFileValue () { fi } +# Function can replace [ -f /some/file* ] tests +# Modified version of http://stackoverflow.com/a/6364244/2635443 +function WildcardFileExists () { + local file="${1}" + local exists=0 + + for f in $file; do + ## Check if the glob gets expanded to existing files. + ## If not, f here will be exactly the pattern above + ## and the exists test will evaluate to false. + if [ -e "$f" ]; then + exists=1 + break + fi + done + + if [ $exists -eq 1 ]; then + echo 1 + else + echo 0 + fi +} + # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap _OFUNCTIONS_BOOTSTRAP=true [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1