Archives
/
osync
mirror of https://github.com/deajan/osync
2
0
Fork 0
Code Issues Releases Wiki Activity

Rebuilt targets

Browse Source
pull/100/head
deajan 7 years ago
parent bbff9d55d6
commit 56b9efd686
3 changed files with 232 additions and 127 deletions
Show Stats Download Patch File Download Diff File

234
dev/debug_osync.sh
Unescape Escape View File

@ -40,8 +40,8 @@ IS_STABLE=yes
# CleanUp no #__WITH_PARANOIA_DEBUG # CleanUp no #__WITH_PARANOIA_DEBUG
_OFUNCTIONS_VERSION=2.1 _OFUNCTIONS_VERSION=2.1.2
_OFUNCTIONS_BUILD=2017032301 _OFUNCTIONS_BUILD=2017052601
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -66,6 +66,9 @@ fi
## Correct output of sort command (language agnostic sorting) ## Correct output of sort command (language agnostic sorting)
export LC_ALL=C export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body # Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -293,7 +296,7 @@ function Logger {
return return
elif [ "$level" == "VERBOSE" ]; then elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value" _Logger "$prefix($level):$value" "$prefix$value"
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
@ -662,6 +665,18 @@ function Spinner {
fi fi
} }
function _PerfProfiler { #__WITH_PARANOIA_DEBUG
local perfString #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
perfString=$(ps -p $$ -o %cpu,%mem,cmd,time) #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
for i in $(pgrep -P $$); do #__WITH_PARANOIA_DEBUG
perfString="$perfString\n"$(ps -p $i -o %cpu,%mem,cmd,time | tail -1) #__WITH_PARANOIA_DEBUG
done #__WITH_PARANOIA_DEBUG
#__WITH_PARANOIA_DEBUG
Logger "PerfProfiler: $perfString" "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
} #__WITH_PARANOIA_DEBUG
# Time control function for background processes, suitable for multiple synchronous processes # Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
@ -794,6 +809,11 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $sleepTime sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done done
Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG Logger "${FUNCNAME[0]} ended for [$callerName] using [$pidCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG
@ -959,6 +979,10 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $sleepTime sleep $sleepTime
if [ "$_PERF_PROFILER" == "yes" ]; then ##__WITH_PARANOIA_DEBUG
_PerfProfiler ##__WITH_PARANOIA_DEBUG
fi ##__WITH_PARANOIA_DEBUG
done done
return $errorCount return $errorCount
@ -1184,6 +1208,62 @@ function GetLocalOS {
LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)" LOCAL_OS_FULL="$localOsVar ($localOsName $localOsVer)"
} }
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function GetRemoteOS { function GetRemoteOS {
__CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG __CheckArguments 0 $# "$@" #__WITH_PARANOIA_DEBUG
@ -1439,62 +1519,6 @@ function CheckConnectivity3rdPartyHosts {
fi #__WITH_PARANOIA_DEBUG fi #__WITH_PARANOIA_DEBUG
} }
#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
# Checks the number of arguments of a function and raises an error if some are missing
if [ "$_DEBUG" == "yes" ]; then
local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
local numberOfGivenArguments="${2}" # Number of arguments that have been passed
local minArgs
local maxArgs
# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
# If any of the arguments contains spaces, bash things there are two aguments
# In order to avoid this, we need to iterate over ${3} and count
callerName="${FUNCNAME[1]}"
local iterate=3
local fetchArguments=true
local argList=""
local countedArguments
while [ $fetchArguments == true ]; do
cmd='argument=${'$iterate'}'
eval $cmd
if [ "$argument" == "" ]; then
fetchArguments=false
else
argList="$argList[Argument $((iterate-2)): $argument] "
iterate=$((iterate+1))
fi
done
countedArguments=$((iterate-3))
if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
minArgs=$numberOfArguments
maxArgs=$numberOfArguments
else
IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
fi
Logger "Entering function [$callerName]." "PARANOIA_DEBUG"
if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
Logger "$callerName arguments: $argList" "ERROR"
else
if [ ! -z "$argList" ]; then
Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
fi
fi
fi
}
#__END_WITH_PARANOIA_DEBUG
function RsyncPatternsAdd { function RsyncPatternsAdd {
local patternType="${1}" # exclude or include local patternType="${1}" # exclude or include
local pattern="${2}" local pattern="${2}"
@ -1845,35 +1869,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () { function VerComp () {
if [[ $1 == $2 ]] if [ "$1" == "" ] || [ "$2" == "" ]; then
then Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 0 return 1
fi fi
local IFS=.
local i ver1=($1) ver2=($2) if [[ $1 == $2 ]]
# fill empty fields in ver1 with zeros then
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) echo 0
do return
ver1[i]=0 fi
done
for ((i=0; i<${#ver1[@]}; i++)) local IFS=.
do local i ver1=($1) ver2=($2)
if [[ -z ${ver2[i]} ]] # fill empty fields in ver1 with zeros
then for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
# fill empty fields in ver2 with zeros do
ver2[i]=0 ver1[i]=0
fi done
if ((10#${ver1[i]} > 10#${ver2[i]})) for ((i=0; i<${#ver1[@]}; i++))
then do
return 1 if [[ -z ${ver2[i]} ]]
fi then
if ((10#${ver1[i]} < 10#${ver2[i]})) # fill empty fields in ver2 with zeros
then ver2[i]=0
return 2 fi
fi if ((10#${ver1[i]} > 10#${ver2[i]}))
done then
return 0 echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
} }
function GetConfFileValue () { function GetConfFileValue () {
@ -1906,6 +1941,29 @@ function SetConfFileValue () {
fi fi
} }
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1

18
install.sh
Unescape Escape View File

@ -12,7 +12,7 @@ PROGRAM_BINARY=$PROGRAM".sh"
PROGRAM_BATCH=$PROGRAM"-batch.sh" PROGRAM_BATCH=$PROGRAM"-batch.sh"
SSH_FILTER="ssh_filter.sh" SSH_FILTER="ssh_filter.sh"
SCRIPT_BUILD=2017032101 SCRIPT_BUILD=2017041701
## osync / obackup / pmocr / zsnap install script ## osync / obackup / pmocr / zsnap install script
## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11 ## Tested on RHEL / CentOS 6 & 7, Fedora 23, Debian 7 & 8, Mint 17 and FreeBSD 8, 10 and 11
@ -342,9 +342,9 @@ function CopyServiceFiles {
elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then elif ([ "$init" == "initV" ] && [ -f "$SCRIPT_PATH/$SERVICE_FILE_INIT" ] && [ -d "$SERVICE_DIR_INIT" ]); then
CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true CopyFile "$SCRIPT_PATH" "$SERVICE_DIR_INIT" "$SERVICE_FILE_INIT" "755" "" "" true
QuickLogger "Created osync-srv service in [$SERVICE_DIR_INIT]." QuickLogger "Created [$SERVICE_NAME] service in [$SERVICE_DIR_INIT]."
QuickLogger "Can be activated with [service $OSYNC_SERVICE_FILE_INIT start]." QuickLogger "Can be activated with [service $SERVICE_FILE_INIT start]."
QuickLogger "Can be enabled on boot with [chkconfig $OSYNC_SERVICE_FILE_INIT on]." QuickLogger "Can be enabled on boot with [chkconfig $SERVICE_FILE_INIT on]."
else else
QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation." QuickLogger "Cannot define what init style is in use on this system. Skipping service file installation."
fi fi
@ -386,7 +386,11 @@ function RemoveFile {
function RemoveAll { function RemoveAll {
RemoveFile "$BIN_DIR/$PROGRAM_BINARY" RemoveFile "$BIN_DIR/$PROGRAM_BINARY"
RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
RemoveFile "$BIN_DIR/$PROGRAM_BATCH"
fi
if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal if [ ! -f "$BIN_DIR/osync.sh" ] && [ ! -f "$BIN_DIR/obackup.sh" ]; then # Check if any other program requiring ssh filter is present before removal
RemoveFile "$BIN_DIR/$SSH_FILTER" RemoveFile "$BIN_DIR/$SSH_FILTER"
else else
@ -447,7 +451,9 @@ else
CreateDir "$BIN_DIR" CreateDir "$BIN_DIR"
CopyExampleFiles CopyExampleFiles
CopyProgram CopyProgram
CopyServiceFiles if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "pmocr" ]; then
CopyServiceFiles
fi
QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM" QuickLogger "$PROGRAM installed. Use with $BIN_DIR/$PROGRAM"
if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then if [ "$PROGRAM" == "osync" ] || [ "$PROGRAM" == "obackup" ]; then
QuickLogger "" QuickLogger ""

107
osync.sh
Unescape Escape View File

@ -9,8 +9,8 @@ IS_STABLE=yes
_OFUNCTIONS_VERSION=2.1 _OFUNCTIONS_VERSION=2.1.2
_OFUNCTIONS_BUILD=2017032301 _OFUNCTIONS_BUILD=2017052601
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr ## BEGIN Generic bash functions written in 2013-2017 by Orsiris de Jong - http://www.netpower.fr - ozy@netpower.fr
@ -35,6 +35,9 @@ fi
## Correct output of sort command (language agnostic sorting) ## Correct output of sort command (language agnostic sorting)
export LC_ALL=C export LC_ALL=C
## Default umask for file creation
umask 0077
# Standard alert mail body # Standard alert mail body
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors." MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."
@ -252,7 +255,7 @@ function Logger {
return return
elif [ "$level" == "VERBOSE" ]; then elif [ "$level" == "VERBOSE" ]; then
if [ $_LOGGER_VERBOSE == true ]; then if [ $_LOGGER_VERBOSE == true ]; then
_Logger "$prefix:$value" "$prefix$value" _Logger "$prefix($level):$value" "$prefix$value"
fi fi
return return
elif [ "$level" == "ALWAYS" ]; then elif [ "$level" == "ALWAYS" ]; then
@ -612,6 +615,7 @@ function Spinner {
} }
# Time control function for background processes, suitable for multiple synchronous processes # Time control function for background processes, suitable for multiple synchronous processes
# Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2 # Fills a global variable called WAIT_FOR_TASK_COMPLETION_$callerName that contains list of failed pids in format pid1:result1;pid2:result2
# Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached # Also sets a global variable called HARD_MAX_EXEC_TIME_REACHED_$callerName to true if hardMaxTime is reached
@ -736,6 +740,8 @@ function WaitForTaskCompletion {
pidsArray=("${newPidsArray[@]}") pidsArray=("${newPidsArray[@]}")
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $sleepTime sleep $sleepTime
done done
@ -894,6 +900,7 @@ function ParallelExec {
# Trivial wait time for bash to not eat up all CPU # Trivial wait time for bash to not eat up all CPU
sleep $sleepTime sleep $sleepTime
done done
return $errorCount return $errorCount
@ -1119,6 +1126,7 @@ function GetLocalOS {
} }
function GetRemoteOS { function GetRemoteOS {
if [ "$REMOTE_OPERATION" != "yes" ]; then if [ "$REMOTE_OPERATION" != "yes" ]; then
@ -1362,7 +1370,6 @@ function CheckConnectivity3rdPartyHosts {
fi fi
} }
function RsyncPatternsAdd { function RsyncPatternsAdd {
local patternType="${1}" # exclude or include local patternType="${1}" # exclude or include
local pattern="${2}" local pattern="${2}"
@ -1706,35 +1713,46 @@ function ParentPid {
# Neat version compare function found at http://stackoverflow.com/a/4025065/2635443 # Neat version compare function found at http://stackoverflow.com/a/4025065/2635443
# Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2 # Returns 0 if equal, 1 if $1 > $2 and 2 if $1 < $2
vercomp () { function VerComp () {
if [[ $1 == $2 ]] if [ "$1" == "" ] || [ "$2" == "" ]; then
then Logger "Bogus Vercomp values [$1] and [$2]." "WARN"
return 0 return 1
fi fi
local IFS=.
local i ver1=($1) ver2=($2) if [[ $1 == $2 ]]
# fill empty fields in ver1 with zeros then
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) echo 0
do return
ver1[i]=0 fi
done
for ((i=0; i<${#ver1[@]}; i++)) local IFS=.
do local i ver1=($1) ver2=($2)
if [[ -z ${ver2[i]} ]] # fill empty fields in ver1 with zeros
then for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
# fill empty fields in ver2 with zeros do
ver2[i]=0 ver1[i]=0
fi done
if ((10#${ver1[i]} > 10#${ver2[i]})) for ((i=0; i<${#ver1[@]}; i++))
then do
return 1 if [[ -z ${ver2[i]} ]]
fi then
if ((10#${ver1[i]} < 10#${ver2[i]})) # fill empty fields in ver2 with zeros
then ver2[i]=0
return 2 fi
fi if ((10#${ver1[i]} > 10#${ver2[i]}))
done then
return 0 echo 1
return
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
echo 2
return
fi
done
echo 0
return
} }
function GetConfFileValue () { function GetConfFileValue () {
@ -1767,6 +1785,29 @@ function SetConfFileValue () {
fi fi
} }
# Function can replace [ -f /some/file* ] tests
# Modified version of http://stackoverflow.com/a/6364244/2635443
function WildcardFileExists () {
local file="${1}"
local exists=0
for f in $file; do
## Check if the glob gets expanded to existing files.
## If not, f here will be exactly the pattern above
## and the exists test will evaluate to false.
if [ -e "$f" ]; then
exists=1
break
fi
done
if [ $exists -eq 1 ]; then
echo 1
else
echo 0
fi
}
# If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap # If using "include" statements, make sure the script does not get executed unless it's loaded by bootstrap
_OFUNCTIONS_BOOTSTRAP=true _OFUNCTIONS_BOOTSTRAP=true
[ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1 [ "$_OFUNCTIONS_BOOTSTRAP" != true ] && echo "Please use bootstrap.sh to load this dev version of $(basename $0)" && exit 1

Write Preview
Loading…
Cancel
Save