## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for
## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed.
## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
localid="${1:-base}"# Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
#TODO: not implemented yet
localsoftPerProcessTime="${2:-0}"
localhardPerProcessTime="${3:-0}"
localsoftMaxTime="${4:-0}"# If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
localhardMaxTime="${5:-0}"# If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
localsleepTime="${6:-.05}"# Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
localkeepLogging="${7:-0}"# Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
localcounting="${8:-true}"# Count time since function has been launched (true), or since script has been launched (false)
localspinner="${9:-true}"# Show spinner (true), do not show anything (false)
localnoTimeErrorLog="${10:-false}"# Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
localnoErrorLogAtAll="${11:-false}"# Do not log errros at all (false)
localexecTasksMode="${12:-1}"# In which mode the function should work, see above
localmainInput="${13}"# Contains list of pids / commands or filepath to list of pids / commands
localauxInput="${14}"# Contains list of conditional commands or filepath to list of conditional commands
localnumberOfProcesses="${15:-2}"# Number of simultaneous commands to run in ParallExec mode
# Mandatory arguments
localmainInput="${1}"# Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments
localid="${2:-base}"# Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
localreadFromFile="${3:-false}"# Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
localsoftPerProcessTime="${4:-0}"# Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
localhardPerProcessTime="${5:-0}"# Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
localsoftMaxTime="${6:-0}"# Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
localhardMaxTime="${7:-0}"# Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
localcounting="${8:-true}"# Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
localsleepTime="${9:-.5}"# Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
localkeepLogging="${10:-1800}"# Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
localspinner="${11:-true}"# Show spinner (true) or do not show anything (false) while running
localnoTimeErrorLog="${12:-false}"# Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
localnoErrorLogsAtAll="${13:-false}"# Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments
localnumberOfProcesses="${14:-0}"# Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
localauxInput="${15}"# Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
localmaxPostponeRetries="${16:-3}"# If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
localminTimeBetweenRetries="${17:-300}"# Time (in seconds) between postponed command retries
localvalidExitCodes="${18:-0}"# Semi colon separated list of valid main command exit codes which will not trigger errors
local i
localcallerName="${FUNCNAME[1]}"
Logger "${FUNCNAME[0]} called in $execTasksMode mode by [${FUNCNAME[0]} < ${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} ...].""PARANOIA_DEBUG"#__WITH_PARANOIA_DEBUG
Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems.""PARANOIA_DEBUG"#__WITH_PARANOIA_DEBUG
# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
function _ExecTasksTimeCheck {
if[$spinner==true];then
Spinner
fi
@ -936,13 +900,13 @@ function ExecTasks {
fi
if[$keepLogging -ne 0];then
if[$((($exec_time +1)%$keepLogging)) -eq 0];then
if[$(((exec_time +1)% keepLogging)) -eq 0];then
if[$log_ttime -ne $exec_time];then# Fix when sleep time lower than 1 second
log_ttime=$exec_time
if[$functionMode=="Wait"];then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})].""NOTICE"
elif[$functionMode=="ParallelExec"];then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})].""NOTICE"
Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})].""NOTICE"
## This function can monitor given pids as background processes, and stop them if max execution time is reached. Suitable for multiple synchronous processes.
## It can also take a list of commands to execute in parallel, and stop them if max execution time is reached.
## Function has 8 execution modes
# WaitForTaskCompletion mode: Monitor given pids as background processes, stop them if max execution time is reached. Suitaable for multiple synhronous processes.
# 1 : WaitForTaskCompletion, semi-colon separated list of pids to monitor
# 2 : WaitForTaskCompletion, list of pids to monior, from file, one per line
# ParallelExecMode: Take list of commands to execute in parallel, stop them if max execution time is reached.
# Also take optional conditional arguments to verifiy before execution main commands. Conditional command exit code 0 means ready to execute. Other exit codes will ignore/postpone main command.
# 3 : ParallelExec, semi-colon separated list of commands to execute in parallel, no conditions
# 4 : ParallelExec, semi-colon separated list of commands to execute in parallel , semi-colon separated list of conditional commands, ignoring main commands on condition failures
# 5 : ParallelExec, semi-colon separated list of commands, semi-colon separated list of conditional commands, postponing main commands on condition failures
# 6 : ParallelExec, list of commands from file, one per line, no conditions
# 7 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, ignoring main commands on condition failures
# 8 : ParallelExec, list of commands from file, one per line, list of conditional commands from file, one per line, postponing main commands on condition failures
# Exmaple: execute four du commands, only if directories exist, warn if execution takes more than 300 seconds, stop if takes longer than 900 seconds. Execute max 3 commands in parallel.
# ParallelExecMode also creates output for commands in "$RUN_DIR/$PROGRAM.ExecTasks.$id.$SCRIPT_PID.$TSTAMP"
## ofunctions.sh subfunction requirements:
## Function can work in:
## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for
## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed.
## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
localid="${1:-base}"# Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
#TODO: not implemented yet
localsoftPerProcessTime="${2:-0}"
localhardPerProcessTime="${3:-0}"
localsoftMaxTime="${4:-0}"# If process(es) with pid(s) $pids take longer than $softMaxTime seconds, will log a warning, unless $softMaxTime equals 0.
localhardMaxTime="${5:-0}"# If process(es) with pid(s) $pids take longer than $hardMaxTime seconds, will stop execution, unless $hardMaxTime equals 0.
localsleepTime="${6:-.05}"# Seconds between each state check, the shorter this value, the snappier it will be, but as a tradeoff cpu power will be used (general values between .05 and 1).
localkeepLogging="${7:-0}"# Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging.
localcounting="${8:-true}"# Count time since function has been launched (true), or since script has been launched (false)
localspinner="${9:-true}"# Show spinner (true), do not show anything (false)
localnoTimeErrorLog="${10:-false}"# Log errors when reaching soft / hard max time (false), do not log errors on those triggers (true)
#TODO not implemented
localnoErrorLogAtAll="${11:-false}"# Do not log errros at all (false)
localexecTasksMode="${12:-1}"# In which mode the function should work, see above
localmainInput="${13}"# Contains list of pids / commands or filepath to list of pids / commands
localauxInput="${14}"# Contains list of conditional commands or filepath to list of conditional commands
localnumberOfProcesses="${15:-2}"# Number of simultaneous commands to run in ParallExec mode
# Mandatory arguments
localmainInput="${1}"# Contains list of pids / commands separated by semicolons or filepath to list of pids / commands
# Optional arguments
localid="${2:-base}"# Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
localreadFromFile="${3:-false}"# Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
localsoftPerProcessTime="${4:-0}"# Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
localhardPerProcessTime="${5:-0}"# Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
localsoftMaxTime="${6:-0}"# Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
localhardMaxTime="${7:-0}"# Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
localcounting="${8:-true}"# Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
localsleepTime="${9:-.5}"# Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
localkeepLogging="${10:-1800}"# Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
localspinner="${11:-true}"# Show spinner (true) or do not show anything (false) while running
localnoTimeErrorLog="${12:-false}"# Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
localnoErrorLogsAtAll="${13:-false}"# Do not log any errros at all (useful for recursive ExecTasks checks)
# Parallelism specific arguments
localnumberOfProcesses="${14:-0}"# Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
localauxInput="${15}"# Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
localmaxPostponeRetries="${16:-3}"# If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
localminTimeBetweenRetries="${17:-300}"# Time (in seconds) between postponed command retries
localvalidExitCodes="${18:-0}"# Semi colon separated list of valid main command exit codes which will not trigger errors
local i
# Since ExecTasks takes up to 15 arguments, do a quick preflight check in DEBUG mode
# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
if["$_DEBUG"=="yes"];then
declare -a booleans=(counting spinner noTimeErrorLog noErrorLogAtAll)
for i in "${num_vars[@]}";do
test="if [ $i != false ] && [ $i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll)
for i in "${booleans[@]}";do
test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval"$test"
done
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging execTasksMode numberOfProcesses)
declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries)
for i in "${integers[@]}";do
test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
eval"$test"
done
fi
# Change '-' to '_' in task id
id="${id/-/_}"
# Expand validExitCodes into array
IFS=';'read -r -a validExitCodes <<<"$validExitCodes"
# ParallelExec specific variables
localauxItemCount=0# Number of conditional commands
# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
function _ExecTasksTimeCheck {
if[$spinner==true];then
Spinner
fi
@ -888,13 +860,13 @@ function ExecTasks {
fi
if[$keepLogging -ne 0];then
if[$((($exec_time +1)%$keepLogging)) -eq 0];then
if[$(((exec_time +1)% keepLogging)) -eq 0];then
if[$log_ttime -ne $exec_time];then# Fix when sleep time lower than 1 second
log_ttime=$exec_time
if[$functionMode=="Wait"];then
Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})].""NOTICE"
elif[$functionMode=="ParallelExec"];then
Logger "There are $((mainItemCount-counter)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})].""NOTICE"
Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})].""NOTICE"
deajan
6 years ago