Standardize orchestrator child-script execution and logging
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
This commit is contained in:
@@ -39,9 +39,12 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root check — required for the docker/system reads used in the report
|
||||
# acquire_lock — prevents overlapping weekly runs
|
||||
# detect_hosts() — MY_ID in banner and summary
|
||||
# Non-fatal steps — a failed script is logged; remaining scripts still run
|
||||
# Flag pass-through — --dry-run and --log forwarded to all child scripts
|
||||
# notify() on failure — pushed only outside --dry-run, matching the runtime-mode contract below
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -70,12 +73,22 @@
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -84,35 +97,6 @@ detect_hosts
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
local extra_log=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && extra_log="--log"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then
|
||||
echo "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -158,7 +142,7 @@ fi
|
||||
for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -176,5 +160,10 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Sunday Morning Coffee Report" "warning"
|
||||
fi
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user