Standardize orchestrator child-script execution and logging

Every orchestrator invoked its children differently — four near-duplicate
run_job() copies, a differently-shaped run_watchdog(), or plain inline bash
calls, each with its own take on path resolution, pass/fail naming, and
dry-run threading. Extracted one shared run_orch_child() into common.sh so
there's a single place to fix or extend this behavior going forward.

Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were
checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log
silently did nothing beyond basic logging on those two. Fixed to
$ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no
trailing exit, so their exit codes reflected whatever the last command
happened to return rather than actual success/failure. transcode_management.sh
had no failure notification and no summary at all. Also made
transcode_management.sh's two-script pipeline config-driven
(TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room
to extend it later without editing the orchestrator itself.
This commit is contained in:
Gmer4Lfe
2026-07-03 10:57:52 -04:00
parent 6fd22ae4ee
commit 2a062e5140
14 changed files with 303 additions and 370 deletions
+22 -33
View File
@@ -39,9 +39,12 @@
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root check — required for the docker/system reads used in the report
# acquire_lock — prevents overlapping weekly runs
# detect_hosts() — MY_ID in banner and summary
# Non-fatal steps — a failed script is logged; remaining scripts still run
# Flag pass-through — --dry-run and --log forwarded to all child scripts
# notify() on failure — pushed only outside --dry-run, matching the runtime-mode contract below
#
# ==============================================================================================
# CONFIGURATION
@@ -70,12 +73,22 @@
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
source "$ECOSYSTEM_ROOT/load_config.sh"
parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
acquire_lock
detect_hosts
# ==============================================================================================
@@ -84,35 +97,6 @@ detect_hosts
JOB_PASS=()
JOB_FAIL=()
run_job() {
local script_entry="$1"
local extra_dry=""
local extra_log=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
[[ "$ENABLE_LOGGING" == true ]] && extra_log="--log"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
return 1
fi
log "Running: $script_name ${extra_args[*]}"
if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then
echo "$script_name — done ✅"
JOB_PASS+=("$script_name")
else
error "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name")
fi
}
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
@@ -158,7 +142,7 @@ fi
for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue
echo ""
run_job "$script_entry"
run_orch_child "$script_entry"
done
# ==============================================================================================
@@ -176,5 +160,10 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
echo "❌ Failed: ${JOB_FAIL[*]}"
fi
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"Sunday Morning Coffee Report" "warning"
fi
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0