Standardize orchestrator child-script execution and logging

Every orchestrator invoked its children differently — four near-duplicate
run_job() copies, a differently-shaped run_watchdog(), or plain inline bash
calls, each with its own take on path resolution, pass/fail naming, and
dry-run threading. Extracted one shared run_orch_child() into common.sh so
there's a single place to fix or extend this behavior going forward.

Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were
checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log
silently did nothing beyond basic logging on those two. Fixed to
$ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no
trailing exit, so their exit codes reflected whatever the last command
happened to return rather than actual success/failure. transcode_management.sh
had no failure notification and no summary at all. Also made
transcode_management.sh's two-script pipeline config-driven
(TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room
to extend it later without editing the orchestrator itself.
This commit is contained in:
Gmer4Lfe
2026-07-03 10:57:52 -04:00
parent 6fd22ae4ee
commit 2a062e5140
14 changed files with 303 additions and 370 deletions
+18 -19
View File
@@ -163,9 +163,8 @@ log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
echo ""
START=$(date +%s)
LAUNCHED=0
FAILED=0
FAILED_SCRIPTS=()
JOB_PASS=()
JOB_FAIL=()
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
[[ -z "$relative_path" ]] && continue
@@ -177,8 +176,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
if [[ ! -f "$SCRIPT_PATH" ]]; then
error "$SCRIPT_NAME — not found"
error " Expected: $SCRIPT_PATH"
(( FAILED++ ))
FAILED_SCRIPTS+=("$SCRIPT_NAME")
JOB_FAIL+=("$SCRIPT_NAME")
continue
fi
@@ -187,15 +185,14 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
warn "$SCRIPT_NAME — not executable, fixing..."
chmod +x "$SCRIPT_PATH" || {
error "$SCRIPT_NAME — chmod +x failed"
(( FAILED++ ))
FAILED_SCRIPTS+=("$SCRIPT_NAME")
JOB_FAIL+=("$SCRIPT_NAME")
continue
}
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would launch: $SCRIPT_NAME"
(( LAUNCHED++ ))
JOB_PASS+=("$SCRIPT_NAME")
continue
fi
@@ -209,19 +206,18 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
if kill -0 "$PID" 2>/dev/null; then
# Still running → continuous script
log "$SCRIPT_NAME — running (PID $PID) ✅"
(( LAUNCHED++ ))
JOB_PASS+=("$SCRIPT_NAME")
else
# Exited — check if one-shot success or failure
wait "$PID"
EXIT_CODE=$?
if [[ "$EXIT_CODE" -eq 0 ]]; then
echo "$SCRIPT_NAME — completed (one-shot) ✅"
(( LAUNCHED++ ))
JOB_PASS+=("$SCRIPT_NAME")
else
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
error " Path: $SCRIPT_PATH"
(( FAILED++ ))
FAILED_SCRIPTS+=("$SCRIPT_NAME")
JOB_FAIL+=("$SCRIPT_NAME")
fi
fi
@@ -235,18 +231,21 @@ END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SUCCESS Launched: $LAUNCHED"
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED${FAILED_SCRIPTS[*]}"
echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]}${JOB_FAIL[*]}"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no scripts launched"
elif [[ "$FAILED" -gt 0 ]]; then
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
"Array Start" "warning"
else
echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0