Standardize orchestrator child-script execution and logging
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
This commit is contained in:
@@ -163,9 +163,8 @@ log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
LAUNCHED=0
|
||||
FAILED=0
|
||||
FAILED_SCRIPTS=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
[[ -z "$relative_path" ]] && continue
|
||||
@@ -177,8 +176,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
error "$SCRIPT_NAME — not found"
|
||||
error " Expected: $SCRIPT_PATH"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
JOB_FAIL+=("$SCRIPT_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -187,15 +185,14 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
warn "$SCRIPT_NAME — not executable, fixing..."
|
||||
chmod +x "$SCRIPT_PATH" || {
|
||||
error "$SCRIPT_NAME — chmod +x failed"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
JOB_FAIL+=("$SCRIPT_NAME")
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would launch: $SCRIPT_NAME"
|
||||
(( LAUNCHED++ ))
|
||||
JOB_PASS+=("$SCRIPT_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -209,19 +206,18 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
# Still running → continuous script
|
||||
log "$SCRIPT_NAME — running (PID $PID) ✅"
|
||||
(( LAUNCHED++ ))
|
||||
JOB_PASS+=("$SCRIPT_NAME")
|
||||
else
|
||||
# Exited — check if one-shot success or failure
|
||||
wait "$PID"
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
echo "$SCRIPT_NAME — completed (one-shot) ✅"
|
||||
(( LAUNCHED++ ))
|
||||
JOB_PASS+=("$SCRIPT_NAME")
|
||||
else
|
||||
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
||||
error " Path: $SCRIPT_PATH"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
JOB_FAIL+=("$SCRIPT_NAME")
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -235,18 +231,21 @@ END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SUCCESS Launched: $LAUNCHED"
|
||||
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}"
|
||||
echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]} — ${JOB_FAIL[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no scripts launched"
|
||||
elif [[ "$FAILED" -gt 0 ]]; then
|
||||
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
|
||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
|
||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user