Standardize orchestrator child-script execution and logging
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
This commit is contained in:
@@ -234,8 +234,8 @@ echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
PASSED=()
|
||||
FAILED=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
STEP=0
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -246,47 +246,11 @@ for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
(( STEP++ ))
|
||||
|
||||
read -r -a parts <<< "$entry"
|
||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
||||
script_name=$(basename "${parts[0]}")
|
||||
extra_args=("${parts[@]:1}")
|
||||
|
||||
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -x "$script_path" ]]; then
|
||||
warn "$script_name — not executable, fixing..."
|
||||
chmod +x "$script_path" || {
|
||||
error "$script_name — chmod +x failed"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
|
||||
PASSED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
local_args=()
|
||||
[[ "$VERBOSE" == true ]] && local_args+=("--log")
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
|
||||
echo "$script_name — done ✅"
|
||||
PASSED+=("$script_name")
|
||||
else
|
||||
warn "$script_name — failed (exit $?) — continuing to next step"
|
||||
FAILED+=("$script_name")
|
||||
fi
|
||||
|
||||
run_orch_child "$entry"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -307,22 +271,22 @@ fi
|
||||
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Monthly Maintenance" "normal"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Monthly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
|
||||
Reference in New Issue
Block a user