Close every orchestrator the same way, and make skipped work a visible outcome
A gated-off section left nothing failed, so the weekly could run for hours and report "all complete" beside "0 shares synced"; skipped is now derived from what was expected rather than self-reported, and the verdict degrades to PARTIAL instead of flattering.
This commit is contained in:
Gmer4Lfe
@@ -266,16 +266,8 @@ echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no scripts launched"
|
||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# The configured list is the denominator — a script the conf names but that never launched is
|
||||
# skipped, not absent, and only shows up if something counts it.
|
||||
JOB_COUNT="${#ARRAY_START_SCRIPTS[@]}"
|
||||
orchestrator_summary "ARRAY START" "$START" "Array Start"
|
||||
exit $?
|
||||
@@ -190,18 +190,6 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Array Stop" "normal"
|
||||
else
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Stop" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
JOB_COUNT="$STEP"
|
||||
orchestrator_summary "ARRAY STOP" "$START" "Array Stop"
|
||||
exit $?
|
||||
|
||||
@@ -280,23 +280,7 @@ fi
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
# Standard ending, quiet mode — 30-min cadence, so a healthy cycle stays one line.
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
orchestrator_summary "CRITICAL SYNC" "$START" "Critical Sync" quiet
|
||||
exit $?
|
||||
@@ -354,11 +354,6 @@ WINDOW_END=$(date +%s)
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
@@ -382,16 +377,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Daily Maintenance" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
# Standard ending — derives skipped from SHARE_COUNT, so a run with rsync gated off reports
|
||||
# PARTIAL instead of "all complete".
|
||||
orchestrator_summary "DAILY MAINTENANCE" "$WINDOW_START" "Daily Maintenance"
|
||||
exit $?
|
||||
@@ -357,20 +357,8 @@ if [[ "$SHOW_FULL" == true ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
if [[ "$SHOW_FULL" == true ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||
else
|
||||
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
||||
fi
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Intermediate Sync" "warning"
|
||||
fi
|
||||
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
|
||||
# anything skipped or failed expands to the full block on its own.
|
||||
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
|
||||
exit $?
|
||||
|
||||
@@ -307,18 +307,8 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Monthly Maintenance" "normal"
|
||||
else
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Monthly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped
|
||||
# step visible rather than absent.
|
||||
JOB_COUNT="$STEP"
|
||||
orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance"
|
||||
exit $?
|
||||
|
||||
@@ -98,6 +98,9 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
# Timed from here so the standard summary can report a real duration; this report had none.
|
||||
REPORT_START=$(date +%s)
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -204,10 +207,9 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Sunday Morning Coffee Report" "warning"
|
||||
fi
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending. The configured section list is the denominator, so a report that quietly
|
||||
# stopped producing one of its sections reads as skipped rather than simply not appearing.
|
||||
JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}"
|
||||
[[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} ))
|
||||
orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report"
|
||||
exit $?
|
||||
|
||||
@@ -139,6 +139,10 @@ detect_hosts
|
||||
|
||||
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
||||
# from a healthy run. Fail loudly instead of silently doing no work.
|
||||
# This orchestrator never timed itself, so its summary could not report a duration. Set before
|
||||
# any work so the figure means the cycle, not the tail of it.
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
||||
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
||||
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
||||
@@ -223,15 +227,9 @@ done
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
|
||||
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||
else
|
||||
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
|
||||
if [[ "$DRY_RUN" != true ]]; then
|
||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Transcode Management" "warning"
|
||||
fi
|
||||
fi
|
||||
# Quiet by default — 7-min cadence. Anything failed or skipped expands on its own.
|
||||
JOB_COUNT="${#TRANSCODE_MANAGEMENT_SCRIPTS[@]}"
|
||||
orchestrator_summary "TRANSCODE CYCLE" "${CYCLE_START:-$(date +%s)}" "Transcode Management" quiet
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
|
||||
@@ -241,21 +241,15 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||
# ==============================================================================================
|
||||
# Per-script detail only when there is something to read; the standard block carries the rest.
|
||||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||||
fi
|
||||
|
||||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
# Quiet by default at a 15-min cadence. The configured script list is the denominator, so a
|
||||
# watchdog that silently stopped running one of its checks shows up as skipped.
|
||||
JOB_COUNT="${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||
orchestrator_summary "WATCHDOG CYCLE" "$CYCLE_START" "Watchdog Orchestrator" "$_mode"
|
||||
exit $?
|
||||
|
||||
@@ -402,18 +402,14 @@ WINDOW_END=$(date +%s)
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
# Per-unit detail first — the standard block that follows carries the verdict and the counts, not
|
||||
# the names, and knowing WHICH share failed is the whole point of reading a log.
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
@@ -422,19 +418,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section
|
||||
# can no longer read as success — this is the run that printed "all complete — 0 shares synced".
|
||||
orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance"
|
||||
exit $?
|
||||
Reference in New Issue
Block a user