Close every orchestrator the same way, and make skipped work a visible outcome
A gated-off section left nothing failed, so the weekly could run for hours and report "all complete" beside "0 shares synced"; skipped is now derived from what was expected rather than self-reported, and the verdict degrades to PARTIAL instead of flattering.
This commit is contained in:
Gmer4Lfe
@@ -266,16 +266,8 @@ echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no scripts launched"
|
||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# The configured list is the denominator — a script the conf names but that never launched is
|
||||
# skipped, not absent, and only shows up if something counts it.
|
||||
JOB_COUNT="${#ARRAY_START_SCRIPTS[@]}"
|
||||
orchestrator_summary "ARRAY START" "$START" "Array Start"
|
||||
exit $?
|
||||
@@ -190,18 +190,6 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Array Stop" "normal"
|
||||
else
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Stop" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
JOB_COUNT="$STEP"
|
||||
orchestrator_summary "ARRAY STOP" "$START" "Array Stop"
|
||||
exit $?
|
||||
|
||||
@@ -280,23 +280,7 @@ fi
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
# Standard ending, quiet mode — 30-min cadence, so a healthy cycle stays one line.
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
orchestrator_summary "CRITICAL SYNC" "$START" "Critical Sync" quiet
|
||||
exit $?
|
||||
@@ -354,11 +354,6 @@ WINDOW_END=$(date +%s)
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
@@ -382,16 +377,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Daily Maintenance" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
# Standard ending — derives skipped from SHARE_COUNT, so a run with rsync gated off reports
|
||||
# PARTIAL instead of "all complete".
|
||||
orchestrator_summary "DAILY MAINTENANCE" "$WINDOW_START" "Daily Maintenance"
|
||||
exit $?
|
||||
@@ -357,20 +357,8 @@ if [[ "$SHOW_FULL" == true ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
if [[ "$SHOW_FULL" == true ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||
else
|
||||
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
||||
fi
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Intermediate Sync" "warning"
|
||||
fi
|
||||
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
|
||||
# anything skipped or failed expands to the full block on its own.
|
||||
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
|
||||
exit $?
|
||||
|
||||
@@ -307,18 +307,8 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Monthly Maintenance" "normal"
|
||||
else
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Monthly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped
|
||||
# step visible rather than absent.
|
||||
JOB_COUNT="$STEP"
|
||||
orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance"
|
||||
exit $?
|
||||
|
||||
@@ -98,6 +98,9 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
# Timed from here so the standard summary can report a real duration; this report had none.
|
||||
REPORT_START=$(date +%s)
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -204,10 +207,9 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Sunday Morning Coffee Report" "warning"
|
||||
fi
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending. The configured section list is the denominator, so a report that quietly
|
||||
# stopped producing one of its sections reads as skipped rather than simply not appearing.
|
||||
JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}"
|
||||
[[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} ))
|
||||
orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report"
|
||||
exit $?
|
||||
|
||||
@@ -139,6 +139,10 @@ detect_hosts
|
||||
|
||||
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
||||
# from a healthy run. Fail loudly instead of silently doing no work.
|
||||
# This orchestrator never timed itself, so its summary could not report a duration. Set before
|
||||
# any work so the figure means the cycle, not the tail of it.
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
||||
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
||||
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
||||
@@ -223,15 +227,9 @@ done
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
|
||||
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||
else
|
||||
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
|
||||
if [[ "$DRY_RUN" != true ]]; then
|
||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Transcode Management" "warning"
|
||||
fi
|
||||
fi
|
||||
# Quiet by default — 7-min cadence. Anything failed or skipped expands on its own.
|
||||
JOB_COUNT="${#TRANSCODE_MANAGEMENT_SCRIPTS[@]}"
|
||||
orchestrator_summary "TRANSCODE CYCLE" "${CYCLE_START:-$(date +%s)}" "Transcode Management" quiet
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
|
||||
@@ -241,21 +241,15 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||
# ==============================================================================================
|
||||
# Per-script detail only when there is something to read; the standard block carries the rest.
|
||||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||||
fi
|
||||
|
||||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
# Quiet by default at a 15-min cadence. The configured script list is the denominator, so a
|
||||
# watchdog that silently stopped running one of its checks shows up as skipped.
|
||||
JOB_COUNT="${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||
orchestrator_summary "WATCHDOG CYCLE" "$CYCLE_START" "Watchdog Orchestrator" "$_mode"
|
||||
exit $?
|
||||
|
||||
@@ -402,18 +402,14 @@ WINDOW_END=$(date +%s)
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
# Per-unit detail first — the standard block that follows carries the verdict and the counts, not
|
||||
# the names, and knowing WHICH share failed is the whole point of reading a log.
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
@@ -422,19 +418,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section
|
||||
# can no longer read as success — this is the run that printed "all complete — 0 shares synced".
|
||||
orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance"
|
||||
exit $?
|
||||
@@ -3507,4 +3507,96 @@ require_partnership() {
|
||||
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
||||
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# ── Standardised orchestrator ending ─────────────────────────────────────────────────────────
|
||||
#
|
||||
# Every orchestrator closes with this, so a run's verdict reads the same everywhere and can be
|
||||
# parsed by one rule instead of ten.
|
||||
#
|
||||
# WHY IT EXISTS: on 2026-08-23 the Sunday weekly ran 2h43m, exited 0, and printed
|
||||
# "Status: all complete ✅ — 0 share(s) synced"
|
||||
# because its two sync jobs were SKIPPED, not failed — TOTAL_FAIL was 0, so it declared success.
|
||||
# Nothing in the old per-script summaries modelled work that was expected and never attempted, so
|
||||
# a gated-off section was indistinguishable from a clean run. Skipped is a first-class outcome here.
|
||||
#
|
||||
# VERDICTS — the headline degrades, never flatters:
|
||||
# DRY RUN nothing was changed
|
||||
# FAILED one or more units failed
|
||||
# PARTIAL nothing failed, but expected work was skipped <- the case that used to read OK
|
||||
# IDLE there was genuinely nothing to do
|
||||
# OK every expected unit ran and passed
|
||||
#
|
||||
# READS these globals if set, treating absent as empty — the names every orchestrator already uses:
|
||||
# PASS FAIL per-share outcomes SHARE_COUNT shares expected
|
||||
# JOB_PASS JOB_FAIL per-job outcomes JOB_COUNT jobs expected (optional)
|
||||
# DRY_RUN MY_ID LOCAL_SERVER_NAME
|
||||
#
|
||||
# ARGUMENTS
|
||||
# $1 display name for the run, e.g. "WEEKLY SYNC MAINTENANCE"
|
||||
# $2 start epoch
|
||||
# $3 optional: notification subject; omitted means do not notify
|
||||
#
|
||||
# RETURNS 0 for DRY RUN / IDLE / OK / PARTIAL, 1 for FAILED — so `exit $?` is the whole contract.
|
||||
# PARTIAL returns 0 deliberately: skipped work is usually a toggle the operator set on purpose, and
|
||||
# a non-zero exit would make cron mail every gated run. It is loud in the log, not in the exit code.
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# $4 "quiet" — for high-cadence runs (7/15 min). Prints ONLY the RESULT line when the verdict
|
||||
# is OK or IDLE, and the full block otherwise. A cycle that skipped or failed something is
|
||||
# never quiet, which is the only case anyone greps for anyway.
|
||||
orchestrator_summary() {
|
||||
local title="$1" start_ts="$2" subject="${3:-}" mode="${4:-full}"
|
||||
local end_ts; end_ts=$(date +%s)
|
||||
|
||||
local sp=${#PASS[@]} sf=${#FAIL[@]}
|
||||
local jp=${#JOB_PASS[@]} jf=${#JOB_FAIL[@]}
|
||||
local sc="${SHARE_COUNT:-$(( sp + sf ))}"
|
||||
local jc="${JOB_COUNT:-$(( jp + jf ))}"
|
||||
|
||||
# Skipped is derived, never reported by the caller — a section that bails early cannot be
|
||||
# relied on to remember to say so, which is exactly how this was missed for weeks.
|
||||
local ss=$(( sc - sp - sf )); [[ "$ss" -lt 0 ]] && ss=0
|
||||
local js=$(( jc - jp - jf )); [[ "$js" -lt 0 ]] && js=0
|
||||
|
||||
local total=$(( sc + jc )) fails=$(( sf + jf )) skips=$(( ss + js ))
|
||||
local verdict icon
|
||||
if [[ "${DRY_RUN:-false}" == true ]]; then verdict="DRY RUN"; icon="$ICON_WARN"
|
||||
elif [[ "$fails" -gt 0 ]]; then verdict="FAILED"; icon="$ICON_ERROR"
|
||||
elif [[ "$skips" -gt 0 ]]; then verdict="PARTIAL"; icon="$ICON_WARN"
|
||||
elif [[ "$total" -eq 0 ]]; then verdict="IDLE"; icon="$ICON_INFO"
|
||||
else verdict="OK"; icon="$ICON_DONE"
|
||||
fi
|
||||
|
||||
# Quiet cycles collapse to the one parseable line — but only when there is nothing to see.
|
||||
if [[ "$mode" == quiet && ( "$verdict" == OK || "$verdict" == IDLE ) ]]; then
|
||||
echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY $title — $verdict ━━━━━"
|
||||
echo "$ICON_HOST Identity: ${MY_ID:-unknown} (${LOCAL_SERVER_NAME:-$(hostname)})"
|
||||
echo "$ICON_TIME Window: $(date -d @"$start_ts" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$end_ts" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( end_ts - start_ts )))"
|
||||
[[ "$sc" -gt 0 ]] && echo "$ICON_SYNC Shares: $sp ok · $sf failed · $ss skipped (of $sc)"
|
||||
[[ "$jc" -gt 0 ]] && echo "$ICON_GEAR Jobs: $jp ok · $jf failed · $js skipped (of $jc)"
|
||||
|
||||
[[ "$sf" -gt 0 ]] && { echo "$ICON_ERROR Failed shares: ${FAIL[*]}"; }
|
||||
[[ "$jf" -gt 0 ]] && { echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"; }
|
||||
|
||||
# One machine-readable line, always last and always the same shape, so the board and any log
|
||||
# scraper have a single thing to match instead of ten prose variants.
|
||||
echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ -n "$subject" && "${DRY_RUN:-false}" != true ]]; then
|
||||
if [[ "$verdict" == "FAILED" ]]; then
|
||||
notify "$title FAILED on $(hostname) (${MY_ID:-?}) — $fails of $total unit(s) failed" "$subject" "warning"
|
||||
elif [[ "$verdict" == "PARTIAL" ]]; then
|
||||
notify "$title PARTIAL on $(hostname) (${MY_ID:-?}) — $skips of $total unit(s) skipped, none failed" "$subject" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$verdict" == "FAILED" ]] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user