diff --git a/Orchestrators/array_started.sh b/Orchestrators/array_started.sh index 55da885..c182dc9 100755 --- a/Orchestrators/array_started.sh +++ b/Orchestrators/array_started.sh @@ -266,16 +266,8 @@ echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no scripts launched" -elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then - warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}" - notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \ - "Array Start" "warning" -else - echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 -exit 0 \ No newline at end of file +# The configured list is the denominator — a script the conf names but that never launched is +# skipped, not absent, and only shows up if something counts it. +JOB_COUNT="${#ARRAY_START_SCRIPTS[@]}" +orchestrator_summary "ARRAY START" "$START" "Array Start" +exit $? \ No newline at end of file diff --git a/Orchestrators/array_stopping.sh b/Orchestrators/array_stopping.sh index 79f0012..e01de1b 100755 --- a/Orchestrators/array_stopping.sh +++ b/Orchestrators/array_stopping.sh @@ -190,18 +190,6 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}" echo "" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no changes made" -elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then - echo "$ICON_DONE Status: all $STEP step(s) complete ✅" - notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ - "Array Stop" "normal" -else - warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}" - notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \ - "Array Stop" "warning" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 -exit 0 +JOB_COUNT="$STEP" +orchestrator_summary "ARRAY STOP" "$START" "Array Stop" +exit $? diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index dd67ddc..fb957f7 100755 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -280,23 +280,7 @@ fi END=$(date +%s) DURATION=$(format_duration $(( END - START ))) -TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) - -# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure. -if [[ "$TOTAL_FAIL" -gt 0 ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" - echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" - echo "$ICON_TIME Duration: $DURATION" - [[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}" - [[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}" - [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \ - "Critical Sync" "warning" - exit 1 -else - echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)" -fi - -exit 0 \ No newline at end of file +# Standard ending, quiet mode — 30-min cadence, so a healthy cycle stays one line. +[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}" +orchestrator_summary "CRITICAL SYNC" "$START" "Critical Sync" quiet +exit $? \ No newline at end of file diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 0edef84..bab1bec 100755 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -354,11 +354,6 @@ WINDOW_END=$(date +%s) # ━━━ Summary ━━━ # ============================================================================================== echo "" -echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━" -echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" -echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" -echo "" echo "$ICON_SYNC Shares ($SHARE_COUNT):" for entry in "${SHARE_TIMES[@]}"; do @@ -382,16 +377,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "" fi -TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) - -if [[ "$TOTAL_FAIL" -gt 0 ]]; then - warn "Status: $TOTAL_FAIL failure(s)" - notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ - "Daily Maintenance" "warning" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - exit 1 -else - echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - exit 0 -fi \ No newline at end of file +# Standard ending — derives skipped from SHARE_COUNT, so a run with rsync gated off reports +# PARTIAL instead of "all complete". +orchestrator_summary "DAILY MAINTENANCE" "$WINDOW_START" "Daily Maintenance" +exit $? \ No newline at end of file diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh index 87a4e31..881ef09 100755 --- a/Orchestrators/intermediate_sync_maintenance.sh +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -357,20 +357,8 @@ if [[ "$SHOW_FULL" == true ]]; then fi fi -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no changes made" -elif [[ "$TOTAL_FAIL" -eq 0 ]]; then - if [[ "$SHOW_FULL" == true ]]; then - echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced" - else - echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))" - fi -else - warn "Status: $TOTAL_FAIL failure(s)" - notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ - "Intermediate Sync" "warning" -fi -[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 -exit 0 +# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and +# anything skipped or failed expands to the full block on its own. +_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full +orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode" +exit $? diff --git a/Orchestrators/monthly_maintenance.sh b/Orchestrators/monthly_maintenance.sh index 579c16b..953dec6 100755 --- a/Orchestrators/monthly_maintenance.sh +++ b/Orchestrators/monthly_maintenance.sh @@ -307,18 +307,8 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}" echo "" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no changes made" -elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then - echo "$ICON_DONE Status: all $STEP step(s) complete ✅" - notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ - "Monthly Maintenance" "normal" -else - warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}" - notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \ - "Monthly Maintenance" "warning" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 -exit 0 +# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped +# step visible rather than absent. +JOB_COUNT="$STEP" +orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance" +exit $? diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 51919ad..a6dd504 100755 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -98,6 +98,9 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$ECOSYSTEM_ROOT/load_config.sh" +# Timed from here so the standard summary can report a real duration; this report had none. +REPORT_START=$(date +%s) + parse_args "$@" # ============================================================================================== @@ -204,10 +207,9 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then echo "❌ Failed: ${JOB_FAIL[*]}" fi -if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then - notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ - "Sunday Morning Coffee Report" "warning" -fi - -[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 -exit 0 +# Standard ending. The configured section list is the denominator, so a report that quietly +# stopped producing one of its sections reads as skipped rather than simply not appearing. +JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}" +[[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} )) +orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report" +exit $? diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh index d3187d7..773d7dd 100755 --- a/Orchestrators/transcode_management.sh +++ b/Orchestrators/transcode_management.sh @@ -139,6 +139,10 @@ detect_hosts # An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable # from a healthy run. Fail loudly instead of silently doing no work. +# This orchestrator never timed itself, so its summary could not report a duration. Set before +# any work so the figure means the cycle, not the tail of it. +CYCLE_START=$(date +%s) + if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run" error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf" @@ -223,15 +227,9 @@ done # ============================================================================================== # ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━ # ============================================================================================== -if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then - echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed" -else - error "Transcode cycle — failed: ${JOB_FAIL[*]}" - if [[ "$DRY_RUN" != true ]]; then - notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ - "Transcode Management" "warning" - fi -fi +# Quiet by default — 7-min cadence. Anything failed or skipped expands on its own. +JOB_COUNT="${#TRANSCODE_MANAGEMENT_SCRIPTS[@]}" +orchestrator_summary "TRANSCODE CYCLE" "${CYCLE_START:-$(date +%s)}" "Transcode Management" quiet # ============================================================================================== # ━━━ Exit ━━━ diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index bb0a74a..762cdf8 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -241,21 +241,15 @@ fi # ============================================================================================== # ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━ # ============================================================================================== +# Per-script detail only when there is something to read; the standard block carries the rest. if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━" for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done - echo "$ICON_TIME Duration: $(format_duration $DURATION)" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -else - echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))" fi -if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then - notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ - "Watchdog Orchestrator" "warning" - exit 1 -fi - -exit 0 +# Quiet by default at a 15-min cadence. The configured script list is the denominator, so a +# watchdog that silently stopped running one of its checks shows up as skipped. +JOB_COUNT="${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}" +_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full +orchestrator_summary "WATCHDOG CYCLE" "$CYCLE_START" "Watchdog Orchestrator" "$_mode" +exit $? diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index fc610f8..cea5916 100755 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -402,18 +402,14 @@ WINDOW_END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== +# Per-unit detail first — the standard block that follows carries the verdict and the counts, not +# the names, and knowing WHICH share failed is the whole point of reading a log. echo "" -echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" -echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" -echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}" echo "" - echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):" for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done -echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "" @@ -422,19 +418,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done fi -TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) - -echo "" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — no changes made" -elif [[ "$TOTAL_FAIL" -eq 0 ]]; then - echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" -else - warn "Status: $TOTAL_FAIL failure(s)" - notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ - "Weekly Maintenance" "warning" -fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - -[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 -exit 0 \ No newline at end of file +# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section +# can no longer read as success — this is the run that printed "all complete — 0 shares synced". +orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance" +exit $? \ No newline at end of file diff --git a/common.sh b/common.sh index ad2847d..5840b49 100755 --- a/common.sh +++ b/common.sh @@ -3507,4 +3507,96 @@ require_partnership() { [[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0 log "PARTNERSHIP_ENABLED=false — skipping cross-server operation" exit 0 -} \ No newline at end of file +} +# ══════════════════════════════════════════════════════════════════════════════════════════════ +# ── Standardised orchestrator ending ───────────────────────────────────────────────────────── +# +# Every orchestrator closes with this, so a run's verdict reads the same everywhere and can be +# parsed by one rule instead of ten. +# +# WHY IT EXISTS: on 2026-08-23 the Sunday weekly ran 2h43m, exited 0, and printed +# "Status: all complete ✅ — 0 share(s) synced" +# because its two sync jobs were SKIPPED, not failed — TOTAL_FAIL was 0, so it declared success. +# Nothing in the old per-script summaries modelled work that was expected and never attempted, so +# a gated-off section was indistinguishable from a clean run. Skipped is a first-class outcome here. +# +# VERDICTS — the headline degrades, never flatters: +# DRY RUN nothing was changed +# FAILED one or more units failed +# PARTIAL nothing failed, but expected work was skipped <- the case that used to read OK +# IDLE there was genuinely nothing to do +# OK every expected unit ran and passed +# +# READS these globals if set, treating absent as empty — the names every orchestrator already uses: +# PASS FAIL per-share outcomes SHARE_COUNT shares expected +# JOB_PASS JOB_FAIL per-job outcomes JOB_COUNT jobs expected (optional) +# DRY_RUN MY_ID LOCAL_SERVER_NAME +# +# ARGUMENTS +# $1 display name for the run, e.g. "WEEKLY SYNC MAINTENANCE" +# $2 start epoch +# $3 optional: notification subject; omitted means do not notify +# +# RETURNS 0 for DRY RUN / IDLE / OK / PARTIAL, 1 for FAILED — so `exit $?` is the whole contract. +# PARTIAL returns 0 deliberately: skipped work is usually a toggle the operator set on purpose, and +# a non-zero exit would make cron mail every gated run. It is loud in the log, not in the exit code. +# ══════════════════════════════════════════════════════════════════════════════════════════════ +# $4 "quiet" — for high-cadence runs (7/15 min). Prints ONLY the RESULT line when the verdict +# is OK or IDLE, and the full block otherwise. A cycle that skipped or failed something is +# never quiet, which is the only case anyone greps for anyway. +orchestrator_summary() { + local title="$1" start_ts="$2" subject="${3:-}" mode="${4:-full}" + local end_ts; end_ts=$(date +%s) + + local sp=${#PASS[@]} sf=${#FAIL[@]} + local jp=${#JOB_PASS[@]} jf=${#JOB_FAIL[@]} + local sc="${SHARE_COUNT:-$(( sp + sf ))}" + local jc="${JOB_COUNT:-$(( jp + jf ))}" + + # Skipped is derived, never reported by the caller — a section that bails early cannot be + # relied on to remember to say so, which is exactly how this was missed for weeks. + local ss=$(( sc - sp - sf )); [[ "$ss" -lt 0 ]] && ss=0 + local js=$(( jc - jp - jf )); [[ "$js" -lt 0 ]] && js=0 + + local total=$(( sc + jc )) fails=$(( sf + jf )) skips=$(( ss + js )) + local verdict icon + if [[ "${DRY_RUN:-false}" == true ]]; then verdict="DRY RUN"; icon="$ICON_WARN" + elif [[ "$fails" -gt 0 ]]; then verdict="FAILED"; icon="$ICON_ERROR" + elif [[ "$skips" -gt 0 ]]; then verdict="PARTIAL"; icon="$ICON_WARN" + elif [[ "$total" -eq 0 ]]; then verdict="IDLE"; icon="$ICON_INFO" + else verdict="OK"; icon="$ICON_DONE" + fi + + # Quiet cycles collapse to the one parseable line — but only when there is nothing to see. + if [[ "$mode" == quiet && ( "$verdict" == OK || "$verdict" == IDLE ) ]]; then + echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s" + return 0 + fi + + echo "" + echo "━━━━━ $ICON_SUMMARY $title — $verdict ━━━━━" + echo "$ICON_HOST Identity: ${MY_ID:-unknown} (${LOCAL_SERVER_NAME:-$(hostname)})" + echo "$ICON_TIME Window: $(date -d @"$start_ts" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$end_ts" '+%H:%M:%S')" + echo "$ICON_TIME Duration: $(format_duration $(( end_ts - start_ts )))" + [[ "$sc" -gt 0 ]] && echo "$ICON_SYNC Shares: $sp ok · $sf failed · $ss skipped (of $sc)" + [[ "$jc" -gt 0 ]] && echo "$ICON_GEAR Jobs: $jp ok · $jf failed · $js skipped (of $jc)" + + [[ "$sf" -gt 0 ]] && { echo "$ICON_ERROR Failed shares: ${FAIL[*]}"; } + [[ "$jf" -gt 0 ]] && { echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"; } + + # One machine-readable line, always last and always the same shape, so the board and any log + # scraper have a single thing to match instead of ten prose variants. + echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + if [[ -n "$subject" && "${DRY_RUN:-false}" != true ]]; then + if [[ "$verdict" == "FAILED" ]]; then + notify "$title FAILED on $(hostname) (${MY_ID:-?}) — $fails of $total unit(s) failed" "$subject" "warning" + elif [[ "$verdict" == "PARTIAL" ]]; then + notify "$title PARTIAL on $(hostname) (${MY_ID:-?}) — $skips of $total unit(s) skipped, none failed" "$subject" "warning" + fi + fi + + [[ "$verdict" == "FAILED" ]] && return 1 + return 0 +}