Close every orchestrator the same way, and make skipped work a visible outcome
A gated-off section left nothing failed, so the weekly could run for hours and report "all complete" beside "0 shares synced"; skipped is now derived from what was expected rather than self-reported, and the verdict degrades to PARTIAL instead of flattering.
This commit is contained in:
Gmer4Lfe
@@ -3507,4 +3507,96 @@ require_partnership() {
|
||||
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
||||
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
||||
exit 0
|
||||
}
|
||||
}
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# ── Standardised orchestrator ending ─────────────────────────────────────────────────────────
|
||||
#
|
||||
# Every orchestrator closes with this, so a run's verdict reads the same everywhere and can be
|
||||
# parsed by one rule instead of ten.
|
||||
#
|
||||
# WHY IT EXISTS: on 2026-08-23 the Sunday weekly ran 2h43m, exited 0, and printed
|
||||
# "Status: all complete ✅ — 0 share(s) synced"
|
||||
# because its two sync jobs were SKIPPED, not failed — TOTAL_FAIL was 0, so it declared success.
|
||||
# Nothing in the old per-script summaries modelled work that was expected and never attempted, so
|
||||
# a gated-off section was indistinguishable from a clean run. Skipped is a first-class outcome here.
|
||||
#
|
||||
# VERDICTS — the headline degrades, never flatters:
|
||||
# DRY RUN nothing was changed
|
||||
# FAILED one or more units failed
|
||||
# PARTIAL nothing failed, but expected work was skipped <- the case that used to read OK
|
||||
# IDLE there was genuinely nothing to do
|
||||
# OK every expected unit ran and passed
|
||||
#
|
||||
# READS these globals if set, treating absent as empty — the names every orchestrator already uses:
|
||||
# PASS FAIL per-share outcomes SHARE_COUNT shares expected
|
||||
# JOB_PASS JOB_FAIL per-job outcomes JOB_COUNT jobs expected (optional)
|
||||
# DRY_RUN MY_ID LOCAL_SERVER_NAME
|
||||
#
|
||||
# ARGUMENTS
|
||||
# $1 display name for the run, e.g. "WEEKLY SYNC MAINTENANCE"
|
||||
# $2 start epoch
|
||||
# $3 optional: notification subject; omitted means do not notify
|
||||
#
|
||||
# RETURNS 0 for DRY RUN / IDLE / OK / PARTIAL, 1 for FAILED — so `exit $?` is the whole contract.
|
||||
# PARTIAL returns 0 deliberately: skipped work is usually a toggle the operator set on purpose, and
|
||||
# a non-zero exit would make cron mail every gated run. It is loud in the log, not in the exit code.
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# $4 "quiet" — for high-cadence runs (7/15 min). Prints ONLY the RESULT line when the verdict
|
||||
# is OK or IDLE, and the full block otherwise. A cycle that skipped or failed something is
|
||||
# never quiet, which is the only case anyone greps for anyway.
|
||||
orchestrator_summary() {
|
||||
local title="$1" start_ts="$2" subject="${3:-}" mode="${4:-full}"
|
||||
local end_ts; end_ts=$(date +%s)
|
||||
|
||||
local sp=${#PASS[@]} sf=${#FAIL[@]}
|
||||
local jp=${#JOB_PASS[@]} jf=${#JOB_FAIL[@]}
|
||||
local sc="${SHARE_COUNT:-$(( sp + sf ))}"
|
||||
local jc="${JOB_COUNT:-$(( jp + jf ))}"
|
||||
|
||||
# Skipped is derived, never reported by the caller — a section that bails early cannot be
|
||||
# relied on to remember to say so, which is exactly how this was missed for weeks.
|
||||
local ss=$(( sc - sp - sf )); [[ "$ss" -lt 0 ]] && ss=0
|
||||
local js=$(( jc - jp - jf )); [[ "$js" -lt 0 ]] && js=0
|
||||
|
||||
local total=$(( sc + jc )) fails=$(( sf + jf )) skips=$(( ss + js ))
|
||||
local verdict icon
|
||||
if [[ "${DRY_RUN:-false}" == true ]]; then verdict="DRY RUN"; icon="$ICON_WARN"
|
||||
elif [[ "$fails" -gt 0 ]]; then verdict="FAILED"; icon="$ICON_ERROR"
|
||||
elif [[ "$skips" -gt 0 ]]; then verdict="PARTIAL"; icon="$ICON_WARN"
|
||||
elif [[ "$total" -eq 0 ]]; then verdict="IDLE"; icon="$ICON_INFO"
|
||||
else verdict="OK"; icon="$ICON_DONE"
|
||||
fi
|
||||
|
||||
# Quiet cycles collapse to the one parseable line — but only when there is nothing to see.
|
||||
if [[ "$mode" == quiet && ( "$verdict" == OK || "$verdict" == IDLE ) ]]; then
|
||||
echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY $title — $verdict ━━━━━"
|
||||
echo "$ICON_HOST Identity: ${MY_ID:-unknown} (${LOCAL_SERVER_NAME:-$(hostname)})"
|
||||
echo "$ICON_TIME Window: $(date -d @"$start_ts" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$end_ts" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( end_ts - start_ts )))"
|
||||
[[ "$sc" -gt 0 ]] && echo "$ICON_SYNC Shares: $sp ok · $sf failed · $ss skipped (of $sc)"
|
||||
[[ "$jc" -gt 0 ]] && echo "$ICON_GEAR Jobs: $jp ok · $jf failed · $js skipped (of $jc)"
|
||||
|
||||
[[ "$sf" -gt 0 ]] && { echo "$ICON_ERROR Failed shares: ${FAIL[*]}"; }
|
||||
[[ "$jf" -gt 0 ]] && { echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"; }
|
||||
|
||||
# One machine-readable line, always last and always the same shape, so the board and any log
|
||||
# scraper have a single thing to match instead of ten prose variants.
|
||||
echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ -n "$subject" && "${DRY_RUN:-false}" != true ]]; then
|
||||
if [[ "$verdict" == "FAILED" ]]; then
|
||||
notify "$title FAILED on $(hostname) (${MY_ID:-?}) — $fails of $total unit(s) failed" "$subject" "warning"
|
||||
elif [[ "$verdict" == "PARTIAL" ]]; then
|
||||
notify "$title PARTIAL on $(hostname) (${MY_ID:-?}) — $skips of $total unit(s) skipped, none failed" "$subject" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$verdict" == "FAILED" ]] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user