Orchestrators echo/log audit pass

Status conclusions, clean-cycle summaries, skipped-feature lines, and phase
completion results → echo across all nine orchestrator scripts. Per-share,
per-job, and per-script detail lines inside loops remain log. README-Orchestrators:
added Output Tiers section.
This commit is contained in:
Gmer4Lfe
2026-05-21 17:25:29 -04:00
parent 3770bf80e7
commit ef8ef436cf
9 changed files with 43 additions and 23 deletions
+20
View File
@@ -142,6 +142,26 @@ Single notification → one bell per run, not one per job
--- ---
## ━━━ OUTPUT TIERS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
All scripts use a two-tier output model: `echo` lines are always visible; `log`
lines only appear when `--log` is passed.
**One-shot orchestrators** (`array_started.sh`, `array_stopping.sh`, `sunday_morning_coffee_report.sh`):
without `--log`, section headers, per-phase results, and the final summary are
visible. Per-item detail suppressed.
**Periodic orchestrators** (`critical_sync_maintenance.sh`, `daily_sync_maintenance.sh`,
`intermediate_sync_maintenance.sh`, `weekly_sync_maintenance.sh`): without `--log`,
phase headers, per-phase completion status, and the final summary are visible. Per-share
and per-job detail suppressed.
**Daemon orchestrators** (`watchdog_orchestrator.sh`, `transcode_management.sh`): silent
during clean cycles. Only state transitions, errors, and startup-grace expiry shown
without `--log`.
---
## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
| Script | What It Orchestrates | Schedule | | Script | What It Orchestrates | Schedule |
+1 -1
View File
@@ -212,6 +212,6 @@ elif [[ "$FAILED" -gt 0 ]]; then
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \ notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
"Array Start" "warning" "Array Start" "warning"
else else
log "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅" echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
fi fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+1 -1
View File
@@ -169,7 +169,7 @@ echo ""
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made" warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then elif [[ ${#FAILED[@]} -eq 0 ]]; then
log "$ICON_DONE Status: all $STEP step(s) complete ✅" echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
"Array Stop" "normal" "Array Stop" "normal"
else else
+5 -5
View File
@@ -120,12 +120,12 @@ PASS=()
FAIL=() FAIL=()
if ! check_rsync_enabled "CRITICAL"; then if ! check_rsync_enabled "CRITICAL"; then
log "Critical rsync disabled — skipping sync, running partnership check only" echo "Critical rsync disabled — skipping sync, running partnership check only"
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID" warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf" warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
else else
log "Critical sync — $MY_ID$REMOTE_ID$(date '+%H:%M:%S')" echo "Critical sync — $MY_ID$REMOTE_ID$(date '+%H:%M:%S')"
# Build dry-run flag to pass through # Build dry-run flag to pass through
RSYNC_DRY="" RSYNC_DRY=""
@@ -204,7 +204,7 @@ if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
--check --remote-unseen $PARTNER_DRY --check --remote-unseen $PARTNER_DRY
fi fi
else else
log "Partnership disabled — skipping check" echo "Partnership disabled — skipping check"
fi fi
# ============================================================================================== # ==============================================================================================
@@ -219,14 +219,14 @@ if [[ ${#FAIL[@]} -gt 0 ]]; then
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $DURATION" echo "$ICON_TIME Duration: $DURATION"
[[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}" [[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
echo "$ICON_ERROR Failed: ${FAIL[*]}" echo "$ICON_ERROR Failed: ${FAIL[*]}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
"Critical Sync" "warning" "Critical Sync" "warning"
exit 1 exit 1
else else
log "Critical sync complete — $MY_ID${DURATION}${#PASS[@]} share(s)" echo "Critical sync complete — $MY_ID${DURATION}${#PASS[@]} share(s)"
fi fi
exit 0 exit 0
+3 -3
View File
@@ -213,7 +213,7 @@ echo "━━━ $ICON_SYNC Arr Sync ━━━"
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh" ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
log "ARR_SYNC_ENABLED=false — skipping" echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping" warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
JOB_FAIL+=("arr_sync.sh") JOB_FAIL+=("arr_sync.sh")
@@ -221,7 +221,7 @@ else
_arr_sync_args=() _arr_sync_args=()
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run") [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
log "Arr sync complete ✅" echo "Arr sync complete ✅"
JOB_PASS+=("arr_sync.sh") JOB_PASS+=("arr_sync.sh")
else else
warn "Arr sync completed with errors — continuing to rsync" warn "Arr sync completed with errors — continuing to rsync"
@@ -354,7 +354,7 @@ if [[ "$TOTAL_FAIL" -gt 0 ]]; then
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 1 exit 1
else else
log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
exit 0 exit 0
fi fi
@@ -162,7 +162,7 @@ echo "━━━ $ICON_SYNC Arr Sync ━━━"
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh" ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
log "ARR_SYNC_ENABLED=false — skipping" echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping" warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
JOB_FAIL+=("arr_sync.sh") JOB_FAIL+=("arr_sync.sh")
@@ -170,7 +170,7 @@ else
_arr_sync_args=() _arr_sync_args=()
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run") [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
log "Arr sync complete ✅" echo "Arr sync complete ✅"
JOB_PASS+=("arr_sync.sh") JOB_PASS+=("arr_sync.sh")
else else
warn "Arr sync completed with errors — continuing" warn "Arr sync completed with errors — continuing"
@@ -192,8 +192,8 @@ SHARE_INDEX=0
ABORT_ALL_SYNCS=false ABORT_ALL_SYNCS=false
if [[ "$SHARE_COUNT" -eq 0 ]]; then if [[ "$SHARE_COUNT" -eq 0 ]]; then
log "No INTERMEDIATE_SYNC_SHARES configured — skipping" echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
log "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync" echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
elif ! check_rsync_enabled "INTERMEDIATE"; then elif ! check_rsync_enabled "INTERMEDIATE"; then
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)" warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
else else
@@ -300,7 +300,7 @@ TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made" warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
log "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced" echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
else else
warn "Status: $TOTAL_FAIL failure(s)" warn "Status: $TOTAL_FAIL failure(s)"
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
@@ -860,7 +860,7 @@ else
NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify" NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify"
if [[ -x "$NOTIFY_SCRIPT" ]]; then if [[ -x "$NOTIFY_SCRIPT" ]]; then
"$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null "$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null
log "unRAID notification sent" echo "unRAID notification sent"
fi fi
fi fi
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
@@ -871,8 +871,8 @@ else
PAYLOAD="{\"content\": ${ESCAPED_BODY}}" PAYLOAD="{\"content\": ${ESCAPED_BODY}}"
curl -sf -H "Content-Type: application/json" \ curl -sf -H "Content-Type: application/json" \
-d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \ -d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \
log "Discord notification sent" || \ echo "Discord notification sent" || \
warn "Discord notification failed" warn "Discord notification failed"
fi fi
log "Report generated — $MY_ID${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)" echo "Report generated — $MY_ID${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)"
fi fi
+1 -1
View File
@@ -87,7 +87,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)" warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
else else
log "Past startup grace — $(format_duration $UPTIME_S) uptime" echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
fi fi
echo "" echo ""
+4 -4
View File
@@ -231,7 +231,7 @@ if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
done done
fi fi
else else
log "WEEKLY_SYNC_UPDATES=false — skipping local updates" echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi fi
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
@@ -261,7 +261,7 @@ if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
done done
fi fi
else else
log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi fi
# ============================================================================================== # ==============================================================================================
@@ -352,7 +352,7 @@ else
_remainder_args=("--remainder") _remainder_args=("--remainder")
[[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run") [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
log "Remainder updates complete ✅" echo "Remainder updates complete ✅"
JOB_PASS+=("docker_update.sh --remainder") JOB_PASS+=("docker_update.sh --remainder")
else else
warn "Remainder updates completed with errors" warn "Remainder updates completed with errors"
@@ -392,7 +392,7 @@ echo ""
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made" warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
else else
warn "Status: $TOTAL_FAIL failure(s)" warn "Status: $TOTAL_FAIL failure(s)"
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \