Orchestrators echo/log audit pass
Status conclusions, clean-cycle summaries, skipped-feature lines, and phase completion results → echo across all nine orchestrator scripts. Per-share, per-job, and per-script detail lines inside loops remain log. README-Orchestrators: added Output Tiers section.
This commit is contained in:
@@ -142,6 +142,26 @@ Single notification → one bell per run, not one per job
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## ━━━ OUTPUT TIERS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
|
||||||
|
All scripts use a two-tier output model: `echo` lines are always visible; `log`
|
||||||
|
lines only appear when `--log` is passed.
|
||||||
|
|
||||||
|
**One-shot orchestrators** (`array_started.sh`, `array_stopping.sh`, `sunday_morning_coffee_report.sh`):
|
||||||
|
without `--log`, section headers, per-phase results, and the final summary are
|
||||||
|
visible. Per-item detail suppressed.
|
||||||
|
|
||||||
|
**Periodic orchestrators** (`critical_sync_maintenance.sh`, `daily_sync_maintenance.sh`,
|
||||||
|
`intermediate_sync_maintenance.sh`, `weekly_sync_maintenance.sh`): without `--log`,
|
||||||
|
phase headers, per-phase completion status, and the final summary are visible. Per-share
|
||||||
|
and per-job detail suppressed.
|
||||||
|
|
||||||
|
**Daemon orchestrators** (`watchdog_orchestrator.sh`, `transcode_management.sh`): silent
|
||||||
|
during clean cycles. Only state transitions, errors, and startup-grace expiry shown
|
||||||
|
without `--log`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
## ━━━ SCRIPTS AT A GLANCE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
|
||||||
| Script | What It Orchestrates | Schedule |
|
| Script | What It Orchestrates | Schedule |
|
||||||
|
|||||||
@@ -212,6 +212,6 @@ elif [[ "$FAILED" -gt 0 ]]; then
|
|||||||
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
|
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
|
||||||
"Array Start" "warning"
|
"Array Start" "warning"
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
|
echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
@@ -169,7 +169,7 @@ echo ""
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||||
log "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||||
"Array Stop" "normal"
|
"Array Stop" "normal"
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -120,12 +120,12 @@ PASS=()
|
|||||||
FAIL=()
|
FAIL=()
|
||||||
|
|
||||||
if ! check_rsync_enabled "CRITICAL"; then
|
if ! check_rsync_enabled "CRITICAL"; then
|
||||||
log "Critical rsync disabled — skipping sync, running partnership check only"
|
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
||||||
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||||
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
||||||
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
|
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
|
||||||
else
|
else
|
||||||
log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
||||||
|
|
||||||
# Build dry-run flag to pass through
|
# Build dry-run flag to pass through
|
||||||
RSYNC_DRY=""
|
RSYNC_DRY=""
|
||||||
@@ -204,7 +204,7 @@ if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
|||||||
--check --remote-unseen $PARTNER_DRY
|
--check --remote-unseen $PARTNER_DRY
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
log "Partnership disabled — skipping check"
|
echo "Partnership disabled — skipping check"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -219,14 +219,14 @@ if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|||||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
echo "$ICON_TIME Duration: $DURATION"
|
echo "$ICON_TIME Duration: $DURATION"
|
||||||
[[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}"
|
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||||
"Critical Sync" "warning"
|
"Critical Sync" "warning"
|
||||||
exit 1
|
exit 1
|
||||||
else
|
else
|
||||||
log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
@@ -213,7 +213,7 @@ echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
|||||||
|
|
||||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||||
log "ARR_SYNC_ENABLED=false — skipping"
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||||
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
||||||
JOB_FAIL+=("arr_sync.sh")
|
JOB_FAIL+=("arr_sync.sh")
|
||||||
@@ -221,7 +221,7 @@ else
|
|||||||
_arr_sync_args=()
|
_arr_sync_args=()
|
||||||
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
||||||
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
||||||
log "Arr sync complete ✅"
|
echo "Arr sync complete ✅"
|
||||||
JOB_PASS+=("arr_sync.sh")
|
JOB_PASS+=("arr_sync.sh")
|
||||||
else
|
else
|
||||||
warn "Arr sync completed with errors — continuing to rsync"
|
warn "Arr sync completed with errors — continuing to rsync"
|
||||||
@@ -354,7 +354,7 @@ if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
|||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
exit 1
|
exit 1
|
||||||
else
|
else
|
||||||
log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
@@ -162,7 +162,7 @@ echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
|||||||
|
|
||||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
|
||||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||||
log "ARR_SYNC_ENABLED=false — skipping"
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||||
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
||||||
JOB_FAIL+=("arr_sync.sh")
|
JOB_FAIL+=("arr_sync.sh")
|
||||||
@@ -170,7 +170,7 @@ else
|
|||||||
_arr_sync_args=()
|
_arr_sync_args=()
|
||||||
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
||||||
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
||||||
log "Arr sync complete ✅"
|
echo "Arr sync complete ✅"
|
||||||
JOB_PASS+=("arr_sync.sh")
|
JOB_PASS+=("arr_sync.sh")
|
||||||
else
|
else
|
||||||
warn "Arr sync completed with errors — continuing"
|
warn "Arr sync completed with errors — continuing"
|
||||||
@@ -192,8 +192,8 @@ SHARE_INDEX=0
|
|||||||
ABORT_ALL_SYNCS=false
|
ABORT_ALL_SYNCS=false
|
||||||
|
|
||||||
if [[ "$SHARE_COUNT" -eq 0 ]]; then
|
if [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||||
log "No INTERMEDIATE_SYNC_SHARES configured — skipping"
|
echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
|
||||||
log "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
|
echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
|
||||||
elif ! check_rsync_enabled "INTERMEDIATE"; then
|
elif ! check_rsync_enabled "INTERMEDIATE"; then
|
||||||
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
|
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
|
||||||
else
|
else
|
||||||
@@ -300,7 +300,7 @@ TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||||
log "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||||
else
|
else
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
warn "Status: $TOTAL_FAIL failure(s)"
|
||||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||||
|
|||||||
@@ -860,7 +860,7 @@ else
|
|||||||
NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify"
|
NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify"
|
||||||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||||||
"$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null
|
"$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null
|
||||||
log "unRAID notification sent"
|
echo "unRAID notification sent"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
||||||
@@ -871,8 +871,8 @@ else
|
|||||||
PAYLOAD="{\"content\": ${ESCAPED_BODY}}"
|
PAYLOAD="{\"content\": ${ESCAPED_BODY}}"
|
||||||
curl -sf -H "Content-Type: application/json" \
|
curl -sf -H "Content-Type: application/json" \
|
||||||
-d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \
|
-d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \
|
||||||
log "Discord notification sent" || \
|
echo "Discord notification sent" || \
|
||||||
warn "Discord notification failed"
|
warn "Discord notification failed"
|
||||||
fi
|
fi
|
||||||
log "Report generated — $MY_ID — ${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)"
|
echo "Report generated — $MY_ID — ${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)"
|
||||||
fi
|
fi
|
||||||
@@ -87,7 +87,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||||||
else
|
else
|
||||||
log "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
@@ -231,7 +231,7 @@ if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
|||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
log "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||||
@@ -261,7 +261,7 @@ if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
|||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -352,7 +352,7 @@ else
|
|||||||
_remainder_args=("--remainder")
|
_remainder_args=("--remainder")
|
||||||
[[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run")
|
||||||
if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
|
if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then
|
||||||
log "Remainder updates complete ✅"
|
echo "Remainder updates complete ✅"
|
||||||
JOB_PASS+=("docker_update.sh --remainder")
|
JOB_PASS+=("docker_update.sh --remainder")
|
||||||
else
|
else
|
||||||
warn "Remainder updates completed with errors"
|
warn "Remainder updates completed with errors"
|
||||||
@@ -392,7 +392,7 @@ echo ""
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||||
log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||||
else
|
else
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
warn "Status: $TOTAL_FAIL failure(s)"
|
||||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||||
|
|||||||
Reference in New Issue
Block a user