audit echo vs log across all scripts — outcomes always visible, verbose for per-item loops
This commit is contained in:
Gmer4Lfe
+215
@@ -0,0 +1,215 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================ Watchdog Orchestrator ===========================================
|
||||
# ==============================================================================================
|
||||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
|
||||
# Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
|
||||
# Containers restarted into a resource-pressured system just fail again.
|
||||
# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
|
||||
# System Watchdog third — system component health after containers are healed.
|
||||
# API key renew fourth — self-heals unraid-api registry loss; check-first, silent when valid.
|
||||
# Stability Watchdog last — only reboots when all prior layers could not resolve the issue.
|
||||
#
|
||||
# ── ARRAY CHECK ───────────────────────────────────────────────────────────────────────────────
|
||||
# Exits immediately if /mnt/user is not mounted as shfs (array not started).
|
||||
# Watchdogs check Docker containers and storage — meaningless without the array.
|
||||
# Prevents false positives and unnecessary reboots when array is stopped or stopping.
|
||||
#
|
||||
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
|
||||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
|
||||
# Prevents false positives from containers still starting at array launch.
|
||||
# Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
|
||||
#
|
||||
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock() — exits immediately if a prior cycle is still in progress.
|
||||
# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||||
#
|
||||
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
|
||||
# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
|
||||
# Those scripts are now single-pass — this orchestrator provides the cadence.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_orchestrator.sh — normal run (called by cron every 15 minutes)
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
|
||||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Skip immediately if another cycle is still running — no pile-up
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||
|
||||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||||
_watchdog_display_name() {
|
||||
local path="$1"
|
||||
local base="${path##*/}"
|
||||
base="${base%.sh}"
|
||||
base="${base//_/ }"
|
||||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||||
else
|
||||
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Sub-scripts ──"
|
||||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||||
label="$(_watchdog_display_name "$entry")"
|
||||
if [[ -f "$local_path" ]]; then
|
||||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${local_path##*/}"
|
||||
else
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Schedule: */15 * * * * (every 15 minutes)"
|
||||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Array Check ━━━
|
||||
# ==============================================================================================
|
||||
if ! platform_storage_healthy; then
|
||||
echo "Array not started — skipping watchdog cycle"
|
||||
exit 0
|
||||
fi
|
||||
log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Startup Grace ━━━
|
||||
# ==============================================================================================
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
||||
exit 0
|
||||
fi
|
||||
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Watchdog Cycle ━━━
|
||||
# ==============================================================================================
|
||||
CYCLE_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
run_watchdog() {
|
||||
local name="$1" script="$2"
|
||||
|
||||
if [[ ! -f "$script" ]]; then
|
||||
error "$name — not found: $script"
|
||||
FAIL+=("$name:missing")
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ ! -x "$script" ]] && chmod +x "$script"
|
||||
|
||||
local extra_args=()
|
||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||
|
||||
local _ws
|
||||
_ws=$(date +%s)
|
||||
log "$ICON_START $name"
|
||||
if bash "$script" "${extra_args[@]}"; then
|
||||
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
|
||||
PASS+=("$name")
|
||||
return 0
|
||||
else
|
||||
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
|
||||
FAIL+=("$name")
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
||||
done
|
||||
|
||||
CYCLE_END=$(date +%s)
|
||||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Heartbeat ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
|
||||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||||
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — only shown on failures or --log ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
fi
|
||||
fi
|
||||
Reference in New Issue
Block a user