229 lines
10 KiB
Bash
Executable File
229 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================ Watchdog Orchestrator ===========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. Replaces the
|
||
# continuous loops previously embedded in individual watchdog scripts — those
|
||
# are now single-pass; this orchestrator provides the cadence.
|
||
# Schedule: */15 * * * * (every 15 minutes)
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Order driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf.
|
||
# Default: resource_watchdog → docker_watchdog → system_watchdog →
|
||
# unraid_api_key_renew → stability_watchdog
|
||
#
|
||
# ARRAY CHECK
|
||
# Exits immediately if /mnt/user is not mounted as shfs. Watchdogs check
|
||
# Docker containers and storage — meaningless without the array. Prevents
|
||
# false positives and unnecessary reboots when array is stopped or stopping.
|
||
#
|
||
# STARTUP GRACE
|
||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. Prevents
|
||
# false positives from containers still starting at array launch. Each
|
||
# sub-script enforces this independently — orchestrator exits early to avoid
|
||
# log noise.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Pressure Before Healing
|
||
# Resource Watchdog runs first — it frees RAM and CPU before any container
|
||
# restart is attempted. Containers restarted into a resource-pressured system
|
||
# just fail again. Docker Watchdog restarts with pressure already reduced.
|
||
# System Watchdog checks component health after containers are healed.
|
||
# Stability Watchdog reboots only when all prior layers could not resolve the
|
||
# issue. API key renew is check-first and silent when valid.
|
||
#
|
||
# Never Queue
|
||
# acquire_lock exits immediately if a prior cycle is still running. Prevents
|
||
# pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root check — watchdog operations require root
|
||
# acquire_lock — strict; no pile-up if prior cycle still active
|
||
# detect_hosts() — MY_ID in logs and notifications
|
||
# Array check — exits early if /mnt/user is not shfs-mounted
|
||
# Startup grace — WATCHDOG_STARTUP_GRACE respected before any checks
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# watchdog_orchestrator.sh
|
||
# Normal run (called by cron every 15 minutes).
|
||
#
|
||
# watchdog_orchestrator.sh --dry-run
|
||
# Pass --dry-run to all sub-scripts.
|
||
#
|
||
# watchdog_orchestrator.sh --status
|
||
# Show script paths and current grace state.
|
||
#
|
||
# watchdog_orchestrator.sh --log
|
||
# Verbose output from all sub-scripts.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
|
||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# Skip immediately if another cycle is still running — no pile-up
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||
|
||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||
_watchdog_display_name() {
|
||
local path="$1"
|
||
local base="${path##*/}"
|
||
base="${base%.sh}"
|
||
base="${base//_/ }"
|
||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
|
||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||
else
|
||
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||
fi
|
||
|
||
echo ""
|
||
echo "── Sub-scripts ──"
|
||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||
label="$(_watchdog_display_name "$entry")"
|
||
if [[ -f "$local_path" ]]; then
|
||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||
echo " $icon $label — ${local_path##*/}"
|
||
else
|
||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo " Schedule: */15 * * * * (every 15 minutes)"
|
||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Array Check ━━━
|
||
# ==============================================================================================
|
||
if ! platform_storage_healthy; then
|
||
echo "Array not started — skipping watchdog cycle"
|
||
exit 0
|
||
fi
|
||
log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Startup Grace ━━━
|
||
# ==============================================================================================
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
||
exit 0
|
||
fi
|
||
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Run Watchdog Cycle ━━━
|
||
# ==============================================================================================
|
||
CYCLE_START=$(date +%s)
|
||
JOB_PASS=()
|
||
JOB_FAIL=()
|
||
|
||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
[[ -z "$_entry" ]] && continue
|
||
run_orch_child "$_entry"
|
||
done
|
||
|
||
CYCLE_END=$(date +%s)
|
||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Heartbeat ━━━
|
||
# ==============================================================================================
|
||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||
HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
|
||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||
log "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||
# ==============================================================================================
|
||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
else
|
||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||
fi
|
||
|
||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||
"Watchdog Orchestrator" "warning"
|
||
exit 1
|
||
fi
|
||
|
||
exit 0
|