208 lines
11 KiB
Bash
Executable File
208 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================ Watchdog Orchestrator ===========================================
|
||
# ==============================================================================================
|
||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||
# Schedule: */15 * * * * (every 15 minutes)
|
||
#
|
||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||
# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
|
||
# Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog
|
||
#
|
||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||
# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
|
||
# Containers restarted into a resource-pressured system just fail again.
|
||
# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
|
||
# System Watchdog third — system component health after containers are healed.
|
||
# API key renew fourth — self-heals unraid-api registry loss; check-first, silent when valid.
|
||
# Stability Watchdog last — only reboots when all prior layers could not resolve the issue.
|
||
#
|
||
# ── ARRAY CHECK ───────────────────────────────────────────────────────────────────────────────
|
||
# Exits immediately if /mnt/user is not mounted as shfs (array not started).
|
||
# Watchdogs check Docker containers and storage — meaningless without the array.
|
||
# Prevents false positives and unnecessary reboots when array is stopped or stopping.
|
||
#
|
||
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
|
||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
|
||
# Prevents false positives from containers still starting at array launch.
|
||
# Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
|
||
#
|
||
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
|
||
# acquire_lock() — exits immediately if a prior cycle is still in progress.
|
||
# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||
#
|
||
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
|
||
# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
|
||
# Those scripts are now single-pass — this orchestrator provides the cadence.
|
||
#
|
||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||
#
|
||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||
# watchdog_orchestrator.sh — normal run (called by cron every 15 minutes)
|
||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
|
||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
|
||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# Skip immediately if another cycle is still running — no pile-up
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||
|
||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||
_watchdog_display_name() {
|
||
local path="$1"
|
||
local base="${path##*/}"
|
||
base="${base%.sh}"
|
||
base="${base//_/ }"
|
||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
|
||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||
else
|
||
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||
fi
|
||
|
||
echo ""
|
||
echo "── Sub-scripts ──"
|
||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||
label="$(_watchdog_display_name "$entry")"
|
||
if [[ -f "$local_path" ]]; then
|
||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||
echo " $icon $label — ${local_path##*/}"
|
||
else
|
||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo " Schedule: */15 * * * * (every 15 minutes)"
|
||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Array Check ━━━
|
||
# ==============================================================================================
|
||
if ! df --output=fstype /mnt/user 2>/dev/null | grep -q shfs; then
|
||
log "Array not started — skipping watchdog cycle"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Startup Grace ━━━
|
||
# ==============================================================================================
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
log "Startup grace — ${UPTIME_SECONDS}s / ${WATCHDOG_STARTUP_GRACE}s — skipping cycle"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Run Watchdog Cycle ━━━
|
||
# ==============================================================================================
|
||
CYCLE_START=$(date +%s)
|
||
PASS=()
|
||
FAIL=()
|
||
|
||
run_watchdog() {
|
||
local name="$1" script="$2"
|
||
|
||
if [[ ! -f "$script" ]]; then
|
||
error "$name — not found: $script"
|
||
FAIL+=("$name:missing")
|
||
return 1
|
||
fi
|
||
|
||
[[ ! -x "$script" ]] && chmod +x "$script"
|
||
|
||
local extra_args=()
|
||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||
|
||
log "$ICON_START $name"
|
||
if bash "$script" "${extra_args[@]}"; then
|
||
PASS+=("$name")
|
||
return 0
|
||
else
|
||
error "$name — non-zero exit"
|
||
FAIL+=("$name")
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
||
done
|
||
|
||
CYCLE_END=$(date +%s)
|
||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Heartbeat ━━━
|
||
# ==============================================================================================
|
||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||
HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
|
||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary — only shown on failures or --log ━━━
|
||
# ==============================================================================================
|
||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||
"Watchdog Orchestrator" "warning"
|
||
fi
|
||
fi
|