Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
262 lines
12 KiB
Bash
Executable File
262 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================ Watchdog Orchestrator ===========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. Replaces the
|
||
# continuous loops previously embedded in individual watchdog scripts — those
|
||
# are now single-pass; this orchestrator provides the cadence.
|
||
# Schedule: */15 * * * * (every 15 minutes)
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Order driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf.
|
||
# Default: resource_watchdog → docker_watchdog → system_watchdog →
|
||
# unraid_api_key_renew → stability_watchdog
|
||
#
|
||
# ARRAY CHECK
|
||
# Exits immediately if /mnt/user is not mounted as shfs. Watchdogs check
|
||
# Docker containers and storage — meaningless without the array. Prevents
|
||
# false positives and unnecessary reboots when array is stopped or stopping.
|
||
#
|
||
# STARTUP GRACE
|
||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. Prevents
|
||
# false positives from containers still starting at array launch. Each
|
||
# sub-script enforces this independently — orchestrator exits early to avoid
|
||
# log noise.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Pressure Before Healing
|
||
# Resource Watchdog runs first — it frees RAM and CPU before any container
|
||
# restart is attempted. Containers restarted into a resource-pressured system
|
||
# just fail again. Docker Watchdog restarts with pressure already reduced.
|
||
# System Watchdog checks component health after containers are healed.
|
||
# Stability Watchdog reboots only when all prior layers could not resolve the
|
||
# issue. API key renew is check-first and silent when valid.
|
||
#
|
||
# Never Queue
|
||
# acquire_lock exits immediately if a prior cycle is still running. Prevents
|
||
# pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Enforcement
|
||
# Every watchdog launched from here requires root. Failing once at the top gives one
|
||
# clear error instead of the same permission failure repeated per child.
|
||
#
|
||
# Lock Acquisition
|
||
# acquire_lock in strict mode — if the previous cycle is still running, this one exits
|
||
# rather than queuing. At a 15-minute cadence a waiting lock would pile up cycles
|
||
# behind a slow watchdog and eventually run them all at once.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() sets MY_ID for logs and notifications.
|
||
#
|
||
# Empty Job List Guard
|
||
# Exits with an error and a notification if WATCHDOG_ORCHESTRATOR_SCRIPTS is empty.
|
||
# Without it the cycle reports "0/0 passed" and exits 0 every cycle — indistinguishable
|
||
# from a healthy run, while nothing at all is being monitored.
|
||
#
|
||
# Array Check
|
||
# Exits early if /mnt/user is not shfs-mounted. Watchdogs that inspect shares would
|
||
# otherwise read an unmounted array as missing data and act on it.
|
||
#
|
||
# Startup Grace
|
||
# WATCHDOG_STARTUP_GRACE is respected before any checks run, so containers still
|
||
# initialising after boot are not judged as unhealthy.
|
||
#
|
||
# Non-Fatal Steps
|
||
# run_orch_child() records a failing or missing watchdog and continues. One broken
|
||
# watchdog never suppresses the rest of the chain.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# watchdog_orchestrator.sh
|
||
# Normal run (called by cron every 15 minutes).
|
||
#
|
||
# watchdog_orchestrator.sh --dry-run
|
||
# Pass --dry-run to all sub-scripts.
|
||
#
|
||
# watchdog_orchestrator.sh --status
|
||
# Show script paths and current grace state.
|
||
#
|
||
# watchdog_orchestrator.sh --log
|
||
# Verbose output from all sub-scripts.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
|
||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# Skip immediately if another cycle is still running — no pile-up
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
||
# from a healthy run. Fail loudly instead of silently doing no work.
|
||
if [[ ${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} -eq 0 ]]; then
|
||
error "WATCHDOG_ORCHESTRATOR_SCRIPTS is empty — no watchdogs will run"
|
||
error "Check WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf"
|
||
notify "watchdogs skipped on $(hostname) ($MY_ID) — WATCHDOG_ORCHESTRATOR_SCRIPTS is empty" \
|
||
"$(basename "$0" .sh)" "warning"
|
||
exit 1
|
||
fi
|
||
|
||
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||
|
||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||
_watchdog_display_name() {
|
||
local path="$1"
|
||
local base="${path##*/}"
|
||
base="${base%.sh}"
|
||
base="${base//_/ }"
|
||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
|
||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||
else
|
||
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||
fi
|
||
|
||
echo ""
|
||
echo "── Sub-scripts ──"
|
||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||
label="$(_watchdog_display_name "$entry")"
|
||
if [[ -f "$local_path" ]]; then
|
||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||
echo " $icon $label — ${local_path##*/}"
|
||
else
|
||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo " Schedule: */15 * * * * (every 15 minutes)"
|
||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Array Check ━━━
|
||
# ==============================================================================================
|
||
if ! platform_storage_healthy; then
|
||
echo "Array not started — skipping watchdog cycle"
|
||
exit 0
|
||
fi
|
||
log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Startup Grace ━━━
|
||
# ==============================================================================================
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
||
exit 0
|
||
fi
|
||
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Run Watchdog Cycle ━━━
|
||
# ==============================================================================================
|
||
CYCLE_START=$(date +%s)
|
||
JOB_PASS=()
|
||
JOB_FAIL=()
|
||
|
||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
[[ -z "$_entry" ]] && continue
|
||
run_orch_child "$_entry"
|
||
done
|
||
|
||
CYCLE_END=$(date +%s)
|
||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Heartbeat ━━━
|
||
# ==============================================================================================
|
||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||
HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
|
||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||
log "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||
# ==============================================================================================
|
||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
else
|
||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||
fi
|
||
|
||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||
"Watchdog Orchestrator" "warning"
|
||
exit 1
|
||
fi
|
||
|
||
exit 0
|