diff --git a/Configurations/master.conf b/Configurations/master.conf index 99bcc8f..234e2e0 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -291,17 +291,25 @@ # Schedule: * * * * * (every minute) # NOT in ARRAY_START_SCRIPTS — has its own cron entry. # Order matters — resource first (frees pressure), docker second (heals with freed resources), -# storage third (pool/data health — never reboots), system last (last line of defense). +# system third (storage + webgui component health), stability last (last line of defense). WATCHDOG_ORCHESTRATOR_SCRIPTS=( "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts "Watchdogs/docker_watchdog.sh" # heal containers with freed resources - "Watchdogs/storage_watchdog.sh" # pool and appdata health — alert and remediate - "Watchdogs/system_watchdog.sh" # reboot if all else fails — last line of defense + "Watchdogs/system_watchdog.sh" # system component health — storage + webgui + "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense ) WATCHDOG_ORCHESTRATOR_HEARTBEAT=true WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1 +# ━━━ System Watchdog ━━━ +# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. +# Called by watchdog_orchestrator.sh — not scheduled directly. + SYSTEM_WATCHDOG_SCRIPTS=( + "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection + "Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp + ) + # ━━━ Critical Sync Maintenance ━━━ # critical_sync_maintenance.sh runs every 15 minutes. # Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check diff --git a/Docker_Essentials/README-Docker_Essentials.md b/Docker_Essentials/README-Docker_Essentials.md index ec5254f..f54543a 100644 --- a/Docker_Essentials/README-Docker_Essentials.md +++ b/Docker_Essentials/README-Docker_Essentials.md @@ -212,8 +212,8 @@ single-pass scripts called every minute by `Orchestrators/watchdog_orchestrator. ``` Watchdogs/resource_watchdog.sh ← reduces pressure before healing attempts Watchdogs/docker_watchdog.sh ← heals containers (reads resource_watchdog state) -Watchdogs/storage_watchdog.sh ← pool growth + runaway log detection -Watchdogs/system_watchdog.sh ← last resort — reboots when healing has failed +Watchdogs/System/storage_watchdog.sh ← pool growth + runaway log detection +Watchdogs/stability_watchdog.sh ← last resort — reboots when healing has failed ``` Scripts in this folder (daily restart, updates, network connect) are unaffected — diff --git a/Monitors/Manual-Monitors.md b/Monitors/Manual-Monitors.md index 77d0284..cff275c 100644 --- a/Monitors/Manual-Monitors.md +++ b/Monitors/Manual-Monitors.md @@ -232,7 +232,7 @@ data — it just doesn't trigger a notification for that condition. | `FALLBACK_STATE_FILE` | `Fallback/fallback.sh` | Current fallback state (NORMAL/FALLBACK/etc.) | | `SYS_WATCHDOG_FAILED_FILE` | `Watchdogs/docker_watchdog.sh` | Container skip list — needs human attention | | `WATCHDOG_STATE_FILE` | `Watchdogs/docker_watchdog.sh` | Active container strike counts | -| `SYS_WATCHDOG_STATE_FILE` | `Watchdogs/system_watchdog.sh` | Active system watchdog strikes | +| `SYS_WATCHDOG_STATE_FILE` | `Watchdogs/stability_watchdog.sh` | Active system watchdog strikes | | `BANDWIDTH_LOG` | `bandwidth_monitor.sh` | Yesterday's transfer history | | `TRANSCODE_DAILY_LOG` | `Transcodes/` | Weekly transcode statistics | | `CERT_MONITOR_DOMAINS` | live openssl check | Current cert status per domain | diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index c740f17..433499b 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -214,7 +214,7 @@ ARRAY_START_SCRIPTS=( # watchdogs check container states # ── Continuous scripts — run until array stops ───────────────────────────── - "Watchdogs/system_watchdog.sh" # system health BEFORE docker watchdog — + "Watchdogs/stability_watchdog.sh" # last line of defense — reboots when all else fails — # system watchdog writes state file that # docker watchdog reads every cycle "Watchdogs/docker_watchdog.sh" # container health BEFORE failover — diff --git a/Orchestrators/array_started.sh b/Orchestrators/array_started.sh index 7c8fb0b..305d6fc 100644 --- a/Orchestrators/array_started.sh +++ b/Orchestrators/array_started.sh @@ -18,7 +18,7 @@ # Docker_Essentials/docker_network_connect.sh — ensure networks + container connections # # CONTINUOUS (run until array stops): -# Watchdogs/system_watchdog.sh — system health monitor (last line of defense) +# Watchdogs/stability_watchdog.sh — system health monitor (last line of defense)) # Watchdogs/docker_watchdog.sh — container health monitor # Fallback/fallback.sh — mutual failover monitor # diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 90b6b4a..760fe08 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -480,7 +480,7 @@ if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running" fi else - issue " system_watchdog has never run (state file missing)" + issue " stability_watchdog has never run (state file missing)" fi if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then diff --git a/README-User_Script_Plug-in.md b/README-User_Script_Plug-in.md index 1ade41d..0174dda 100644 --- a/README-User_Script_Plug-in.md +++ b/README-User_Script_Plug-in.md @@ -1201,23 +1201,23 @@ flip count, session split ramdisk vs SSD). ## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ > **system_watchdog.sh is started by array_start.sh — not scheduled separately.** -> **webgui_restart.sh, system_tuning_monitor.sh, clear_logs.sh are scheduled below.** +> **webgui_watchdog.sh, system_tuning_monitor.sh, clear_logs.sh are scheduled below.** > **inotify_tuning.sh, php_fpm_max_children.sh, docker_syslog_filter.sh run via array_start.sh.** --- -### ── webgui_restart ─────────────────────────────────────────────────────────── +### ── webgui_watchdog ─────────────────────────────────────────────────────────── ``` Category: unRAID_Essentials -Name: webgui_restart +Name: webgui_watchdog Schedule: */10 * * * * (every 10 minutes) Background: YES ``` ```bash #!/bin/bash -/mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh +/mnt/user/appdata/unraid_scripts/Watchdogs/System/webgui_watchdog.sh ``` **What it does:** @@ -1275,7 +1275,7 @@ Background: NO ```bash #!/bin/bash -/mnt/user/appdata/unraid_scripts/Watchdogs/system_watchdog.sh --status +/mnt/user/appdata/unraid_scripts/Watchdogs/stability_watchdog.sh --status ``` **What it shows:** @@ -1650,7 +1650,7 @@ At Startup of Array: transcode_management BG: YES — cleanup then manager */10 * * * * (every 10 min): - webgui_restart BG: YES — silent when healthy + webgui_watchdog BG: YES — silent when healthy */15 * * * * (every 15 min): critical_sync_maintenance BG: YES — auth stack + Emby dirty + partnership check diff --git a/Watchdogs/Manual-Watchdogs.md b/Watchdogs/Manual-Watchdogs.md index 9ea818f..587827a 100644 --- a/Watchdogs/Manual-Watchdogs.md +++ b/Watchdogs/Manual-Watchdogs.md @@ -637,5 +637,5 @@ grep "system_watchdog" /var/log/syslog | tail -20 Orchestrators/watchdog_orchestrator.sh --status # Run the slow watchdog directly with --log to see where it's hanging: -Watchdogs/system_watchdog.sh --log --dry-run +Watchdogs/stability_watchdog.sh --log --dry-run ``` diff --git a/Watchdogs/storage_watchdog.sh b/Watchdogs/System/storage_watchdog.sh similarity index 99% rename from Watchdogs/storage_watchdog.sh rename to Watchdogs/System/storage_watchdog.sh index 01e85f9..b1e5996 100755 --- a/Watchdogs/storage_watchdog.sh +++ b/Watchdogs/System/storage_watchdog.sh @@ -140,7 +140,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../load_config.sh" +source "$SCRIPT_DIR/../../load_config.sh" parse_args "$@" diff --git a/unRAID_Essentials/webgui_restart.sh b/Watchdogs/System/webgui_watchdog.sh similarity index 98% rename from unRAID_Essentials/webgui_restart.sh rename to Watchdogs/System/webgui_watchdog.sh index 4a2e708..c964cbd 100644 --- a/unRAID_Essentials/webgui_restart.sh +++ b/Watchdogs/System/webgui_watchdog.sh @@ -76,23 +76,23 @@ # RUNTIME MODES # ============================================================================================== # -# webgui_restart.sh +# webgui_watchdog.sh # Check WebGUI. Escalate through nginx → php-fpm → emhttp if unresponsive. # -# webgui_restart.sh --dry-run +# webgui_watchdog.sh --dry-run # Show which services would be restarted. No restarts, no waits. # -# webgui_restart.sh --status +# webgui_watchdog.sh --status # Show current WebGUI response state and nginx/php-fpm/emhttp process states. # -# webgui_restart.sh --log +# webgui_watchdog.sh --log # Verbose output — show each check, each restart attempt, each wait. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../load_config.sh" +source "$SCRIPT_DIR/../../load_config.sh" parse_args "$@" diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index dccfbca..a4a18c1 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -52,9 +52,9 @@ # Parity awareness — skips restart actions during parity check # Timeout protection — all docker commands wrapped in timeout # Docker daemon check — each cycle begins with daemon health check; hung daemon → -# restart via rc.docker → system_watchdog.sh escalates if needed +# restart via rc.docker → stability_watchdog.sh escalates if needed # RAM emergency defer — reads SYS_WATCHDOG_STATE_FILE; stands down while -# system_watchdog.sh is managing a RAM emergency +# stability_watchdog.sh is managing a RAM emergency # # ============================================================================================== # DESIGN PRINCIPLES @@ -113,13 +113,13 @@ # Docker Daemon Health Check # First operation every cycle. Daemon not responding within DOCKER_TIMEOUT → # restart via /etc/rc.d/rc.docker → verify recovery. If still hung: log -# critical, skip cycle. system_watchdog.sh handles further escalation. +# critical, skip cycle. stability_watchdog.sh handles further escalation. # # RAM Emergency Deferral -# Reads SYS_WATCHDOG_STATE_FILE each cycle. If system_watchdog.sh has set +# Reads SYS_WATCHDOG_STATE_FILE each cycle. If stability_watchdog.sh has set # mem_shutdown_active=true, all restart logic defers until the flag clears. # Stale state guard: if file is >2 hours old with flag still set, -# system_watchdog.sh has likely stopped — watchdog resumes normal operation. +# stability_watchdog.sh has likely stopped — watchdog resumes normal operation. # # Timeout Protection # All docker commands wrapped in timeout. Daemon hangs cannot stall the @@ -465,7 +465,7 @@ is_parity_running() { # At WATCHDOG_DAEMON_STRIKE_LIMIT → attempt daemon restart via rc.docker # After restart → wait WATCHDOG_DAEMON_RESTART_WAIT seconds → verify # If verified → clear strikes, continue cycle ✅ -# If still hung → notify critical, skip cycle → system_watchdog.sh escalates from here +# If still hung → notify critical, skip cycle → stability_watchdog.sh escalates from here # # Returns: 0 = daemon healthy | 1 = daemon down, skip this cycle @@ -543,7 +543,7 @@ check_docker_daemon() { if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then error "Docker daemon still unresponsive after restart attempt" - error "system_watchdog.sh will handle further escalation" + error "stability_watchdog.sh will handle further escalation" queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical" flush_notify return 1 diff --git a/Watchdogs/stability_watchdog.sh b/Watchdogs/stability_watchdog.sh new file mode 100755 index 0000000..b90b3a2 --- /dev/null +++ b/Watchdogs/stability_watchdog.sh @@ -0,0 +1,805 @@ +#!/bin/bash +# ============================================================================================== +# ================================= System Watchdog ============================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Last line of defense — reboots the system cleanly if it is about to become +# unstable. Runs continuously as a background process started by +# array_started.sh at array start. Works alongside docker_watchdog.sh which +# handles container-level healing first. Only escalates to reboot when +# docker_watchdog.sh cannot resolve the condition. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Three-Tier Response System +# +# Tier 1 — CRITICAL (bypass all strikes, reboot immediately) +# Docker daemon unresponsive — nothing can be healed; running it longer makes it worse +# rootfs at 99%+ — writes failing; SSH may stop; no recovery options +# Kernel oops/BUG in dmesg — kernel running with corrupted state +# File descriptor exhaustion — new connections and processes failing silently +# /boot read-only unexpectedly — state files and config writes silently failing +# +# Tier 2 — URGENT (bypass strikes when OOM confirms active crisis) +# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle. +# OOM kills at this rate means the system is dying faster than watchdogs can heal. +# Without OOM confirmation → standard strike system applies. +# +# Tier 3 — STANDARD (N consecutive failures → reboot) +# RAM tiers, load, CPU temp, zombies, /var/log, /tmp, containers, NIC, mdstat. +# +# RAM Tiers +# MEM_WARN_GB (10GB) — warn + notify only +# MEM_SHUTDOWN_GB (6GB) — stop non-essential containers, wait for recovery +# MEM_GB (4GB) — strike system → reboot (bypass with OOM confirmation) +# MEM_RECOVER_GB (30GB) — RAM must reach this before stopped containers restart +# +# Container Shutdown Logic (at MEM_SHUTDOWN_GB) +# Stops all containers not in SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED. +# Stopped containers tracked in shutdown list — won't restart until RAM recovers. +# Strike system prevents flip-flopping — shutdown only once per degradation event. +# +# Abort Conditions (prevent reboot during sensitive operations) +# ZFS pool unhealthy, parity running, mover running — each toggleable. +# CRITICAL tier bypasses all abort conditions — imminent crash overrides data safety. +# +# Checks Run Every Cycle +# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg, +# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops, +# file descriptor exhaustion, array disk errors, NIC state, required containers. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# Reboot and container stop require root. +# +# Single Instance Lock +# acquire_lock prevents a second watchdog instance from starting. +# +# State File Verification +# All state files verified writable at startup — errors if any cannot be created. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf — System Watchdog section +# Full variable listing in master.conf. Key variables: +# +# SYS_WATCHDOG_REBOOT_WINDOW_HRS — reboot rate limit window (default: 2) +# SYS_WATCHDOG_MAX_REBOOTS — max reboots in window before giving up (default: 3) +# SYS_WATCHDOG_STRIKES — consecutive failures before reboot (default: 3) +# SYS_WATCHDOG_OOM_LIMIT — OOM kills/cycle to trigger URGENT bypass (default: 3) +# SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED — containers exempt from memory shutdown +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state +# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting +# SYS_WATCHDOG_FAILED_FILE — containers confirmed down for skip list integration +# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# stability_watchdog.sh +# Start continuous monitoring loop. Runs until stopped or system reboots. +# +# stability_watchdog.sh --dry-run +# Run detection logic without rebooting or stopping containers. +# +# stability_watchdog.sh --status +# Show config, thresholds, current system state, and strike counts. +# +# stability_watchdog.sh --log +# Verbose per-cycle output — show every check result and threshold comparison. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup — runs once at start ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +detect_hosts + +TOTAL_CORES=$(nproc) +DOCKER_TIMEOUT=10 +SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) + +# Ensure state files exist +for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \ + "$SYS_WATCHDOG_FAILED_FILE" "$SYS_WATCHDOG_OOM_FILE"; do + touch "$state_file" 2>/dev/null || { + error "Cannot create state file: $state_file" + exit 1 + } +done + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + echo "── Tier 1 — CRITICAL (bypass strikes immediately) ──" + echo "$ICON_DISK rootfs critical: ${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}%" + echo "$ICON_GEAR FD critical: ${SYS_WATCHDOG_FD_CRITICAL_PCT}%" + echo "$ICON_GEAR /boot read-only: check=${SYS_WATCHDOG_CHECK_BOOT}" + echo "$ICON_GEAR Kernel oops: check=${SYS_WATCHDOG_CHECK_KERNEL_OOPS}" + echo "$ICON_CONTAINERS Docker daemon: check=${SYS_WATCHDOG_CHECK_DOCKER_DAEMON}" + echo "" + echo "── Tier 2 — URGENT (bypass strikes with OOM confirmation) ──" + echo "$ICON_MEM RAM critical: < ${SYS_WATCHDOG_MEM_GB}GB" + echo "$ICON_GEAR OOM limit: ${SYS_WATCHDOG_OOM_LIMIT} kills/cycle" + echo "" + echo "── Tier 3 — STANDARD (strike system) ──" + echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%" + echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%" + echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%" + echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)" + echo " (RAM warn/shutdown/recover managed by resource_watchdog.sh)" + echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%" + echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)" + echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}" + echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C" + echo "$ICON_GEAR Strike limit: ${SYS_WATCHDOG_STRIKE_LIMIT} cycles" + echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" + echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr" + echo "" + echo "" + echo "── Check Toggles ──" + echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM" + echo " arc=$SYS_WATCHDOG_CHECK_ARC cpu_temp=$SYS_WATCHDOG_CHECK_CPU_TEMP load=$SYS_WATCHDOG_CHECK_LOAD" + echo " zombies=$SYS_WATCHDOG_CHECK_ZOMBIES docker=$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" + echo " containers=$SYS_WATCHDOG_CHECK_CONTAINERS oom=$SYS_WATCHDOG_CHECK_OOM" + echo " tmp=$SYS_WATCHDOG_CHECK_TMP fd=$SYS_WATCHDOG_CHECK_FD boot=$SYS_WATCHDOG_CHECK_BOOT" + echo " kernel_oops=$SYS_WATCHDOG_CHECK_KERNEL_OOPS sshd=$SYS_WATCHDOG_CHECK_SSHD" + echo " network=$SYS_WATCHDOG_CHECK_NETWORK mdstat=$SYS_WATCHDOG_CHECK_MDSTAT" + echo " runaway=$SYS_WATCHDOG_CHECK_RUNAWAY" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ── STATE HELPERS ───────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +get_strikes() { + grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 +} + +set_strikes() { + local key="$1" count="$2" + grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" +} + +increment_strikes() { + local key="$1" + local current + current=$(get_strikes "$key") + [[ -z "$current" ]] && current=0 + (( current++ )) + set_strikes "$key" "$current" + echo "$current" +} + +reset_strikes() { + set_strikes "$1" 0 +} + +get_state_val() { + grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2 +} + +set_state_val() { + local key="$1" val="$2" + grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" +} + +purge_old_reboots() { + local now cutoff + now=$(date +%s) + cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) + grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do + [[ "$ts" -gt "$cutoff" ]] && echo "$ts" + done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" + mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" +} + +count_recent_reboots() { + purge_old_reboots + grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 +} + +log_reboot() { + date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" +} + +# ============================================================================================== +# ── OOM TRACKING ────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Reads /proc/vmstat oom_kill counter — delta per cycle = rate of OOM kills +# Used for Tier 2 bypass and diagnostic context in reboot messages + +get_oom_delta() { + local current_oom + current_oom=$(grep "^oom_kill " /proc/vmstat 2>/dev/null | awk '{print $2}') + [[ -z "$current_oom" ]] && echo 0 && return + + local prev_oom + prev_oom=$(cat "$SYS_WATCHDOG_OOM_FILE" 2>/dev/null || echo 0) + echo "$current_oom" > "$SYS_WATCHDOG_OOM_FILE" + + local delta=$(( current_oom - prev_oom )) + [[ "$delta" -lt 0 ]] && delta=0 # counter reset on reboot + echo "$delta" +} + +get_oom_victims() { + # Get process names from dmesg that were OOM killed this boot + dmesg -T 2>/dev/null | grep -i "Killed process" | \ + awk '{print $NF}' | sort | uniq -c | sort -rn | head -5 | \ + awk '{printf "%s×%d ", $2, $1}' | sed 's/ $//' +} + +# ============================================================================================== +# ── ABORT CONDITIONS ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Returns 1 if reboot should be aborted, 0 if reboot should proceed +# CRITICAL tier bypasses this function entirely + +check_abort_conditions() { + local should_abort=false + + if command -v zpool >/dev/null 2>&1; then + local unhealthy + unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) + if [[ -n "$unhealthy" ]]; then + if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then + error "ZFS pool unhealthy — aborting reboot to prevent data loss" + notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — ZFS pool unhealthy" \ + "System Watchdog" "warning" + should_abort=true + else + warn "ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)" + fi + fi + fi + + if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then + if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then + error "Parity check running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — parity running" \ + "System Watchdog" "warning" + should_abort=true + else + warn "Parity check running — continuing reboot (ABORT_ON_PARITY=false)" + fi + fi + + if pgrep -f "mover" >/dev/null 2>&1; then + if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then + error "Mover running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — mover running" \ + "System Watchdog" "warning" + should_abort=true + else + warn "Mover running — continuing reboot (ABORT_ON_MOVER=false)" + fi + fi + + [[ "$should_abort" == true ]] && return 1 + return 0 +} + +# ============================================================================================== +# ── STANDARD STRIKE CHECK ───────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Returns 0 = reboot now | 1 = not yet + +run_strike_check() { + local key="$1" triggered="$2" description="$3" + if [[ "$triggered" == true ]]; then + local strikes + strikes=$(increment_strikes "$key") + warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" + if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then + error "$description — strike limit hit, reboot triggered" + reset_strikes "$key" + return 0 + fi + else + local current + current=$(get_strikes "$key") + [[ -n "$current" && "$current" -gt 0 ]] && reset_strikes "$key" + fi + return 1 +} + +# ── Exit Trap — restart containers stopped before an aborted reboot ─────────────────────────── +_SYS_REBOOT_STOPPED=() +_trap_sys_reboot_restart() { + [[ ${#_SYS_REBOOT_STOPPED[@]} -eq 0 ]] && return + warn "Exit trap: restarting containers stopped before aborted reboot" + for c in "${_SYS_REBOOT_STOPPED[@]}"; do + [[ -z "$c" ]] && continue + docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true + done +} + +# ============================================================================================== +# ── DO REBOOT ───────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# tier: "critical" (bypass abort) | "urgent" | "standard" + +do_reboot() { + local tier="${1:-standard}" + shift + local triggers=("$@") + + # Get OOM context for reboot message + local oom_victims="" + if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]]; then + oom_victims=$(get_oom_victims) + [[ -n "$oom_victims" ]] && triggers+=("oom_victims: $oom_victims") + fi + + # Abort check — CRITICAL bypasses this + if [[ "$tier" != "critical" ]]; then + if ! check_abort_conditions; then + return + fi + else + warn "CRITICAL tier — bypassing abort conditions" + fi + + RECENT_REBOOTS=$(count_recent_reboots) + log "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + + if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then + error "Reboot loop detected — shutting down instead of rebooting" + notify "Reboot loop on $(hostname) ($MY_ID) — shutting down after $RECENT_REBOOTS reboots — ${triggers[*]}" \ + "System Watchdog" "warning" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would shutdown now" + return + fi + sync + /sbin/poweroff + return + fi + + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " $ICON_REBOOT_SMART SYSTEM WATCHDOG — REBOOT TRIGGERED" + echo " Tier: ${tier^^}" + echo " Host: $MY_ID ($LOCAL_SERVER_NAME)" + for t in "${triggers[@]}"; do + echo " → $t" + done + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + notify "System watchdog ${tier^^} reboot on $(hostname) ($MY_ID) — ${triggers[*]}" \ + "System Watchdog" "warning" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — reboot sequence would begin now" + return + fi + + log_reboot + + # Graceful shutdown sequence + warn "Shutting down VMs..." + if command -v virsh >/dev/null 2>&1; then + for VM in $(virsh list --name 2>/dev/null); do + [[ -z "$VM" ]] && continue + virsh shutdown "$VM" >/dev/null 2>&1 + done + sleep 30 + fi + + warn "Stopping Docker containers..." + if command -v docker >/dev/null 2>&1; then + mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null) + trap _trap_sys_reboot_restart EXIT + timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1 + fi + + warn "Stopping User Scripts..." + pkill -f "/tmp/user.scripts" 2>/dev/null || true + + warn "Syncing disks..." + sync + + trap - EXIT # committed to reboot — containers should stay down + sleep 5 + /sbin/reboot +} + +# ============================================================================================== +# ━━━ Single-Pass Health Check ━━━ +# ============================================================================================== +warn "System watchdog — $MY_ID — $(date '+%H:%M:%S')" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + TRIGGERS=() + CRITICAL_TRIGGERS=() + URGENT_OOM_CONFIRMED=false + + # ── OOM Delta — read every cycle for bypass decisions ───────────────────────────────────── + OOM_DELTA=0 + if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]]; then + OOM_DELTA=$(get_oom_delta) + [[ "$OOM_DELTA" -gt 0 ]] && \ + log "OOM kills this cycle: $OOM_DELTA (limit: ${SYS_WATCHDOG_OOM_LIMIT})" + fi + + # ========================================================================================== + # ━━━ TIER 1 — CRITICAL CHECKS (bypass all strikes, reboot immediately) ━━━ + # ========================================================================================== + + # ── Docker daemon — critical: nothing can heal without it ───────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then + if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + error "Docker daemon unresponsive — CRITICAL" + + # Attempt daemon restart before rebooting + warn "Attempting Docker daemon restart..." + if [[ "$DRY_RUN" == false ]]; then + /etc/rc.d/rc.docker restart >/dev/null 2>&1 + sleep 15 + if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then + warn "Docker daemon restarted successfully — continuing monitoring" + else + error "Docker daemon restart failed — adding to CRITICAL triggers" + CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") + fi + else + warn "DRY RUN — would attempt Docker daemon restart" + CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") + fi + else + log "Docker daemon healthy ✅" + fi + fi + + # ── rootfs critical — at 99%+ writes are failing ───────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + ROOTFS_USED=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') + if [[ "$ROOTFS_USED" -ge "${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT:-99}" ]]; then + error "rootfs ${ROOTFS_USED}% — CRITICAL (writes failing)" + CRITICAL_TRIGGERS+=("rootfs_full=${ROOTFS_USED}%") + fi + fi + + # ── Kernel oops/BUG — kernel running with corrupted state ──────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_KERNEL_OOPS" == true ]]; then + PREV_OOPS=$(get_state_val "kernel_oops_count") + CURRENT_OOPS=$(dmesg 2>/dev/null | grep -cE "BUG:|kernel BUG|Oops:" || echo 0) + CURRENT_OOPS="${CURRENT_OOPS//[^0-9]/}"; CURRENT_OOPS="${CURRENT_OOPS:-0}" + set_state_val "kernel_oops_count" "$CURRENT_OOPS" + + if [[ -n "$PREV_OOPS" && "$PREV_OOPS" =~ ^[0-9]+$ ]]; then + OOPS_DELTA=$(( CURRENT_OOPS - PREV_OOPS )) + if [[ "$OOPS_DELTA" -gt 0 ]]; then + error "Kernel oops/BUG detected — $OOPS_DELTA new since last cycle — CRITICAL" + CRITICAL_TRIGGERS+=("kernel_oops=${OOPS_DELTA}_new") + fi + fi + fi + + # ── File descriptor exhaustion — new connections failing silently ───────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_FD" == true ]]; then + FD_LINE=$(cat /proc/sys/fs/file-nr 2>/dev/null) + FD_OPEN=$(echo "$FD_LINE" | awk '{print $1}') + FD_MAX=$(echo "$FD_LINE" | awk '{print $3}') + if [[ -n "$FD_OPEN" && -n "$FD_MAX" && "$FD_MAX" -gt 0 ]]; then + FD_PCT=$(( FD_OPEN * 100 / FD_MAX )) + if [[ "$FD_PCT" -ge "${SYS_WATCHDOG_FD_CRITICAL_PCT:-95}" ]]; then + error "File descriptors ${FD_PCT}% exhausted (${FD_OPEN}/${FD_MAX}) — CRITICAL" + CRITICAL_TRIGGERS+=("fd_exhaustion=${FD_PCT}%") + else + log "File descriptors: ${FD_PCT}% (${FD_OPEN}/${FD_MAX})" + fi + fi + fi + + # ── /boot read-only — state and config writes failing silently ──────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_BOOT" == true ]]; then + BOOT_TEST="/boot/.watchdog_write_test" + if ! touch "$BOOT_TEST" 2>/dev/null; then + error "/boot is read-only — config writes failing silently — CRITICAL" + CRITICAL_TRIGGERS+=("boot_read_only") + else + rm -f "$BOOT_TEST" 2>/dev/null + log "/boot is writable ✅" + fi + fi + + # ── Act on CRITICAL triggers immediately ───────────────────────────────────────────────── + if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT ━━━" + for t in "${CRITICAL_TRIGGERS[@]}"; do + error " CRITICAL: $t" + done + do_reboot "critical" "${CRITICAL_TRIGGERS[@]}" + exit 0 + fi + + # ========================================================================================== + # ━━━ TIER 3 — STANDARD CHECKS (strike system) ━━━ + # ========================================================================================== + + # ── rootfs standard ────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + ROOTFS_USED=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') + TRIGGERED=false + [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true + run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ + TRIGGERS+=("rootfs=${ROOTFS_USED}%") + fi + + # ── /var/log ───────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then + LOG_USED=$(df -P /var/log 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') + TRIGGERED=false + [[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true + run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ + TRIGGERS+=("log=${LOG_USED}%") + fi + + # ── /tmp ───────────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_TMP" == true ]]; then + TMP_USED=$(df -P /tmp 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') + if [[ "${TMP_USED:-0}" -ge "${SYS_WATCHDOG_TMP_CRITICAL_PCT:-98}" ]]; then + # Try to clear before escalating + warn "/tmp ${TMP_USED}% — attempting cleanup..." + find /tmp -type f -mmin +60 -not -name "*.lock" -delete 2>/dev/null + TMP_USED_AFTER=$(df -P /tmp 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') + if [[ "${TMP_USED_AFTER:-0}" -ge "${SYS_WATCHDOG_TMP_CRITICAL_PCT:-98}" ]]; then + error "/tmp still ${TMP_USED_AFTER}% after cleanup — adding to triggers" + TRIGGERED=true + else + warn "/tmp cleared to ${TMP_USED_AFTER}% ✅" + TRIGGERED=false + fi + elif [[ "${TMP_USED:-0}" -ge "${SYS_WATCHDOG_TMP_PCT:-90}" ]]; then + TRIGGERED=true + else + TRIGGERED=false + fi + run_strike_check "tmp" "$TRIGGERED" "/tmp ${TMP_USED}%" && \ + TRIGGERS+=("tmp=${TMP_USED}%") + fi + + # ── RAM — reboot tier only (warn/shutdown/recover handled by resource_watchdog.sh) ────────── + if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then + MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + MEM_GB=$(( MEM_KB / 1024 / 1024 )) + + if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then + # Tier 2 — bypass strike system if OOM confirms active crisis + if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \ + [[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then + error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this run — URGENT bypass" + OOM_VICTIMS=$(get_oom_victims) + URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}") + [[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS") + do_reboot "urgent" "${URGENT_TRIGGERS[@]}" + exit 0 + fi + # Standard strike path + run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \ + TRIGGERS+=("low_ram=${MEM_GB}GB") + else + reset_strikes "ram" + log "RAM ${MEM_GB}GB free ✅" + fi + fi + + # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) + TRIGGERED=false + if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then + sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5 + ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) + [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true + fi + run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned ${ARC_PCT}%" && \ + TRIGGERS+=("arc_pinned=${ARC_PCT}%") + fi + + # ── CPU temperature ─────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then + CPU_TEMP="" + if command -v sensors >/dev/null 2>&1; then + CPU_TEMP=$(sensors 2>/dev/null | \ + grep -i "Package id 0\|Tctl\|CPU Temp" | \ + awk '{print $NF}' | tr -d '+°C' | head -1) + fi + if [[ -n "$CPU_TEMP" ]]; then + CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") + TRIGGERED=false + [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true + run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \ + TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") + fi + fi + + # ── Load average ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then + LOAD=$(awk '{print $1}' /proc/loadavg) + LOAD_INT=$(printf "%.0f" "$LOAD") + LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) + TRIGGERED=false + [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true + run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ + TRIGGERS+=("load=${LOAD}") + fi + + # ── Zombie processes ───────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then + ZOMBIE_COUNT=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" || echo 0) + ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"; ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" + TRIGGERED=false + [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true + run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ + TRIGGERS+=("zombies=${ZOMBIE_COUNT}") + fi + + # ── Array disk errors — accumulating mdstat errors ──────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_MDSTAT" == true ]]; then + PREV_MD_ERRORS=$(get_state_val "mdstat_errors") + CURRENT_MD_ERRORS=$(grep -oP "(?<=\[)[^\]]*[U_][^\]]*(?=\])" \ + /proc/mdstat 2>/dev/null | grep -o "_" | wc -l || echo 0) + CURRENT_MD_ERRORS="${CURRENT_MD_ERRORS//[^0-9]/}"; CURRENT_MD_ERRORS="${CURRENT_MD_ERRORS:-0}" + set_state_val "mdstat_errors" "$CURRENT_MD_ERRORS" + + if [[ -n "$PREV_MD_ERRORS" && "$PREV_MD_ERRORS" =~ ^[0-9]+$ ]]; then + MD_DELTA=$(( CURRENT_MD_ERRORS - PREV_MD_ERRORS )) + if [[ "$MD_DELTA" -ge "${SYS_WATCHDOG_MDSTAT_ERROR_LIMIT:-5}" ]]; then + TRIGGERED=true + run_strike_check "mdstat" "$TRIGGERED" \ + "mdstat errors +${MD_DELTA} (total: ${CURRENT_MD_ERRORS})" && \ + TRIGGERS+=("mdstat_errors=+${MD_DELTA}") + else + run_strike_check "mdstat" false "mdstat" > /dev/null + fi + fi + fi + + # ── Network interface state ─────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then + NIC="${SYS_WATCHDOG_NIC:-eth0}" + NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown") + TRIGGERED=false + [[ "$NIC_STATE" != "up" ]] && TRIGGERED=true + run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \ + TRIGGERS+=("nic_down=${NIC}") + fi + + # ── sshd — try restart before escalating ───────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_SSHD" == true ]]; then + if ! pgrep -x sshd >/dev/null 2>&1; then + warn "sshd not running — attempting restart..." + if [[ "$DRY_RUN" == false ]]; then + /etc/rc.d/rc.sshd start >/dev/null 2>&1 + sleep 3 + if pgrep -x sshd >/dev/null 2>&1; then + warn "sshd restarted successfully ✅" + reset_strikes "sshd" + notify "sshd was down on $(hostname) ($MY_ID) — restarted automatically" \ + "System Watchdog" "warning" + else + error "sshd restart failed — remote access unavailable" + run_strike_check "sshd" true "sshd not running" && \ + TRIGGERS+=("sshd_down") + fi + else + warn "DRY RUN — would restart sshd" + fi + else + reset_strikes "sshd" + fi + fi + + # ── Runaway process ─────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_RUNAWAY" == true ]]; then + RUNAWAY_PCT="${SYS_WATCHDOG_RUNAWAY_CPU_PCT:-90}" + TOP_CPU_PCT=$(ps aux 2>/dev/null | awk 'NR>1{print $3}' | sort -rn | head -1) + TOP_CPU_INT=$(printf "%.0f" "${TOP_CPU_PCT:-0}") + TOP_CPU_NAME=$(ps aux 2>/dev/null | sort -k3 -rn | awk 'NR==2{print $11}') + TRIGGERED=false + [[ "$TOP_CPU_INT" -ge "$RUNAWAY_PCT" ]] && TRIGGERED=true + # Runaway uses SYS_WATCHDOG_RUNAWAY_STRIKES not global strike limit + if [[ "$TRIGGERED" == true ]]; then + RAWAY_S=$(increment_strikes "runaway") + RLIMIT="${SYS_WATCHDOG_RUNAWAY_STRIKES:-3}" + warn "Runaway ${TOP_CPU_NAME} ${TOP_CPU_PCT}% CPU -- strike $RAWAY_S/$RLIMIT" + if (( RAWAY_S >= RLIMIT )); then + error "Runaway process ${TOP_CPU_NAME} -- strike limit hit" + reset_strikes "runaway" + TRIGGERS+=("runaway=${TOP_CPU_NAME}@${TOP_CPU_PCT}%") + fi + else + RAWAY_CUR=$(get_strikes "runaway") + [[ "${RAWAY_CUR:-0}" -gt 0 ]] && reset_strikes "runaway" + fi + fi + + # ── Required containers from docker_watchdog skip list ──────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + FAILED_CONTAINERS=() + while IFS= read -r container; do + [[ -z "$container" ]] && continue + STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ + '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") + done < "$SYS_WATCHDOG_FAILED_FILE" + + TRIGGERED=false + [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]] && TRIGGERED=true + run_strike_check "failed_containers" "$TRIGGERED" \ + "required containers stopped: ${FAILED_CONTAINERS[*]:-}" && \ + TRIGGERS+=("containers=${FAILED_CONTAINERS[*]:-}") + fi + + # ========================================================================================== + # ━━━ Evaluate Standard Triggers ━━━ + # ========================================================================================== + if [[ ${#TRIGGERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_REBOOT_SMART System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + for t in "${TRIGGERS[@]}"; do + echo " $ICON_REBOOT_SMART $t" + done + [[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this run: $OOM_DELTA" + echo "" + do_reboot "standard" "${TRIGGERS[@]}" + exit 0 + else + echo "System healthy ✅ ($(date '+%H:%M:%S'))" + fi + + # Keep state file mtime fresh — docker_watchdog stale guard checks this + set_state_val "watchdog_cycle" "$(date +%s)" \ No newline at end of file diff --git a/Watchdogs/system_watchdog.sh b/Watchdogs/system_watchdog.sh index 24c4640..c308d3d 100755 --- a/Watchdogs/system_watchdog.sh +++ b/Watchdogs/system_watchdog.sh @@ -2,117 +2,43 @@ # ============================================================================================== # ================================= System Watchdog ============================================ # ============================================================================================== +# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially. +# Called by watchdog_orchestrator.sh each cycle. Covers system component health: +# storage pool growth, runaway logs, and WebGUI availability. # -# PURPOSE -# ───────────────────────────────────────────────────────────────────────────── -# Last line of defense — reboots the system cleanly if it is about to become -# unstable. Runs continuously as a background process started by -# array_started.sh at array start. Works alongside docker_watchdog.sh which -# handles container-level healing first. Only escalates to reboot when -# docker_watchdog.sh cannot resolve the condition. +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there. +# Default: storage_watchdog → webgui_watchdog # -# ============================================================================================== -# OPERATIONAL MODEL -# ============================================================================================== +# ── SEQUENTIAL EXECUTION ───────────────────────────────────────────────────────────────────── +# All scripts run in the foreground. Each must complete before the next starts. +# A failed script is logged but does not prevent remaining scripts from running. # -# Three-Tier Response System +# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── +# Root check — child scripts require root +# acquire_lock — prevents concurrent system watchdog runs +# detect_hosts() — MY_ID in notifications and logs +# Non-fatal steps — a failed step is logged; remaining steps still run # -# Tier 1 — CRITICAL (bypass all strikes, reboot immediately) -# Docker daemon unresponsive — nothing can be healed; running it longer makes it worse -# rootfs at 99%+ — writes failing; SSH may stop; no recovery options -# Kernel oops/BUG in dmesg — kernel running with corrupted state -# File descriptor exhaustion — new connections and processes failing silently -# /boot read-only unexpectedly — state files and config writes silently failing -# -# Tier 2 — URGENT (bypass strikes when OOM confirms active crisis) -# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle. -# OOM kills at this rate means the system is dying faster than watchdogs can heal. -# Without OOM confirmation → standard strike system applies. -# -# Tier 3 — STANDARD (N consecutive failures → reboot) -# RAM tiers, load, CPU temp, zombies, /var/log, /tmp, containers, NIC, mdstat. -# -# RAM Tiers -# MEM_WARN_GB (10GB) — warn + notify only -# MEM_SHUTDOWN_GB (6GB) — stop non-essential containers, wait for recovery -# MEM_GB (4GB) — strike system → reboot (bypass with OOM confirmation) -# MEM_RECOVER_GB (30GB) — RAM must reach this before stopped containers restart -# -# Container Shutdown Logic (at MEM_SHUTDOWN_GB) -# Stops all containers not in SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED. -# Stopped containers tracked in shutdown list — won't restart until RAM recovers. -# Strike system prevents flip-flopping — shutdown only once per degradation event. -# -# Abort Conditions (prevent reboot during sensitive operations) -# ZFS pool unhealthy, parity running, mover running — each toggleable. -# CRITICAL tier bypasses all abort conditions — imminent crash overrides data safety. -# -# Checks Run Every Cycle -# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg, -# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops, -# file descriptor exhaustion, array disk errors, NIC state, required containers. -# -# ============================================================================================== -# OPERATIONAL SAFEGUARDS -# ============================================================================================== -# -# Root Required -# Reboot and container stop require root. -# -# Single Instance Lock -# acquire_lock prevents a second watchdog instance from starting. -# -# State File Verification -# All state files verified writable at startup — errors if any cannot be created. -# -# ============================================================================================== -# CONFIGURATION -# ============================================================================================== -# -# master.conf — System Watchdog section -# Full variable listing in master.conf. Key variables: -# -# SYS_WATCHDOG_REBOOT_WINDOW_HRS — reboot rate limit window (default: 2) -# SYS_WATCHDOG_MAX_REBOOTS — max reboots in window before giving up (default: 3) -# SYS_WATCHDOG_STRIKES — consecutive failures before reboot (default: 3) -# SYS_WATCHDOG_OOM_LIMIT — OOM kills/cycle to trigger URGENT bypass (default: 3) -# SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED — containers exempt from memory shutdown -# -# ============================================================================================== -# STATE FILES -# ============================================================================================== -# -# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state -# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting -# SYS_WATCHDOG_FAILED_FILE — containers confirmed down for skip list integration -# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle -# -# ============================================================================================== -# RUNTIME MODES -# ============================================================================================== -# -# system_watchdog.sh -# Start continuous monitoring loop. Runs until stopped or system reboots. -# -# system_watchdog.sh --dry-run -# Run detection logic without rebooting or stopping containers. -# -# system_watchdog.sh --status -# Show config, thresholds, current system state, and strike counts. -# -# system_watchdog.sh --log -# Verbose per-cycle output — show every check result and threshold comparison. +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# SYSTEM_WATCHDOG_SCRIPTS — ordered list of system component watchdog scripts to run # +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# system_watchdog.sh — run all system component watchdogs +# system_watchdog.sh --dry-run — preview without running anything +# system_watchdog.sh --status — show configured scripts and exit +# system_watchdog.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -source "$SCRIPT_DIR/../load_config.sh" +source "$ECOSYSTEM_ROOT/load_config.sh" parse_args "$@" # ============================================================================================== -# ━━━ Setup — runs once at start ━━━ +# ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" @@ -128,20 +54,7 @@ acquire_lock detect_hosts -TOTAL_CORES=$(nproc) -DOCKER_TIMEOUT=10 -SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) - -# Ensure state files exist -for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \ - "$SYS_WATCHDOG_FAILED_FILE" "$SYS_WATCHDOG_OOM_FILE"; do - touch "$state_file" 2>/dev/null || { - error "Cannot create state file: $state_file" - exit 1 - } -done - -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboots or container shutdowns will occur" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no watchdog scripts will be executed" # ============================================================================================== # ━━━ Status ━━━ @@ -149,657 +62,76 @@ done if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━" - echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Scripts: ${#SYSTEM_WATCHDOG_SCRIPTS[@]} configured" echo "" - echo "── Tier 1 — CRITICAL (bypass strikes immediately) ──" - echo "$ICON_DISK rootfs critical: ${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT}%" - echo "$ICON_GEAR FD critical: ${SYS_WATCHDOG_FD_CRITICAL_PCT}%" - echo "$ICON_GEAR /boot read-only: check=${SYS_WATCHDOG_CHECK_BOOT}" - echo "$ICON_GEAR Kernel oops: check=${SYS_WATCHDOG_CHECK_KERNEL_OOPS}" - echo "$ICON_CONTAINERS Docker daemon: check=${SYS_WATCHDOG_CHECK_DOCKER_DAEMON}" - echo "" - echo "── Tier 2 — URGENT (bypass strikes with OOM confirmation) ──" - echo "$ICON_MEM RAM critical: < ${SYS_WATCHDOG_MEM_GB}GB" - echo "$ICON_GEAR OOM limit: ${SYS_WATCHDOG_OOM_LIMIT} kills/cycle" - echo "" - echo "── Tier 3 — STANDARD (strike system) ──" - echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%" - echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%" - echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%" - echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)" - echo " (RAM warn/shutdown/recover managed by resource_watchdog.sh)" - echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%" - echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)" - echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}" - echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C" - echo "$ICON_GEAR Strike limit: ${SYS_WATCHDOG_STRIKE_LIMIT} cycles" - echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" - echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr" - echo "" - echo "" - echo "── Check Toggles ──" - echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM" - echo " arc=$SYS_WATCHDOG_CHECK_ARC cpu_temp=$SYS_WATCHDOG_CHECK_CPU_TEMP load=$SYS_WATCHDOG_CHECK_LOAD" - echo " zombies=$SYS_WATCHDOG_CHECK_ZOMBIES docker=$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" - echo " containers=$SYS_WATCHDOG_CHECK_CONTAINERS oom=$SYS_WATCHDOG_CHECK_OOM" - echo " tmp=$SYS_WATCHDOG_CHECK_TMP fd=$SYS_WATCHDOG_CHECK_FD boot=$SYS_WATCHDOG_CHECK_BOOT" - echo " kernel_oops=$SYS_WATCHDOG_CHECK_KERNEL_OOPS sshd=$SYS_WATCHDOG_CHECK_SSHD" - echo " network=$SYS_WATCHDOG_CHECK_NETWORK mdstat=$SYS_WATCHDOG_CHECK_MDSTAT" - echo " runaway=$SYS_WATCHDOG_CHECK_RUNAWAY" + + for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do + [[ -z "$entry" ]] && continue + read -r -a parts <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") + if [[ ! -f "$script_path" ]]; then + echo " $ICON_ERROR $script_name — FILE NOT FOUND" + else + echo " $ICON_GEAR $script_name" + fi + done echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== -# ── STATE HELPERS ───────────────────────────────────────────────────────────────────────────── +# ━━━ Run Sequence ━━━ # ============================================================================================== +START=$(date +%s) +PASSED=() +FAILED=() +STEP=0 -get_strikes() { - grep -E "^${1}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 -} +for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do + [[ -z "$entry" ]] && continue + (( STEP++ )) -set_strikes() { - local key="$1" count="$2" - grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" - echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" - mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" -} + read -r -a parts <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") -increment_strikes() { - local key="$1" - local current - current=$(get_strikes "$key") - [[ -z "$current" ]] && current=0 - (( current++ )) - set_strikes "$key" "$current" - echo "$current" -} - -reset_strikes() { - set_strikes "$1" 0 -} - -get_state_val() { - grep -E "^${1}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d'=' -f2 -} - -set_state_val() { - local key="$1" val="$2" - grep -vE "^${key}=" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" - echo "${key}=${val}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" - mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" -} - -purge_old_reboots() { - local now cutoff - now=$(date +%s) - cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) - grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do - [[ "$ts" -gt "$cutoff" ]] && echo "$ts" - done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" - mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" -} - -count_recent_reboots() { - purge_old_reboots - grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 -} - -log_reboot() { - date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" -} - -# ============================================================================================== -# ── OOM TRACKING ────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Reads /proc/vmstat oom_kill counter — delta per cycle = rate of OOM kills -# Used for Tier 2 bypass and diagnostic context in reboot messages - -get_oom_delta() { - local current_oom - current_oom=$(grep "^oom_kill " /proc/vmstat 2>/dev/null | awk '{print $2}') - [[ -z "$current_oom" ]] && echo 0 && return - - local prev_oom - prev_oom=$(cat "$SYS_WATCHDOG_OOM_FILE" 2>/dev/null || echo 0) - echo "$current_oom" > "$SYS_WATCHDOG_OOM_FILE" - - local delta=$(( current_oom - prev_oom )) - [[ "$delta" -lt 0 ]] && delta=0 # counter reset on reboot - echo "$delta" -} - -get_oom_victims() { - # Get process names from dmesg that were OOM killed this boot - dmesg -T 2>/dev/null | grep -i "Killed process" | \ - awk '{print $NF}' | sort | uniq -c | sort -rn | head -5 | \ - awk '{printf "%s×%d ", $2, $1}' | sed 's/ $//' -} - -# ============================================================================================== -# ── ABORT CONDITIONS ────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Returns 1 if reboot should be aborted, 0 if reboot should proceed -# CRITICAL tier bypasses this function entirely - -check_abort_conditions() { - local should_abort=false - - if command -v zpool >/dev/null 2>&1; then - local unhealthy - unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) - if [[ -n "$unhealthy" ]]; then - if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then - error "ZFS pool unhealthy — aborting reboot to prevent data loss" - notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — ZFS pool unhealthy" \ - "System Watchdog" "warning" - should_abort=true - else - warn "ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)" - fi - fi + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + FAILED+=("$script_name") + continue fi - if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then - if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then - error "Parity check running — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — parity running" \ - "System Watchdog" "warning" - should_abort=true - else - warn "Parity check running — continuing reboot (ABORT_ON_PARITY=false)" - fi + if [[ ! -x "$script_path" ]]; then + chmod +x "$script_path" || { + error "$script_name — chmod +x failed" + FAILED+=("$script_name") + continue + } fi - if pgrep -f "mover" >/dev/null 2>&1; then - if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then - error "Mover running — aborting reboot" - notify "System watchdog aborted reboot on $(hostname) ($MY_ID) — mover running" \ - "System Watchdog" "warning" - should_abort=true - else - warn "Mover running — continuing reboot (ABORT_ON_MOVER=false)" - fi - fi - - [[ "$should_abort" == true ]] && return 1 - return 0 -} - -# ============================================================================================== -# ── STANDARD STRIKE CHECK ───────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Returns 0 = reboot now | 1 = not yet - -run_strike_check() { - local key="$1" triggered="$2" description="$3" - if [[ "$triggered" == true ]]; then - local strikes - strikes=$(increment_strikes "$key") - warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" - if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then - error "$description — strike limit hit, reboot triggered" - reset_strikes "$key" - return 0 - fi - else - local current - current=$(get_strikes "$key") - [[ -n "$current" && "$current" -gt 0 ]] && reset_strikes "$key" - fi - return 1 -} - -# ── Exit Trap — restart containers stopped before an aborted reboot ─────────────────────────── -_SYS_REBOOT_STOPPED=() -_trap_sys_reboot_restart() { - [[ ${#_SYS_REBOOT_STOPPED[@]} -eq 0 ]] && return - warn "Exit trap: restarting containers stopped before aborted reboot" - for c in "${_SYS_REBOOT_STOPPED[@]}"; do - [[ -z "$c" ]] && continue - docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true - done -} - -# ============================================================================================== -# ── DO REBOOT ───────────────────────────────────────────────────────────────────────────────── -# ============================================================================================== -# tier: "critical" (bypass abort) | "urgent" | "standard" - -do_reboot() { - local tier="${1:-standard}" - shift - local triggers=("$@") - - # Get OOM context for reboot message - local oom_victims="" - if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]]; then - oom_victims=$(get_oom_victims) - [[ -n "$oom_victims" ]] && triggers+=("oom_victims: $oom_victims") - fi - - # Abort check — CRITICAL bypasses this - if [[ "$tier" != "critical" ]]; then - if ! check_abort_conditions; then - return - fi - else - warn "CRITICAL tier — bypassing abort conditions" - fi - - RECENT_REBOOTS=$(count_recent_reboots) - log "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" - - if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then - error "Reboot loop detected — shutting down instead of rebooting" - notify "Reboot loop on $(hostname) ($MY_ID) — shutting down after $RECENT_REBOOTS reboots — ${triggers[*]}" \ - "System Watchdog" "warning" - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would shutdown now" - return - fi - sync - /sbin/poweroff - return - fi - - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo " $ICON_REBOOT_SMART SYSTEM WATCHDOG — REBOOT TRIGGERED" - echo " Tier: ${tier^^}" - echo " Host: $MY_ID ($LOCAL_SERVER_NAME)" - for t in "${triggers[@]}"; do - echo " → $t" - done - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - notify "System watchdog ${tier^^} reboot on $(hostname) ($MY_ID) — ${triggers[*]}" \ - "System Watchdog" "warning" - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — reboot sequence would begin now" - return + warn "DRY RUN — would run: $script_name" + PASSED+=("$script_name") + continue fi - log_reboot - - # Graceful shutdown sequence - warn "Shutting down VMs..." - if command -v virsh >/dev/null 2>&1; then - for VM in $(virsh list --name 2>/dev/null); do - [[ -z "$VM" ]] && continue - virsh shutdown "$VM" >/dev/null 2>&1 - done - sleep 30 - fi - - warn "Stopping Docker containers..." - if command -v docker >/dev/null 2>&1; then - mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null) - trap _trap_sys_reboot_restart EXIT - timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1 - fi - - warn "Stopping User Scripts..." - pkill -f "/tmp/user.scripts" 2>/dev/null || true - - warn "Syncing disks..." - sync - - trap - EXIT # committed to reboot — containers should stay down - sleep 5 - /sbin/reboot -} - -# ============================================================================================== -# ━━━ Single-Pass Health Check ━━━ -# ============================================================================================== -warn "System watchdog — $MY_ID — $(date '+%H:%M:%S')" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - TRIGGERS=() - CRITICAL_TRIGGERS=() - URGENT_OOM_CONFIRMED=false - - # ── OOM Delta — read every cycle for bypass decisions ───────────────────────────────────── - OOM_DELTA=0 - if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]]; then - OOM_DELTA=$(get_oom_delta) - [[ "$OOM_DELTA" -gt 0 ]] && \ - log "OOM kills this cycle: $OOM_DELTA (limit: ${SYS_WATCHDOG_OOM_LIMIT})" - fi - - # ========================================================================================== - # ━━━ TIER 1 — CRITICAL CHECKS (bypass all strikes, reboot immediately) ━━━ - # ========================================================================================== - - # ── Docker daemon — critical: nothing can heal without it ───────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then - if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then - error "Docker daemon unresponsive — CRITICAL" - - # Attempt daemon restart before rebooting - warn "Attempting Docker daemon restart..." - if [[ "$DRY_RUN" == false ]]; then - /etc/rc.d/rc.docker restart >/dev/null 2>&1 - sleep 15 - if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then - warn "Docker daemon restarted successfully — continuing monitoring" - else - error "Docker daemon restart failed — adding to CRITICAL triggers" - CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") - fi - else - warn "DRY RUN — would attempt Docker daemon restart" - CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") - fi - else - log "Docker daemon healthy ✅" - fi - fi - - # ── rootfs critical — at 99%+ writes are failing ───────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then - ROOTFS_USED=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') - if [[ "$ROOTFS_USED" -ge "${SYS_WATCHDOG_ROOTFS_CRITICAL_PCT:-99}" ]]; then - error "rootfs ${ROOTFS_USED}% — CRITICAL (writes failing)" - CRITICAL_TRIGGERS+=("rootfs_full=${ROOTFS_USED}%") - fi - fi - - # ── Kernel oops/BUG — kernel running with corrupted state ──────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_KERNEL_OOPS" == true ]]; then - PREV_OOPS=$(get_state_val "kernel_oops_count") - CURRENT_OOPS=$(dmesg 2>/dev/null | grep -cE "BUG:|kernel BUG|Oops:" || echo 0) - CURRENT_OOPS="${CURRENT_OOPS//[^0-9]/}"; CURRENT_OOPS="${CURRENT_OOPS:-0}" - set_state_val "kernel_oops_count" "$CURRENT_OOPS" - - if [[ -n "$PREV_OOPS" && "$PREV_OOPS" =~ ^[0-9]+$ ]]; then - OOPS_DELTA=$(( CURRENT_OOPS - PREV_OOPS )) - if [[ "$OOPS_DELTA" -gt 0 ]]; then - error "Kernel oops/BUG detected — $OOPS_DELTA new since last cycle — CRITICAL" - CRITICAL_TRIGGERS+=("kernel_oops=${OOPS_DELTA}_new") - fi - fi - fi - - # ── File descriptor exhaustion — new connections failing silently ───────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_FD" == true ]]; then - FD_LINE=$(cat /proc/sys/fs/file-nr 2>/dev/null) - FD_OPEN=$(echo "$FD_LINE" | awk '{print $1}') - FD_MAX=$(echo "$FD_LINE" | awk '{print $3}') - if [[ -n "$FD_OPEN" && -n "$FD_MAX" && "$FD_MAX" -gt 0 ]]; then - FD_PCT=$(( FD_OPEN * 100 / FD_MAX )) - if [[ "$FD_PCT" -ge "${SYS_WATCHDOG_FD_CRITICAL_PCT:-95}" ]]; then - error "File descriptors ${FD_PCT}% exhausted (${FD_OPEN}/${FD_MAX}) — CRITICAL" - CRITICAL_TRIGGERS+=("fd_exhaustion=${FD_PCT}%") - else - log "File descriptors: ${FD_PCT}% (${FD_OPEN}/${FD_MAX})" - fi - fi - fi - - # ── /boot read-only — state and config writes failing silently ──────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_BOOT" == true ]]; then - BOOT_TEST="/boot/.watchdog_write_test" - if ! touch "$BOOT_TEST" 2>/dev/null; then - error "/boot is read-only — config writes failing silently — CRITICAL" - CRITICAL_TRIGGERS+=("boot_read_only") - else - rm -f "$BOOT_TEST" 2>/dev/null - log "/boot is writable ✅" - fi - fi - - # ── Act on CRITICAL triggers immediately ───────────────────────────────────────────────── - if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT ━━━" - for t in "${CRITICAL_TRIGGERS[@]}"; do - error " CRITICAL: $t" - done - do_reboot "critical" "${CRITICAL_TRIGGERS[@]}" - exit 0 - fi - - # ========================================================================================== - # ━━━ TIER 3 — STANDARD CHECKS (strike system) ━━━ - # ========================================================================================== - - # ── rootfs standard ────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then - ROOTFS_USED=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') - TRIGGERED=false - [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true - run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ - TRIGGERS+=("rootfs=${ROOTFS_USED}%") - fi - - # ── /var/log ───────────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then - LOG_USED=$(df -P /var/log 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') - TRIGGERED=false - [[ "${LOG_USED:-0}" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true - run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ - TRIGGERS+=("log=${LOG_USED}%") - fi - - # ── /tmp ───────────────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_TMP" == true ]]; then - TMP_USED=$(df -P /tmp 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') - if [[ "${TMP_USED:-0}" -ge "${SYS_WATCHDOG_TMP_CRITICAL_PCT:-98}" ]]; then - # Try to clear before escalating - warn "/tmp ${TMP_USED}% — attempting cleanup..." - find /tmp -type f -mmin +60 -not -name "*.lock" -delete 2>/dev/null - TMP_USED_AFTER=$(df -P /tmp 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') - if [[ "${TMP_USED_AFTER:-0}" -ge "${SYS_WATCHDOG_TMP_CRITICAL_PCT:-98}" ]]; then - error "/tmp still ${TMP_USED_AFTER}% after cleanup — adding to triggers" - TRIGGERED=true - else - warn "/tmp cleared to ${TMP_USED_AFTER}% ✅" - TRIGGERED=false - fi - elif [[ "${TMP_USED:-0}" -ge "${SYS_WATCHDOG_TMP_PCT:-90}" ]]; then - TRIGGERED=true - else - TRIGGERED=false - fi - run_strike_check "tmp" "$TRIGGERED" "/tmp ${TMP_USED}%" && \ - TRIGGERS+=("tmp=${TMP_USED}%") - fi - - # ── RAM — reboot tier only (warn/shutdown/recover handled by resource_watchdog.sh) ────────── - if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then - MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) - MEM_GB=$(( MEM_KB / 1024 / 1024 )) - - if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then - # Tier 2 — bypass strike system if OOM confirms active crisis - if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \ - [[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then - error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this run — URGENT bypass" - OOM_VICTIMS=$(get_oom_victims) - URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}") - [[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS") - do_reboot "urgent" "${URGENT_TRIGGERS[@]}" - exit 0 - fi - # Standard strike path - run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \ - TRIGGERS+=("low_ram=${MEM_GB}GB") - else - reset_strikes "ram" - log "RAM ${MEM_GB}GB free ✅" - fi - fi - - # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) - TRIGGERED=false - if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then - sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5 - ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) - ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) - [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true - fi - run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned ${ARC_PCT}%" && \ - TRIGGERS+=("arc_pinned=${ARC_PCT}%") - fi - - # ── CPU temperature ─────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then - CPU_TEMP="" - if command -v sensors >/dev/null 2>&1; then - CPU_TEMP=$(sensors 2>/dev/null | \ - grep -i "Package id 0\|Tctl\|CPU Temp" | \ - awk '{print $NF}' | tr -d '+°C' | head -1) - fi - if [[ -n "$CPU_TEMP" ]]; then - CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") - TRIGGERED=false - [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true - run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \ - TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") - fi - fi - - # ── Load average ───────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then - LOAD=$(awk '{print $1}' /proc/loadavg) - LOAD_INT=$(printf "%.0f" "$LOAD") - LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) - TRIGGERED=false - [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true - run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ - TRIGGERS+=("load=${LOAD}") - fi - - # ── Zombie processes ───────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then - ZOMBIE_COUNT=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" || echo 0) - ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"; ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" - TRIGGERED=false - [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true - run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ - TRIGGERS+=("zombies=${ZOMBIE_COUNT}") - fi - - # ── Array disk errors — accumulating mdstat errors ──────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_MDSTAT" == true ]]; then - PREV_MD_ERRORS=$(get_state_val "mdstat_errors") - CURRENT_MD_ERRORS=$(grep -oP "(?<=\[)[^\]]*[U_][^\]]*(?=\])" \ - /proc/mdstat 2>/dev/null | grep -o "_" | wc -l || echo 0) - CURRENT_MD_ERRORS="${CURRENT_MD_ERRORS//[^0-9]/}"; CURRENT_MD_ERRORS="${CURRENT_MD_ERRORS:-0}" - set_state_val "mdstat_errors" "$CURRENT_MD_ERRORS" - - if [[ -n "$PREV_MD_ERRORS" && "$PREV_MD_ERRORS" =~ ^[0-9]+$ ]]; then - MD_DELTA=$(( CURRENT_MD_ERRORS - PREV_MD_ERRORS )) - if [[ "$MD_DELTA" -ge "${SYS_WATCHDOG_MDSTAT_ERROR_LIMIT:-5}" ]]; then - TRIGGERED=true - run_strike_check "mdstat" "$TRIGGERED" \ - "mdstat errors +${MD_DELTA} (total: ${CURRENT_MD_ERRORS})" && \ - TRIGGERS+=("mdstat_errors=+${MD_DELTA}") - else - run_strike_check "mdstat" false "mdstat" > /dev/null - fi - fi - fi - - # ── Network interface state ─────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_NETWORK" == true ]]; then - NIC="${SYS_WATCHDOG_NIC:-eth0}" - NIC_STATE=$(cat "/sys/class/net/${NIC}/operstate" 2>/dev/null || echo "unknown") - TRIGGERED=false - [[ "$NIC_STATE" != "up" ]] && TRIGGERED=true - run_strike_check "network" "$TRIGGERED" "${NIC} state: ${NIC_STATE}" && \ - TRIGGERS+=("nic_down=${NIC}") - fi - - # ── sshd — try restart before escalating ───────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_SSHD" == true ]]; then - if ! pgrep -x sshd >/dev/null 2>&1; then - warn "sshd not running — attempting restart..." - if [[ "$DRY_RUN" == false ]]; then - /etc/rc.d/rc.sshd start >/dev/null 2>&1 - sleep 3 - if pgrep -x sshd >/dev/null 2>&1; then - warn "sshd restarted successfully ✅" - reset_strikes "sshd" - notify "sshd was down on $(hostname) ($MY_ID) — restarted automatically" \ - "System Watchdog" "warning" - else - error "sshd restart failed — remote access unavailable" - run_strike_check "sshd" true "sshd not running" && \ - TRIGGERS+=("sshd_down") - fi - else - warn "DRY RUN — would restart sshd" - fi - else - reset_strikes "sshd" - fi - fi - - # ── Runaway process ─────────────────────────────────────────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_RUNAWAY" == true ]]; then - RUNAWAY_PCT="${SYS_WATCHDOG_RUNAWAY_CPU_PCT:-90}" - TOP_CPU_PCT=$(ps aux 2>/dev/null | awk 'NR>1{print $3}' | sort -rn | head -1) - TOP_CPU_INT=$(printf "%.0f" "${TOP_CPU_PCT:-0}") - TOP_CPU_NAME=$(ps aux 2>/dev/null | sort -k3 -rn | awk 'NR==2{print $11}') - TRIGGERED=false - [[ "$TOP_CPU_INT" -ge "$RUNAWAY_PCT" ]] && TRIGGERED=true - # Runaway uses SYS_WATCHDOG_RUNAWAY_STRIKES not global strike limit - if [[ "$TRIGGERED" == true ]]; then - RAWAY_S=$(increment_strikes "runaway") - RLIMIT="${SYS_WATCHDOG_RUNAWAY_STRIKES:-3}" - warn "Runaway ${TOP_CPU_NAME} ${TOP_CPU_PCT}% CPU -- strike $RAWAY_S/$RLIMIT" - if (( RAWAY_S >= RLIMIT )); then - error "Runaway process ${TOP_CPU_NAME} -- strike limit hit" - reset_strikes "runaway" - TRIGGERS+=("runaway=${TOP_CPU_NAME}@${TOP_CPU_PCT}%") - fi - else - RAWAY_CUR=$(get_strikes "runaway") - [[ "${RAWAY_CUR:-0}" -gt 0 ]] && reset_strikes "runaway" - fi - fi - - # ── Required containers from docker_watchdog skip list ──────────────────────────────────── - if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - FAILED_CONTAINERS=() - while IFS= read -r container; do - [[ -z "$container" ]] && continue - STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ - '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") - done < "$SYS_WATCHDOG_FAILED_FILE" - - TRIGGERED=false - [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]] && TRIGGERED=true - run_strike_check "failed_containers" "$TRIGGERED" \ - "required containers stopped: ${FAILED_CONTAINERS[*]:-}" && \ - TRIGGERS+=("containers=${FAILED_CONTAINERS[*]:-}") - fi - - # ========================================================================================== - # ━━━ Evaluate Standard Triggers ━━━ - # ========================================================================================== - if [[ ${#TRIGGERS[@]} -gt 0 ]]; then - echo "" - echo "━━━ $ICON_REBOOT_SMART System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - for t in "${TRIGGERS[@]}"; do - echo " $ICON_REBOOT_SMART $t" - done - [[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this run: $OOM_DELTA" - echo "" - do_reboot "standard" "${TRIGGERS[@]}" - exit 0 + if bash "$script_path"; then + log "$script_name — done ✅" + PASSED+=("$script_name") else - echo "System healthy ✅ ($(date '+%H:%M:%S'))" + warn "$script_name — exit non-zero (issues found or fixed) — continuing" + FAILED+=("$script_name") fi +done - # Keep state file mtime fresh — docker_watchdog stale guard checks this - set_state_val "watchdog_cycle" "$(date +%s)" \ No newline at end of file +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +log "System watchdog — $STEP script(s) — $(format_duration $(( END - START )))" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 diff --git a/common.sh b/common.sh index c994aff..360a864 100644 --- a/common.sh +++ b/common.sh @@ -1322,7 +1322,7 @@ acquire_lock() { else error "Another instance of $script_name is already running (PID $existing_pid) — exiting" case "$script_name" in - fallback|transcode_management|daily_sync_maintenance|system_watchdog) + fallback|transcode_management|daily_sync_maintenance|stability_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" ;; esac diff --git a/unRAID_Essentials/Manual-Unraid_Essentials.md b/unRAID_Essentials/Manual-Unraid_Essentials.md index 0590b4a..5367ab7 100644 --- a/unRAID_Essentials/Manual-Unraid_Essentials.md +++ b/unRAID_Essentials/Manual-Unraid_Essentials.md @@ -13,7 +13,7 @@ adding or reordering scripts at array start. ## ━━━ CONTENTS ━━━ - [ARRAY_START_SCRIPTS Order](#array_start_scripts-order) -- [webgui_restart.sh](#webgui_restartsh) +- [webgui_watchdog.sh](#webgui_watchdogsh) - [inotify_tuning.sh](#inotify_tuningsh) - [php_fpm_max_children.sh](#php_fpm_max_childrensh) - [docker_syslog_filter.sh](#docker_syslog_filtersh) @@ -32,7 +32,7 @@ adding or reordering scripts at array start. All scripts use a two-tier output model: `echo` lines are always visible; `log` lines only appear when `--log` is passed. -**Daemon scripts** (`webgui_restart.sh`): run on every cycle. Without `--log`, only +**Daemon scripts** (`webgui_watchdog.sh`): run on every cycle. Without `--log`, only state transitions, warnings, errors, and the clean-cycle conclusion line are visible. Per-check detail suppressed. @@ -76,7 +76,7 @@ generates veth messages — these will appear in syslog if the filter isn't acti --- -## webgui_restart.sh +## webgui_watchdog.sh ### Escalation Logic @@ -113,7 +113,7 @@ WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before recheck ```bash # Check which services are running: -webgui_restart.sh --status +webgui_watchdog.sh --status # Try manual restart sequence (same as the script): /etc/rc.d/rc.nginx restart @@ -196,8 +196,8 @@ grep "^pm.max_children" /etc/php83/php-fpm.d/www.conf ```bash # Check PHP-FPM worker utilization (requires system_tuning_monitor.sh in Monitors/): -# Look at the webgui_restart.sh escalation — step 2 (php-fpm restart) is specifically -# for worker exhaustion. If webgui_restart.sh is regularly hitting step 2, the +# Look at the webgui_watchdog.sh escalation — step 2 (php-fpm restart) is specifically +# for worker exhaustion. If webgui_watchdog.sh is regularly hitting step 2, the # pm.max_children value may still be too low, or there's a PHP worker leak. # Check running worker count: diff --git a/unRAID_Essentials/README-Unraid_Essentials.md b/unRAID_Essentials/README-Unraid_Essentials.md index 46614a7..7603b59 100644 --- a/unRAID_Essentials/README-Unraid_Essentials.md +++ b/unRAID_Essentials/README-Unraid_Essentials.md @@ -27,7 +27,7 @@ load, settings don't save, containers can't be started or stopped via the UI. No container-level alert fires because this isn't a container problem — it's a web server problem. By the time someone notices it may have been broken for hours. -Fix: `webgui_restart.sh` — checks every 10 minutes, escalates through nginx → +Fix: `webgui_watchdog.sh` — checks every 10 minutes, escalates through nginx → php-fpm → emhttp. Lightest fix first. Silent when healthy. **50+ Containers Starting and Filling Syslog With Veth Noise** @@ -71,7 +71,7 @@ SIGTERM (graceful — finishes current file), SIGKILL only if needed. ## ━━━ WHAT THIS FOLDER DOES ━━━ ``` -WebGUI availability webgui_restart.sh — nginx → php-fpm → emhttp escalation +WebGUI availability webgui_watchdog.sh — nginx → php-fpm → emhttp escalation Kernel tuning inotify_tuning.sh — file watch limits php_fpm_max_children.sh — PHP worker count Log hygiene docker_syslog_filter.sh — suppress veth noise at start @@ -110,7 +110,7 @@ Watchdogs/ | Script | Role | When It Runs | |--------|------|-------------| -| `webgui_restart.sh` | WebGUI availability — nginx → php-fpm → emhttp | Every 10 min via User Scripts | +| `webgui_watchdog.sh` | WebGUI availability — nginx → php-fpm → emhttp | Every 10 min via User Scripts | | `inotify_tuning.sh` | Raise inotify kernel limits | At array start — FIRST | | `php_fpm_max_children.sh` | Set PHP-FPM max worker count | At array start | | `docker_syslog_filter.sh` | Suppress Docker veth syslog noise | At array start — before containers | @@ -133,11 +133,11 @@ Array starts Every minute (watchdog_orchestrator.sh in Orchestrators/): → Watchdogs/resource_watchdog.sh → Watchdogs/docker_watchdog.sh - → Watchdogs/storage_watchdog.sh → Watchdogs/system_watchdog.sh + → Watchdogs/System/storage_watchdog.sh → Watchdogs/system_watchdog.sh (see Watchdogs/README-Watchdogs.md for full flow) Every 10 minutes (User Scripts): - └─ webgui_restart.sh + └─ webgui_watchdog.sh WebGUI OK → silent exit Not responding: Step 1: restart nginx → recheck diff --git a/unRAID_Essentials/server_reboot.sh b/unRAID_Essentials/server_reboot.sh index 7a76457..adac9e2 100644 --- a/unRAID_Essentials/server_reboot.sh +++ b/unRAID_Essentials/server_reboot.sh @@ -19,9 +19,9 @@ # 2. Wall message to all logged-in terminal users # 3. unRAID dashboard notification # 4. Wait REBOOT_SLEEP seconds — users time to save work -# 5. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT -# 6. Stop libvirt (VM Manager) -# 7. Stop Docker service +# 5. array_stopping.sh — user scripts, rsync, mover, containers (verified stop) +# 6. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT +# 7. Stop libvirt (VM Manager) # 8. sync — filesystem buffers flushed to disk # 9. /sbin/reboot # @@ -121,15 +121,6 @@ acquire_lock detect_hosts -# ── Exit Trap — restart Docker service if reboot sequence aborts after stopping it ──────────── -_REBOOT_DOCKER_STOPPED=false -_trap_restart_docker_service() { - [[ "$_REBOOT_DOCKER_STOPPED" == true ]] || return - warn "Exit trap: restarting Docker service after aborted reboot sequence" - /etc/rc.d/rc.docker start >/dev/null 2>&1 || true -} -trap _trap_restart_docker_service EXIT - [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur" # ============================================================================================== @@ -239,6 +230,25 @@ if [[ "$REBOOT_SLEEP" -gt 0 ]]; then fi fi +# ============================================================================================== +# ━━━ Array Stop Orchestrator ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_STOP Array Stop Orchestrator ━━━" +ARRAY_STOP_SCRIPT="$SCRIPT_DIR/../Orchestrators/array_stopping.sh" + +if [[ ! -f "$ARRAY_STOP_SCRIPT" ]]; then + warn "array_stopping.sh not found — skipping orchestrated stop" +elif [[ "$DRY_RUN" == true ]]; then + bash "$ARRAY_STOP_SCRIPT" --dry-run +else + if bash "$ARRAY_STOP_SCRIPT"; then + log "Array stop complete ✅" + else + warn "array_stopping.sh reported failures — proceeding with reboot" + fi +fi + # ============================================================================================== # ━━━ Graceful VM Shutdown ━━━ # ============================================================================================== @@ -280,22 +290,6 @@ else fi fi -# ============================================================================================== -# ━━━ Stop Docker ━━━ -# ============================================================================================== -echo "" -echo "━━━ $ICON_CONTAINERS Stop Docker ━━━" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would stop Docker service" -else - if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then - _REBOOT_DOCKER_STOPPED=true - warn "Docker stopped ✅" - else - warn "Docker stop returned non-zero — may already be stopped" - fi -fi - # ============================================================================================== # ━━━ Sync Disks ━━━ # ============================================================================================== @@ -327,6 +321,5 @@ if [[ "$DRY_RUN" == true ]]; then else warn "$ICON_REBOOT Rebooting $MY_ID now..." echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - trap - EXIT # committed to reboot — Docker should stay down /sbin/reboot fi \ No newline at end of file diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 77b9030..fff46f3 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -25,7 +25,7 @@ # weekly_sync_maintenance.sh 2:30am Sunday — clean sync + image updates # sunday_morning_coffee_report.sh 7am Sunday — full weekly digest # weekly_health_digest.sh 8am daily — profile-controlled health notification -# webgui_restart.sh every 10 min — WebGUI availability watchdog +# webgui_watchdog.sh every 10 min — WebGUI availability watchdog # system_tuning_monitor.sh every 6 hours — inotify + php-fpm trend tracking # clear_logs.sh 5am Sunday — size-threshold log cleanup # [Sunday monitor block] 6am-11am Sun — certs, SMART, backups, reports @@ -401,7 +401,7 @@ # 4. Restart emhttp (core daemon) wait WEBGUI_EMHTTP_WAIT=30s, recheck # 5. All failed → notify warning, manual intervention needed # -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/webgui_watchdog.sh # ── SYSTEM TUNING MONITOR ──────────────────────────────────────────────────────────────────── @@ -733,8 +733,8 @@ # Reboot loop protection: N reboots in X hours → shutdown instead. # State file heartbeat: writes watchdog_cycle=N every cycle (docker_watchdog stale guard). # -# bash /mnt/user/appdata/unraid_scripts/Watchdogs/system_watchdog.sh --status -# bash /mnt/user/appdata/unraid_scripts/Watchdogs/system_watchdog.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/stability_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/stability_watchdog.sh --dry-run # resource_watchdog.sh — three-level pressure reduction layer (single-pass, called by watchdog_orchestrator) # Reduces system load intelligently BEFORE docker_watchdog attempts container restarts. @@ -787,9 +787,9 @@ # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh -# webgui_restart.sh — WebGUI availability watchdog (see ORCHESTRATORS section above) -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh --dry-run -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/webgui_restart.sh --status +# webgui_watchdog.sh — WebGUI availability watchdog (see ORCHESTRATORS section above) +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/webgui_watchdog.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Watchdogs/System/webgui_watchdog.sh --status # clear_logs.sh — size-threshold log cleanup (see ORCHESTRATORS section above) # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh --dry-run @@ -1301,7 +1301,7 @@ # Orchestrators/transcode_management.sh # # */10 * * * * every 10 minutes: -# unRAID_Essentials/webgui_restart.sh +# Watchdogs/System/webgui_watchdog.sh # # */15 * * * * every 15 minutes: # Orchestrators/critical_sync_maintenance.sh