From d3454a3dab74a22ad9df94cf8e994f25ed8d88e3 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 1 Jun 2026 19:31:16 -0400 Subject: [PATCH] =?UTF-8?q?docs:=20update=20watchdog=20headers=20=E2=80=94?= =?UTF-8?q?=20single-pass=20cron=20model,=20correct=20caller=20refs,=20sta?= =?UTF-8?q?bility=20vs=20system=20watchdog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Watchdogs/System/storage_watchdog.sh | 6 +++--- Watchdogs/System/webgui_watchdog.sh | 4 ++-- Watchdogs/docker_watchdog.sh | 14 ++++---------- Watchdogs/resource_watchdog.sh | 6 +++--- Watchdogs/stability_watchdog.sh | 12 ++++++------ Watchdogs/system_watchdog.sh | 4 ++-- 6 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Watchdogs/System/storage_watchdog.sh b/Watchdogs/System/storage_watchdog.sh index c111b76..f840af1 100755 --- a/Watchdogs/System/storage_watchdog.sh +++ b/Watchdogs/System/storage_watchdog.sh @@ -6,9 +6,9 @@ # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Pool and storage health monitoring — catches runaway data growth before it -# fills a pool. Runs as a single-pass script called by watchdog_orchestrator.sh -# every cycle. Sits between docker_watchdog.sh (container health) and -# system_watchdog.sh (last line of defense). Never reboots — detects, alerts, +# fills a pool. Runs as a single-pass script called by system_watchdog.sh each +# cycle. Sits between docker_watchdog.sh (container health) and +# stability_watchdog.sh (last line of defense). Never reboots — detects, alerts, # and optionally remediates. # # ============================================================================================== diff --git a/Watchdogs/System/webgui_watchdog.sh b/Watchdogs/System/webgui_watchdog.sh index 9c39168..a19dc06 100755 --- a/Watchdogs/System/webgui_watchdog.sh +++ b/Watchdogs/System/webgui_watchdog.sh @@ -6,8 +6,8 @@ # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a -# three-step escalating strategy — lightest fix first, heaviest last. Run -# every 5–10 minutes via the User Scripts plugin. Silent when healthy. +# three-step escalating strategy — lightest fix first, heaviest last. Called +# by system_watchdog.sh each cycle. Silent when healthy. # # ============================================================================================== # OPERATIONAL MODEL diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index a364b57..cfa687e 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -5,11 +5,8 @@ # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── -# Two-tier self-healing container monitoring system. Runs as a continuous -# background daemon started by array_started.sh at array start. Shuts down -# cleanly on SIGTERM/SIGINT when the array stops. -# -# Every DOCKER_WATCHDOG_INTERVAL seconds the watchdog runs a full cycle: +# Two-tier self-healing container monitoring system. Called by +# watchdog_orchestrator.sh via cron every 15 minutes as a single-pass run. # Tier 1 applies specific thresholds to explicitly configured containers. # Tier 2 scans everything else for generic health problems. Silent on clean # cycles, loud when something needs attention. @@ -166,9 +163,6 @@ # # master.conf # -# DOCKER_WATCHDOG_INTERVAL -# Seconds between full watchdog cycles (default: 900) -# # WATCHDOG_STARTUP_GRACE # Seconds before restart actions begin after watchdog starts (default: 600) # @@ -204,7 +198,7 @@ # ============================================================================================== # # docker_watchdog.sh -# Start continuous monitoring loop — normally launched by array_started.sh +# Single-pass monitoring cycle — called by watchdog_orchestrator.sh every 15 min # # docker_watchdog.sh --dry-run # Run a full watchdog cycle without restarting anything. Shows what would @@ -276,7 +270,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" - echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s" + echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)" echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" diff --git a/Watchdogs/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh index c58c7b1..aba89cd 100755 --- a/Watchdogs/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -6,9 +6,9 @@ # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Pressure reduction layer — detects rising system load and reduces it before -# things break. Called by watchdog_orchestrator.sh every minute as a single- -# pass run. The middle layer between docker_watchdog.sh (fixes broken -# containers) and system_watchdog.sh (reboots). Does neither of those things. +# things break. Called by watchdog_orchestrator.sh every 15 minutes as a +# single-pass run. The middle layer between docker_watchdog.sh (fixes broken +# containers) and stability_watchdog.sh (reboots). Does neither of those things. # # ============================================================================================== # OPERATIONAL MODEL diff --git a/Watchdogs/stability_watchdog.sh b/Watchdogs/stability_watchdog.sh index 89c9f79..9bbcd50 100755 --- a/Watchdogs/stability_watchdog.sh +++ b/Watchdogs/stability_watchdog.sh @@ -1,14 +1,14 @@ #!/bin/bash # ============================================================================================== -# ================================= System Watchdog ============================================ +# ================================= Stability Watchdog ========================================= # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Last line of defense — reboots the system cleanly if it is about to become -# unstable. Runs continuously as a background process started by -# array_started.sh at array start. Works alongside docker_watchdog.sh which -# handles container-level healing first. Only escalates to reboot when +# unstable. Called by watchdog_orchestrator.sh via cron every 15 minutes as a +# single-pass run. Works alongside docker_watchdog.sh which handles +# container-level healing first. Only escalates to reboot when # docker_watchdog.sh cannot resolve the condition. # # ============================================================================================== @@ -92,7 +92,7 @@ # ============================================================================================== # # stability_watchdog.sh -# Start continuous monitoring loop. Runs until stopped or system reboots. +# Single-pass stability check — called by watchdog_orchestrator.sh every 15 min # # stability_watchdog.sh --dry-run # Run detection logic without rebooting or stopping containers. @@ -172,7 +172,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}" echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C" echo "$ICON_GEAR Strike limit: ${SYS_WATCHDOG_STRIKE_LIMIT} cycles" - echo "$ICON_TIME Interval: 60s (cron — every minute)" + echo "$ICON_TIME Schedule: every 15 min (cron via watchdog_orchestrator)" echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr" echo "" echo "" diff --git a/Watchdogs/system_watchdog.sh b/Watchdogs/system_watchdog.sh index 7332e67..c07ac8b 100755 --- a/Watchdogs/system_watchdog.sh +++ b/Watchdogs/system_watchdog.sh @@ -4,11 +4,11 @@ # ============================================================================================== # Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially. # Called by watchdog_orchestrator.sh each cycle. Covers system component health: -# storage pool growth, runaway logs, and WebGUI availability. +# storage pool growth, runaway logs, WebGUI availability, and network connectivity. # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there. -# Default: storage_watchdog → webgui_watchdog +# Default: storage_watchdog → webgui_watchdog → network_watchdog # # ── SEQUENTIAL EXECUTION ───────────────────────────────────────────────────────────────────── # All scripts run in the foreground. Each must complete before the next starts.