docs: update watchdog headers — single-pass cron model, correct caller refs, stability vs system watchdog

This commit is contained in:
Gmer4Lfe
2026-06-01 19:31:16 -04:00
parent ee26eebc6b
commit d3454a3dab
6 changed files with 20 additions and 26 deletions
+3 -3
View File
@@ -6,9 +6,9 @@
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pool and storage health monitoring — catches runaway data growth before it
# fills a pool. Runs as a single-pass script called by watchdog_orchestrator.sh
# every cycle. Sits between docker_watchdog.sh (container health) and
# system_watchdog.sh (last line of defense). Never reboots — detects, alerts,
# fills a pool. Runs as a single-pass script called by system_watchdog.sh each
# cycle. Sits between docker_watchdog.sh (container health) and
# stability_watchdog.sh (last line of defense). Never reboots — detects, alerts,
# and optionally remediates.
#
# ==============================================================================================
+2 -2
View File
@@ -6,8 +6,8 @@
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a
# three-step escalating strategy — lightest fix first, heaviest last. Run
# every 510 minutes via the User Scripts plugin. Silent when healthy.
# three-step escalating strategy — lightest fix first, heaviest last. Called
# by system_watchdog.sh each cycle. Silent when healthy.
#
# ==============================================================================================
# OPERATIONAL MODEL
+4 -10
View File
@@ -5,11 +5,8 @@
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Two-tier self-healing container monitoring system. Runs as a continuous
# background daemon started by array_started.sh at array start. Shuts down
# cleanly on SIGTERM/SIGINT when the array stops.
#
# Every DOCKER_WATCHDOG_INTERVAL seconds the watchdog runs a full cycle:
# Two-tier self-healing container monitoring system. Called by
# watchdog_orchestrator.sh via cron every 15 minutes as a single-pass run.
# Tier 1 applies specific thresholds to explicitly configured containers.
# Tier 2 scans everything else for generic health problems. Silent on clean
# cycles, loud when something needs attention.
@@ -166,9 +163,6 @@
#
# master.conf
#
# DOCKER_WATCHDOG_INTERVAL
# Seconds between full watchdog cycles (default: 900)
#
# WATCHDOG_STARTUP_GRACE
# Seconds before restart actions begin after watchdog starts (default: 600)
#
@@ -204,7 +198,7 @@
# ==============================================================================================
#
# docker_watchdog.sh
# Start continuous monitoring loop — normally launched by array_started.sh
# Single-pass monitoring cycle — called by watchdog_orchestrator.sh every 15 min
#
# docker_watchdog.sh --dry-run
# Run a full watchdog cycle without restarting anything. Shows what would
@@ -276,7 +270,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
+3 -3
View File
@@ -6,9 +6,9 @@
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pressure reduction layer — detects rising system load and reduces it before
# things break. Called by watchdog_orchestrator.sh every minute as a single-
# pass run. The middle layer between docker_watchdog.sh (fixes broken
# containers) and system_watchdog.sh (reboots). Does neither of those things.
# things break. Called by watchdog_orchestrator.sh every 15 minutes as a
# single-pass run. The middle layer between docker_watchdog.sh (fixes broken
# containers) and stability_watchdog.sh (reboots). Does neither of those things.
#
# ==============================================================================================
# OPERATIONAL MODEL
+6 -6
View File
@@ -1,14 +1,14 @@
#!/bin/bash
# ==============================================================================================
# ================================= System Watchdog ============================================
# ================================= Stability Watchdog =========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Last line of defense — reboots the system cleanly if it is about to become
# unstable. Runs continuously as a background process started by
# array_started.sh at array start. Works alongside docker_watchdog.sh which
# handles container-level healing first. Only escalates to reboot when
# unstable. Called by watchdog_orchestrator.sh via cron every 15 minutes as a
# single-pass run. Works alongside docker_watchdog.sh which handles
# container-level healing first. Only escalates to reboot when
# docker_watchdog.sh cannot resolve the condition.
#
# ==============================================================================================
@@ -92,7 +92,7 @@
# ==============================================================================================
#
# stability_watchdog.sh
# Start continuous monitoring loop. Runs until stopped or system reboots.
# Single-pass stability check — called by watchdog_orchestrator.sh every 15 min
#
# stability_watchdog.sh --dry-run
# Run detection logic without rebooting or stopping containers.
@@ -172,7 +172,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C"
echo "$ICON_GEAR Strike limit: ${SYS_WATCHDOG_STRIKE_LIMIT} cycles"
echo "$ICON_TIME Interval: 60s (cron — every minute)"
echo "$ICON_TIME Schedule: every 15 min (cron via watchdog_orchestrator)"
echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr"
echo ""
echo ""
+2 -2
View File
@@ -4,11 +4,11 @@
# ==============================================================================================
# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially.
# Called by watchdog_orchestrator.sh each cycle. Covers system component health:
# storage pool growth, runaway logs, and WebGUI availability.
# storage pool growth, runaway logs, WebGUI availability, and network connectivity.
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there.
# Default: storage_watchdog → webgui_watchdog
# Default: storage_watchdog → webgui_watchdog → network_watchdog
#
# ── SEQUENTIAL EXECUTION ─────────────────────────────────────────────────────────────────────
# All scripts run in the foreground. Each must complete before the next starts.