docs: update watchdog headers — single-pass cron model, correct caller refs, stability vs system watchdog
This commit is contained in:
@@ -6,9 +6,9 @@
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pool and storage health monitoring — catches runaway data growth before it
|
||||
# fills a pool. Runs as a single-pass script called by watchdog_orchestrator.sh
|
||||
# every cycle. Sits between docker_watchdog.sh (container health) and
|
||||
# system_watchdog.sh (last line of defense). Never reboots — detects, alerts,
|
||||
# fills a pool. Runs as a single-pass script called by system_watchdog.sh each
|
||||
# cycle. Sits between docker_watchdog.sh (container health) and
|
||||
# stability_watchdog.sh (last line of defense). Never reboots — detects, alerts,
|
||||
# and optionally remediates.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Monitors the unRAID WebGUI and restarts services if unresponsive. Uses a
|
||||
# three-step escalating strategy — lightest fix first, heaviest last. Run
|
||||
# every 5–10 minutes via the User Scripts plugin. Silent when healthy.
|
||||
# three-step escalating strategy — lightest fix first, heaviest last. Called
|
||||
# by system_watchdog.sh each cycle. Silent when healthy.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
|
||||
@@ -5,11 +5,8 @@
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Two-tier self-healing container monitoring system. Runs as a continuous
|
||||
# background daemon started by array_started.sh at array start. Shuts down
|
||||
# cleanly on SIGTERM/SIGINT when the array stops.
|
||||
#
|
||||
# Every DOCKER_WATCHDOG_INTERVAL seconds the watchdog runs a full cycle:
|
||||
# Two-tier self-healing container monitoring system. Called by
|
||||
# watchdog_orchestrator.sh via cron every 15 minutes as a single-pass run.
|
||||
# Tier 1 applies specific thresholds to explicitly configured containers.
|
||||
# Tier 2 scans everything else for generic health problems. Silent on clean
|
||||
# cycles, loud when something needs attention.
|
||||
@@ -166,9 +163,6 @@
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# DOCKER_WATCHDOG_INTERVAL
|
||||
# Seconds between full watchdog cycles (default: 900)
|
||||
#
|
||||
# WATCHDOG_STARTUP_GRACE
|
||||
# Seconds before restart actions begin after watchdog starts (default: 600)
|
||||
#
|
||||
@@ -204,7 +198,7 @@
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_watchdog.sh
|
||||
# Start continuous monitoring loop — normally launched by array_started.sh
|
||||
# Single-pass monitoring cycle — called by watchdog_orchestrator.sh every 15 min
|
||||
#
|
||||
# docker_watchdog.sh --dry-run
|
||||
# Run a full watchdog cycle without restarting anything. Shows what would
|
||||
@@ -276,7 +270,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
|
||||
|
||||
@@ -6,9 +6,9 @@
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pressure reduction layer — detects rising system load and reduces it before
|
||||
# things break. Called by watchdog_orchestrator.sh every minute as a single-
|
||||
# pass run. The middle layer between docker_watchdog.sh (fixes broken
|
||||
# containers) and system_watchdog.sh (reboots). Does neither of those things.
|
||||
# things break. Called by watchdog_orchestrator.sh every 15 minutes as a
|
||||
# single-pass run. The middle layer between docker_watchdog.sh (fixes broken
|
||||
# containers) and stability_watchdog.sh (reboots). Does neither of those things.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= System Watchdog ============================================
|
||||
# ================================= Stability Watchdog =========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Last line of defense — reboots the system cleanly if it is about to become
|
||||
# unstable. Runs continuously as a background process started by
|
||||
# array_started.sh at array start. Works alongside docker_watchdog.sh which
|
||||
# handles container-level healing first. Only escalates to reboot when
|
||||
# unstable. Called by watchdog_orchestrator.sh via cron every 15 minutes as a
|
||||
# single-pass run. Works alongside docker_watchdog.sh which handles
|
||||
# container-level healing first. Only escalates to reboot when
|
||||
# docker_watchdog.sh cannot resolve the condition.
|
||||
#
|
||||
# ==============================================================================================
|
||||
@@ -92,7 +92,7 @@
|
||||
# ==============================================================================================
|
||||
#
|
||||
# stability_watchdog.sh
|
||||
# Start continuous monitoring loop. Runs until stopped or system reboots.
|
||||
# Single-pass stability check — called by watchdog_orchestrator.sh every 15 min
|
||||
#
|
||||
# stability_watchdog.sh --dry-run
|
||||
# Run detection logic without rebooting or stopping containers.
|
||||
@@ -172,7 +172,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C"
|
||||
echo "$ICON_GEAR Strike limit: ${SYS_WATCHDOG_STRIKE_LIMIT} cycles"
|
||||
echo "$ICON_TIME Interval: 60s (cron — every minute)"
|
||||
echo "$ICON_TIME Schedule: every 15 min (cron via watchdog_orchestrator)"
|
||||
echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr"
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
@@ -4,11 +4,11 @@
|
||||
# ==============================================================================================
|
||||
# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially.
|
||||
# Called by watchdog_orchestrator.sh each cycle. Covers system component health:
|
||||
# storage pool growth, runaway logs, and WebGUI availability.
|
||||
# storage pool growth, runaway logs, WebGUI availability, and network connectivity.
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there.
|
||||
# Default: storage_watchdog → webgui_watchdog
|
||||
# Default: storage_watchdog → webgui_watchdog → network_watchdog
|
||||
#
|
||||
# ── SEQUENTIAL EXECUTION ─────────────────────────────────────────────────────────────────────
|
||||
# All scripts run in the foreground. Each must complete before the next starts.
|
||||
|
||||
Reference in New Issue
Block a user