Bring script headers onto the template and close safeguard gaps

Headers claimed protections the code never had, and several destructive paths had no
guard against a collapsed config value.

Watchdogs/System/conf_cache_watchdog.sh

@@ -21,6 +21,25 @@
 # No-op when FALLBACK_ENABLED=false or CONF_SYNC_ENABLED=false.
 #
 # ==============================================================================================
+# OPERATIONAL MODEL
+# ==============================================================================================
+#
+# One decision, driven entirely by remote reachability:
+#
+#   1. Gates
+#        → PARTNERSHIP_ENABLED, FALLBACK_ENABLED, CONF_SYNC_ENABLED, REMOTE_ID
+#        → any gate closed means exit 0, no work, no output
+#
+#   2. ping_remote
+#        REACHABLE   → remove $PERSISTENT_CONF_CACHE if it exists, exit
+#        UNREACHABLE → refresh the backup from the RAM cache
+#
+#   3. Refresh (remote offline only)
+#        → copy every host*.conf from the RAM cache except this host's own
+#        → own conf is excluded: it is already on disk, the backup exists
+#          solely to survive a reboot without the partner's vars
+#
+# ==============================================================================================
 # DESIGN PRINCIPLES
 # ==============================================================================================
 #
@@ -38,11 +57,70 @@
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
-#   require_partnership        — exits early if PARTNERSHIP_ENABLED=false
-#   FALLBACK_ENABLED gate      — exits if fallback is disabled
-#   CONF_SYNC_ENABLED gate     — exits if conf sync is disabled
-#   REMOTE_ID presence check   — exits if partner identity is unset
-#   --dry-run mode             — shows what would happen without touching the backup
+# Root Enforcement
+#   Writes to and removes $PERSISTENT_CONF_CACHE, which lives under the plugin
+#   directory and is not user-writable.
+#
+# Lock Acquisition
+#   acquire_lock() prevents concurrent execution. Without it a slow run can still
+#   be copying confs into the backup while the next run, seeing the remote back
+#   online, rm -rf's the directory out from under it.
+#
+# Cache Path Sanity Guard
+#   PERSISTENT_CONF_CACHE is validated as an absolute path at least three levels
+#   deep before any rm -rf. It is built from ${SCRIPTS_DIR} — if that is ever
+#   unset the path collapses toward the filesystem root, and this script would
+#   otherwise recursively delete whatever it collapsed to.
+#
+# Partnership Gate
+#   require_partnership() exits early if PARTNERSHIP_ENABLED=false.
+#
+# FALLBACK_ENABLED / CONF_SYNC_ENABLED Gates
+#   Exits cleanly if either is disabled — the backup only has meaning when
+#   fallback can actually consume it.
+#
+# REMOTE_ID Presence Check
+#   Exits if partner identity is unset. Without a partner there is nothing to
+#   back up and the own-conf exclusion below could not be applied correctly.
+#
+# Own-Conf Exclusion
+#   This host's own conf is never written into the partner backup. Restoring it
+#   later would overwrite live local config with a stale copy.
+#
+# Dry Run Support
+#   --dry-run reports every removal and copy without touching the backup.
+#
+# ==============================================================================================
+# CONFIGURATION
+# ==============================================================================================
+#
+# master.conf
+#
+#   PERSISTENT_CONF_CACHE
+#       Destination for the partner conf backup. Must survive a reboot, so it
+#       lives under ${SCRIPTS_DIR}, not in /tmp.
+#
+#   FALLBACK_ENABLED
+#       Master fallback toggle. Backup is pointless when fallback cannot run.
+#
+#   CONF_SYNC_ENABLED
+#       Conf sync toggle. When off, no RAM cache is being maintained to back up.
+#
+#   PARTNERSHIP_ENABLED
+#       Checked via require_partnership().
+#
+# ==============================================================================================
+# RUNTIME MODES
+# ==============================================================================================
+#
+# conf_cache_watchdog.sh
+#     Refresh or remove the persistent partner conf backup based on remote state
+#
+# conf_cache_watchdog.sh --dry-run
+#     Report what would be written or removed without changing the backup
+#
+# conf_cache_watchdog.sh --log
+#     Verbose per-file output
 #
 # ==============================================================================================
 
@@ -50,6 +128,17 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "$SCRIPT_DIR/../../load_config.sh"
 
 parse_args "$@"
+
+# ==============================================================================================
+# ━━━ Setup ━━━
+# ==============================================================================================
+if [[ "$EUID" -ne 0 ]]; then
+    error "Must be run as root"
+    exit 1
+fi
+
+acquire_lock
+
 detect_hosts
 require_partnership
 
@@ -58,7 +147,19 @@ require_partnership
 [[ -z "${REMOTE_ID:-}"         ]]            && exit 0
 
 RAM_CACHE="/tmp/.cache/vv/d"
-SAVE_DIR="$PERSISTENT_CONF_CACHE"
+SAVE_DIR="${PERSISTENT_CONF_CACHE:-}"
+
+# SAVE_DIR is rm -rf'd below and is built from ${SCRIPTS_DIR}. If that is ever unset the
+# path collapses toward / — require an absolute path at least three levels deep so a
+# collapsed or empty value can never name a system directory.
+_slashes="${SAVE_DIR//[^\/]/}"
+if [[ -z "$SAVE_DIR" || "$SAVE_DIR" != /* || "${#_slashes}" -lt 3 ]]; then
+    error "PERSISTENT_CONF_CACHE is unset or unsafe ('${SAVE_DIR:-unset}') — refusing to manage conf backup"
+    notify "conf_cache_watchdog aborted on $(hostname) — PERSISTENT_CONF_CACHE is '${SAVE_DIR:-unset}'" \
+        "Conf Cache Watchdog" "warning"
+    exit 1
+fi
+unset _slashes
 
 if ping_remote; then
     if [[ -d "$SAVE_DIR" ]]; then

Watchdogs/System/storage_watchdog.sh

@@ -73,6 +73,63 @@
 #   attention.
 #
 # ==============================================================================================
+# OPERATIONAL SAFEGUARDS
+# ==============================================================================================
+#
+# Root Enforcement
+#   Truncating container-owned log files requires root.
+#
+# Lock Acquisition
+#   acquire_lock() prevents concurrent runs. Two instances would race on the
+#   strike state file and the growth baseline, double-counting strikes and
+#   potentially truncating a file one cycle early.
+#
+# Host Detection
+#   detect_hosts() aliases HOST*_WATCHDOG_APPDATA_SIZES to the correct host's
+#   suppress ceilings.
+#
+# WATCHDOG_CHECK_APPDATA Toggle
+#   Exits cleanly before any scanning when the master toggle is off.
+#
+# Path Existence Guard
+#   Every entry in WATCHDOG_APPDATA_PATHS is skipped unless it is a non-empty
+#   string naming a real directory. An unconfigured array cannot cause a scan
+#   from an unintended location.
+#
+# Truncate-Never-Delete
+#   Action is always truncate -s 0, never rm. The container keeps its open file
+#   handle and space is reclaimed immediately, so a still-running service does
+#   not lose its log destination mid-write.
+#
+# Filename Restriction
+#   Only *.log and *.log.* files are ever truncation candidates. Databases,
+#   caches, game saves and every other growing file are alert-only — detected
+#   and reported, never modified.
+#
+# Truncation Opt-In
+#   WATCHDOG_APPDATA_TRUNCATE_LOGS defaults to false. Without it explicitly
+#   enabled the action cycle escalates to a critical notification and holds
+#   strikes rather than touching any file.
+#
+# Strike Threshold
+#   Nothing acts on first detection. WATCHDOG_APPDATA_STRIKE_LIMIT consecutive
+#   cycles are required, separating a legitimate library scan or save burst
+#   from a genuine runaway. Strikes auto-clear when the condition resolves.
+#
+# Suppress Ceiling
+#   Containers listed in WATCHDOG_APPDATA_SIZES are exempt from growth alerts
+#   while under their configured ceiling — prevents known-large stable data
+#   from generating recurring false alarms.
+#
+# Dry Run Support
+#   --dry-run reports every truncation that would occur and performs none.
+#
+# Atomic Baseline Update
+#   The growth baseline is written to a temp file and moved into place, so an
+#   interrupted run cannot leave a half-written baseline that would read as
+#   false growth on the next cycle.
+#
+# ==============================================================================================
 # CONFIGURATION
 # ==============================================================================================
 #

Watchdogs/docker_watchdog.sh

@@ -89,6 +89,17 @@
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
+# Root Enforcement
+#   Container restarts and daemon service control require root.
+#
+# Docker Enabled Check
+#   Exits cleanly when Docker is disabled in the platform's own settings. A
+#   deliberately disabled Docker service is not a fault and must not be
+#   "healed" by restarting the daemon.
+#
+# Docker Presence Check
+#   Verifies the docker binary exists before the cycle begins.
+#
 # Lock Acquisition
 #   Prevents concurrent execution via acquire_lock(). Safe at array start —
 #   only one watchdog instance runs at a time.
@@ -110,8 +121,11 @@
 #
 # Docker Daemon Health Check
 #   First operation every cycle. Daemon not responding within DOCKER_TIMEOUT →
-#   restart via /etc/rc.d/rc.docker → verify recovery. If still hung: log
-#   critical, skip cycle. stability_watchdog.sh handles further escalation.
+#   restart via platform_restart_service docker → verify recovery. If still hung:
+#   log critical, skip cycle, and set daemon_confirmed_down so
+#   stability_watchdog.sh owns any further escalation. The restart itself is
+#   bounded by a 180 second timeout — a daemon stop can block for 30+ minutes on
+#   a busy host, and the watchdog must not be held hostage to it.
 #
 # RAM Emergency Deferral
 #   Reads RW_STATE_FILE each cycle. If resource_watchdog.sh has set

Watchdogs/resource_watchdog.sh

@@ -42,6 +42,32 @@
 #   Cleared when pressure resolves and containers are restarted.
 #
 # ==============================================================================================
+# DESIGN PRINCIPLES
+# ==============================================================================================
+#
+# Graduated Response
+#   Pressure is answered with the smallest effective action first — throttle,
+#   then pause, then stop. Each level is only reached because the level below it
+#   failed to relieve pressure. Nothing jumps straight to stopping containers.
+#
+# Reversibility First
+#   docker pause suspends a container without losing its state and is instantly
+#   reversible, so it is preferred at level 2. docker stop, which discards
+#   in-memory state, is held back to level 3 and applied only to services
+#   explicitly listed as expendable in RW_STOP_CONTAINERS.
+#
+# Hysteresis on Recovery
+#   Restoring requires RW_RECOVER_CYCLES consecutive clear cycles and
+#   de-escalates one level at a time. Recovering instantly on a single good
+#   reading would flap — restore, re-trigger, restore — under sustained load.
+#
+# Cross-Watchdog Coordination
+#   Level 3 publishes mem_shutdown_active=true so docker_watchdog.sh defers its
+#   restart logic. Two watchdogs acting on the same containers with opposite
+#   intent would otherwise fight: one stopping to free RAM, the other restarting
+#   to restore health.
+#
+# ==============================================================================================
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
@@ -51,13 +77,45 @@
 # Single Instance Lock
 #   acquire_lock prevents concurrent runs from racing on state file writes.
 #
+# Docker Presence Check
+#   Verifies the docker binary exists before any pressure response — every
+#   level-2 and level-3 action depends on it.
+#
+# Host Detection
+#   detect_hosts() aliases HOST*_RW_PAUSE_CONTAINERS, HOST*_RW_STOP_CONTAINERS
+#   and the downloader credentials to the correct host's values.
+#
+# State File Verification
+#   Exits if RW_STATE_FILE cannot be created. Without durable state the script
+#   cannot track recovery cycles or know which containers it paused, and would
+#   never restore them.
+#
 # RW_CRITICAL_CONTAINERS
-#   Containers listed here are never paused or stopped regardless of pressure level.
+#   Containers listed here are never paused or stopped regardless of pressure
+#   level. Enforced by is_critical(), which gates both the pause and the stop
+#   path — not just the configuration lists.
 #
 # RW_ENABLED Flag
 #   Set RW_ENABLED=false to disable the entire script without removing it from
 #   the orchestrator schedule.
 #
+# Timeout Protection
+#   All docker commands wrapped in a 15 second timeout. Pressure response runs
+#   during a degraded system, which is exactly when the daemon is most likely
+#   to be slow — a hang here would stall the whole watchdog chain every minute.
+#
+# Downloader Availability Guards
+#   SABnzbd and qBittorrent throttling no-ops when the service is disabled or
+#   its URL/credentials are unset. A missing downloader never blocks the
+#   container-level pressure response.
+#
+# Recovery Hysteresis
+#   Restoration requires RW_RECOVER_CYCLES consecutive clear cycles and
+#   de-escalates one level per cycle, preventing flapping under sustained load.
+#
+# Dry Run Support
+#   --dry-run reports every throttle, pause and stop without performing any.
+#
 # ==============================================================================================
 # CONFIGURATION
 # ==============================================================================================

Watchdogs/stability_watchdog.sh

@@ -54,6 +54,39 @@
 #   (Container health is owned by docker_watchdog — not checked here.)
 #
 # ==============================================================================================
+# DESIGN PRINCIPLES
+# ==============================================================================================
+#
+# Last Line of Defense
+#   Every other watchdog tries to heal a specific subsystem. This one assumes
+#   those attempts have already failed and holds the only irreversible remedy in
+#   the ecosystem — a reboot. That authority is why nearly every check here is
+#   gated behind strikes, tiers and abort conditions.
+#
+# Evidence Before Reboot
+#   Strikes are the default; bypassing them requires corroboration, not just a
+#   worse number. Tier 2 needs low RAM AND active OOM kills before it acts —
+#   low RAM alone is a reading, low RAM plus processes being killed is a crisis.
+#
+# Unrecoverable Conditions Skip the Queue
+#   Tier 1 conditions share one property: the system cannot heal from them and
+#   waiting makes recovery less likely. A full rootfs or a kernel oops degrades
+#   further every cycle, and strike-counting through it only guarantees the
+#   reboot happens from a worse state.
+#
+# Data Safety Outranks Uptime
+#   Reboots abort while a ZFS pool is unhealthy, parity is running, or the mover
+#   is active. Interrupting those risks the data itself, which no amount of
+#   uptime justifies. Tier 1 is the sole exception — an imminent crash will
+#   interrupt them anyway, less gracefully.
+#
+# Clear Ownership Boundaries
+#   Container health belongs to docker_watchdog.sh and is deliberately not
+#   checked here. The Docker daemon check writes a flag for docker_watchdog
+#   rather than acting on it. Two watchdogs remediating the same subsystem
+#   would race.
+#
+# ==============================================================================================
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
@@ -61,10 +94,61 @@
 #   Reboot and container stop require root.
 #
 # Single Instance Lock
-#   acquire_lock prevents a second watchdog instance from starting.
+#   acquire_lock prevents a second watchdog instance from starting. Two instances
+#   could each count strikes against the same condition and reach the reboot
+#   threshold in half the intended time.
 #
 # State File Verification
 #   All state files verified writable at startup — errors if any cannot be created.
+#   Strike counts and the reboot log live in these files; if they silently failed
+#   to persist, every cycle would look like strike 1 and the reboot rate limit
+#   would never accumulate.
+#
+# Reboot Rate Limiting
+#   No more than SYS_WATCHDOG_REBOOT_LIMIT reboots within
+#   SYS_WATCHDOG_REBOOT_WINDOW_HRS. On hitting the limit the host powers off
+#   instead of rebooting — a fault that survives repeated reboots will not be
+#   fixed by more of them, and a box cycling endlessly is worse than one that
+#   is cleanly down and obviously needs attention.
+#
+# Abort Conditions
+#   Reboots are aborted while a ZFS pool is unhealthy, parity is running, or the
+#   mover is active — each individually toggleable. Interrupting any of these
+#   risks the data itself.
+#
+# Critical Tier Override
+#   Tier 1 conditions bypass both strikes and abort conditions. These are states
+#   the system cannot recover from and which degrade every cycle; waiting only
+#   guarantees the eventual reboot happens from a worse position.
+#
+# Strike Threshold
+#   Tier 3 requires SYS_WATCHDOG_STRIKE_LIMIT consecutive failing cycles. A
+#   single bad sample — a momentary load spike, a transient RAM dip — never
+#   reboots the system.
+#
+# OOM Corroboration
+#   Tier 2 escalation requires low RAM AND active OOM kills in the same cycle.
+#   Low RAM alone stays in the strike system.
+#
+# Ownership Boundary
+#   Container health is not checked here — docker_watchdog.sh owns it. The Docker
+#   daemon check writes daemon_confirmed_down for docker_watchdog rather than
+#   remediating, so the two never act on the same subsystem.
+#
+# Aborted-Reboot Recovery
+#   An EXIT trap is armed the moment containers start being stopped for a reboot
+#   and disarmed only once the reboot is committed. If the script dies anywhere
+#   in between, the trap restarts everything it stopped — the failure mode is a
+#   running system, never a host left with all containers down and no reboot.
+#
+# Sync Before Reboot
+#   sync is issued before both /sbin/poweroff and /sbin/reboot so pending writes
+#   are flushed. Container stop is additionally bounded by a 60 second timeout so
+#   one unresponsive container cannot hold the shutdown sequence open forever.
+#
+# Dry Run Support
+#   --dry-run runs the full detection path and reports the reboot or shutdown
+#   that would occur without issuing either.
 #
 # ==============================================================================================
 # CONFIGURATION
@@ -74,7 +158,7 @@
 #   Full variable listing in master.conf. Key variables:
 #
 #   SYS_WATCHDOG_REBOOT_WINDOW_HRS  — reboot rate limit window (default: 12)
-#   SYS_WATCHDOG_MAX_REBOOTS        — max reboots in window before giving up (default: 3)
+#   SYS_WATCHDOG_REBOOT_LIMIT       — max reboots in window before giving up (default: 3)
 #   SYS_WATCHDOG_STRIKE_LIMIT       — consecutive failures before reboot (default: 2)
 #   SYS_WATCHDOG_OOM_LIMIT          — OOM kills/cycle to trigger URGENT bypass (default: 3)
 #   SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED — containers exempt from memory shutdown

Watchdogs/system_watchdog.sh

@@ -35,10 +35,33 @@
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
-#   Root check       — child scripts require root
-#   acquire_lock     — prevents concurrent system watchdog runs
-#   detect_hosts()   — MY_ID in notifications and logs
-#   Non-fatal steps  — a failed step is logged; remaining steps still run
+# Root Enforcement
+#   Every child script requires root. Failing here gives one clear error instead
+#   of the same permission failure repeated once per child.
+#
+# Lock Acquisition
+#   acquire_lock() prevents concurrent system watchdog runs. This is called every
+#   cycle by watchdog_orchestrator.sh — a slow child must not cause two chains to
+#   overlap and run the same watchdog twice.
+#
+# Host Detection
+#   detect_hosts() sets MY_ID for notifications and logs.
+#
+# Empty List Guard
+#   Warns and exits if SYSTEM_WATCHDOG_SCRIPTS is unconfigured. An empty list
+#   would otherwise report "0/0 passed" every cycle — indistinguishable from
+#   healthy, while no system monitoring is actually running.
+#
+# Missing Script Tolerance
+#   run_orch_child() records a missing or failing child as a failed step and
+#   continues. One broken watchdog never suppresses the rest of the chain.
+#
+# Non-Fatal Steps
+#   A failed step is logged and surfaces in the summary and notification, but
+#   remaining steps still execute. Partial coverage beats a halted chain.
+#
+# Dry Run Propagation
+#   --dry-run and --log are passed through to every child script.
 #
 # ==============================================================================================
 # CONFIGURATION
@@ -86,6 +109,13 @@ acquire_lock
 
 detect_hosts
 
+# An empty list reports "0/0 passed" every cycle — reads as healthy while nothing is monitored.
+if [[ ${#SYSTEM_WATCHDOG_SCRIPTS[@]} -eq 0 ]]; then
+    warn "SYSTEM_WATCHDOG_SCRIPTS is empty — no system watchdogs will run"
+    warn "Check SYSTEM_WATCHDOG_SCRIPTS in master.conf"
+    exit 0
+fi
+
 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
 
 # ==============================================================================================