feat: watchdog architecture v2 — resource manager + single-pass orchestrator

Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs:

- resource_manager.sh (new): single-pass pressure reduction layer; throttles
  SABnzbd/qBit at level 1, docker-pauses background containers at level 2,
  docker-stops optional containers and signals docker_watchdog to defer at
  level 3; graduated recovery with hysteresis

- watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager →
  docker_watchdog → system_watchdog in sequence; intended for per-minute cron
  via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat

- docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to
  state file across runs; cross-script coordination reads RM_STATE_FILE instead
  of SYS_WATCHDOG_STATE_FILE

- system_watchdog.sh: de-looped to single-pass; stripped of all container
  management (shutdown_non_essential_containers removed); reboot-only last resort

- master.conf: removed system_watchdog and docker_watchdog from
  ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections

- master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays

- common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts()

- continuous_scripts_status.sh: moved to Tools/ (preserved for future use)

- sunday_morning_coffee_report.sh: watchdog section updated to use state file
  mtime checks instead of is_running; added Resource Manager subsection;
  fixed mem_shutdown grep filter pointing to wrong state file

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
2026-05-12 17:41:59 -04:00
co-authored by Claude Sonnet 4.6
parent f16c962ac0
commit 309546e615
9 changed files with 925 additions and 277 deletions
+42 -14
View File
@@ -14,7 +14,7 @@
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth
# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures
# 🛡️ Watchdog — system watchdog, docker watchdog, fallback state
# 🛡️ Watchdog — resource manager, system watchdog, docker watchdog, fallback state
# 🔐 Security — SSL cert expiry per domain
# 📊 Emby — weekly stream count, active now, top users
# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync
@@ -468,17 +468,21 @@ fi
section "🛡️ WATCHDOG"
# ── System Watchdog ───────────────────────────────────────────────────────────────────────────
line "⚙️ System Watchdog"
SYS_PID=$(_get_lock_pid "system_watchdog")
if _is_running "system_watchdog"; then
SYS_AGE=$(_lock_age "system_watchdog")
line " ✅ Running │ PID: $SYS_PID │ Uptime: $(_fmt_uptime "$SYS_AGE") │ ~Cycle: $(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))"
line "⚙️ System Watchdog (cron via watchdog_orchestrator)"
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
_sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
_sw_ago=$(( NOW - _sw_last ))
if [[ "$_sw_ago" -lt 600 ]]; then
line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago"
else
issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running"
fi
else
issue "system_watchdog NOT RUNNING"
issue " system_watchdog has never run (state file missing)"
fi
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=\|^mem_shutdown" \
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \
"$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$SYS_ACTIVE" ]]; then
while IFS=: read -r key count; do
@@ -519,13 +523,17 @@ line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB
REPORT+=("")
# ── Docker Watchdog ───────────────────────────────────────────────────────────────────────────
line "🐳 Docker Watchdog"
DOCKER_PID=$(_get_lock_pid "docker_watchdog")
if _is_running "docker_watchdog"; then
DOCKER_AGE=$(_lock_age "docker_watchdog")
line " ✅ Running │ PID: $DOCKER_PID │ Uptime: $(_fmt_uptime "$DOCKER_AGE") │ ~Cycle: $(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))"
line "🐳 Docker Watchdog (cron via watchdog_orchestrator)"
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
_dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
_dw_ago=$(( NOW - _dw_last ))
if [[ "$_dw_ago" -lt 600 ]]; then
line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago"
else
issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running"
fi
else
issue "docker_watchdog NOT RUNNING"
issue " docker_watchdog has never run (state file missing)"
fi
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
@@ -554,6 +562,26 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
fi
fi
# ── Resource Manager ─────────────────────────────────────────────────────────────────────────
line "🎛️ Resource Manager"
if [[ -f "$RM_STATE_FILE" ]]; then
_rm_last=$(stat -c %Y "$RM_STATE_FILE" 2>/dev/null || echo 0)
_rm_ago=$(( NOW - _rm_last ))
_rm_level=$(grep "^current_level:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2)
_rm_level="${_rm_level:-0}"
if [[ "$_rm_level" -gt 0 ]]; then
issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago"
elif [[ "$_rm_ago" -lt 600 ]]; then
line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago"
else
issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running"
fi
else
line " ️ State file not found (resource_manager may not have run yet)"
fi
REPORT+=("")
if command -v docker >/dev/null 2>&1; then
RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)