feat: watchdog architecture v2 — resource manager + single-pass orchestrator
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f16c962ac0
commit
309546e615
@@ -14,7 +14,7 @@
|
||||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth
|
||||
# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures
|
||||
# 🛡️ Watchdog — system watchdog, docker watchdog, fallback state
|
||||
# 🛡️ Watchdog — resource manager, system watchdog, docker watchdog, fallback state
|
||||
# 🔐 Security — SSL cert expiry per domain
|
||||
# 📊 Emby — weekly stream count, active now, top users
|
||||
# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync
|
||||
@@ -468,17 +468,21 @@ fi
|
||||
section "🛡️ WATCHDOG"
|
||||
|
||||
# ── System Watchdog ───────────────────────────────────────────────────────────────────────────
|
||||
line "⚙️ System Watchdog"
|
||||
SYS_PID=$(_get_lock_pid "system_watchdog")
|
||||
if _is_running "system_watchdog"; then
|
||||
SYS_AGE=$(_lock_age "system_watchdog")
|
||||
line " ✅ Running │ PID: $SYS_PID │ Uptime: $(_fmt_uptime "$SYS_AGE") │ ~Cycle: $(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))"
|
||||
line "⚙️ System Watchdog (cron via watchdog_orchestrator)"
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
_sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||||
_sw_ago=$(( NOW - _sw_last ))
|
||||
if [[ "$_sw_ago" -lt 600 ]]; then
|
||||
line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago"
|
||||
else
|
||||
issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running"
|
||||
fi
|
||||
else
|
||||
issue "system_watchdog NOT RUNNING"
|
||||
issue " system_watchdog has never run (state file missing)"
|
||||
fi
|
||||
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=\|^mem_shutdown" \
|
||||
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \
|
||||
"$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$SYS_ACTIVE" ]]; then
|
||||
while IFS=: read -r key count; do
|
||||
@@ -519,13 +523,17 @@ line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB
|
||||
REPORT+=("")
|
||||
|
||||
# ── Docker Watchdog ───────────────────────────────────────────────────────────────────────────
|
||||
line "🐳 Docker Watchdog"
|
||||
DOCKER_PID=$(_get_lock_pid "docker_watchdog")
|
||||
if _is_running "docker_watchdog"; then
|
||||
DOCKER_AGE=$(_lock_age "docker_watchdog")
|
||||
line " ✅ Running │ PID: $DOCKER_PID │ Uptime: $(_fmt_uptime "$DOCKER_AGE") │ ~Cycle: $(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))"
|
||||
line "🐳 Docker Watchdog (cron via watchdog_orchestrator)"
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
_dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||||
_dw_ago=$(( NOW - _dw_last ))
|
||||
if [[ "$_dw_ago" -lt 600 ]]; then
|
||||
line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago"
|
||||
else
|
||||
issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running"
|
||||
fi
|
||||
else
|
||||
issue "docker_watchdog NOT RUNNING"
|
||||
issue " docker_watchdog has never run (state file missing)"
|
||||
fi
|
||||
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
@@ -554,6 +562,26 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Resource Manager ─────────────────────────────────────────────────────────────────────────
|
||||
line "🎛️ Resource Manager"
|
||||
if [[ -f "$RM_STATE_FILE" ]]; then
|
||||
_rm_last=$(stat -c %Y "$RM_STATE_FILE" 2>/dev/null || echo 0)
|
||||
_rm_ago=$(( NOW - _rm_last ))
|
||||
_rm_level=$(grep "^current_level:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2)
|
||||
_rm_level="${_rm_level:-0}"
|
||||
if [[ "$_rm_level" -gt 0 ]]; then
|
||||
issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||||
elif [[ "$_rm_ago" -lt 600 ]]; then
|
||||
line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||||
else
|
||||
issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running"
|
||||
fi
|
||||
else
|
||||
line " ℹ️ State file not found (resource_manager may not have run yet)"
|
||||
fi
|
||||
|
||||
REPORT+=("")
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||||
|
||||
Reference in New Issue
Block a user