From 309546e6152a2de9b3faf6292feebba3e485e9f5 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Tue, 12 May 2026 17:41:59 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20watchdog=20architecture=20v2=20?= =?UTF-8?q?=E2=80=94=20resource=20manager=20+=20single-pass=20orchestrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 --- Docker_Essentials/docker_watchdog.sh | 113 ++-- Orchestrators/sunday_morning_coffee_report.sh | 56 +- Orchestrators/watchdog_orchestrator.sh | 192 +++++++ .../continuous_scripts_status.sh | 0 common.sh | 2 + master.conf | 122 ++-- master_host1.conf | 26 + unRAID_Essentials/resource_manager.sh | 523 ++++++++++++++++++ unRAID_Essentials/system_watchdog.sh | 168 +----- 9 files changed, 925 insertions(+), 277 deletions(-) create mode 100755 Orchestrators/watchdog_orchestrator.sh rename {Monitors => Tools}/continuous_scripts_status.sh (100%) create mode 100755 unRAID_Essentials/resource_manager.sh diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index 7fb0414..ef03814 100755 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -97,8 +97,8 @@ if [[ "$EUID" -ne 0 ]]; then fi success "Running as root" -# Continuous mode — skip gracefully if healthy instance already running -acquire_lock "continuous" +# Skip if another instance is running — no pile-up during long operations +acquire_lock # detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays detect_hosts @@ -332,8 +332,11 @@ is_parity_running() { WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying -WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers -WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session +# Loaded from state file — persists across single-pass runs +WATCHDOG_DAEMON_STRIKES=$(get_strikes "daemon_strikes" "$WATCHDOG_STATE_FILE") +WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES//[^0-9]/}"; WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES:-0}" +_dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE") +[[ "$_dr_raw" == "true" ]] && WATCHDOG_DAEMON_RESTARTED=true || WATCHDOG_DAEMON_RESTARTED=false # ============================================================================================== # ── SYSTEM WATCHDOG COORDINATION ────────────────────────────────────────────────────────────── @@ -348,10 +351,10 @@ WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart thi # 1 = RAM emergency active — defer container management this cycle check_system_watchdog_state() { - # Returns 0 = normal operation | 1 = defer, RAM emergency active - local state_file="$SYS_WATCHDOG_STATE_FILE" + # Returns 0 = normal operation | 1 = defer, resource_manager RAM emergency active + local state_file="$RM_STATE_FILE" - # No state file = system_watchdog not running or not yet written — assume normal + # No state file = resource_manager not yet run — assume normal [[ ! -f "$state_file" ]] && return 0 local mem_shutdown @@ -361,7 +364,7 @@ check_system_watchdog_state() { # ── Stale state guard ───────────────────────────────────────────────────────────────────── # If mem_shutdown_active=true but state file hasn't been updated in > 2 hours, - # system_watchdog.sh may have died — don't be silenced forever by a stale flag. + # resource_manager.sh may not be running — don't defer indefinitely on stale state. local state_mtime now age_seconds stale_limit=7200 # 2 hours state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0) now=$(date +%s) @@ -369,8 +372,7 @@ check_system_watchdog_state() { if [[ "$age_seconds" -gt "$stale_limit" ]]; then warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale" - warn "system_watchdog.sh may not be running — resuming normal container management" - warn "If RAM is still low this will be caught on next system_watchdog.sh cycle" + warn "resource_manager.sh may not be running — resuming normal container management" return 0 # Resume normal — don't defer indefinitely on stale state fi @@ -385,11 +387,14 @@ check_docker_daemon() { queue_notify "Docker daemon recovered on $(hostname)" "normal" WATCHDOG_DAEMON_STRIKES=0 WATCHDOG_DAEMON_RESTARTED=false + set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE" + set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE" fi return 0 fi WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 )) + set_strikes "daemon_strikes" "$WATCHDOG_DAEMON_STRIKES" "$WATCHDOG_STATE_FILE" warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)" if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then @@ -415,6 +420,7 @@ check_docker_daemon() { # Restart daemon — unRAID uses rc.d scripts, not systemd WATCHDOG_DAEMON_RESTARTED=true + set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE" if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..." sleep "$WATCHDOG_DAEMON_RESTART_WAIT" @@ -424,6 +430,8 @@ check_docker_daemon() { notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal" WATCHDOG_DAEMON_STRIKES=0 WATCHDOG_DAEMON_RESTARTED=false + set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE" + set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE" return 0 else error "Docker daemon did not recover after restart" @@ -440,38 +448,14 @@ check_docker_daemon() { } # ============================================================================================== -# ── CLEAN SHUTDOWN ──────────────────────────────────────────────────────────────────────────── +# ━━━ Single-Pass Monitoring Run ━━━ # ============================================================================================== -WATCHDOG_RUNNING=true - -cleanup() { - echo "" - warn "Docker watchdog received shutdown signal — stopping cleanly" - WATCHDOG_RUNNING=false - exit 0 -} - -trap cleanup SIGTERM SIGINT - -# ============================================================================================== -# ━━━ Continuous Monitoring Loop ━━━ -# ============================================================================================== -info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s" +info "$ICON_WATCHDOG Docker watchdog — $MY_ID — $(date '+%H:%M:%S')" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -CYCLE=0 +CYCLE_START=$(date +%s) -while [[ "$WATCHDOG_RUNNING" == true ]]; do - ((CYCLE++)) - CYCLE_START=$(date +%s) - - # ── Re-source config each cycle ────────────────────────────────────────────────────────── - # Picks up config changes (new containers, threshold adjustments) without restart. - # detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source. - source "$SCRIPT_DIR/../load_config.sh" - detect_hosts - - # ── Per-cycle state — cleared each iteration ────────────────────────────────────────────── + # ── Per-run state ───────────────────────────────────────────────────────────────────────── NOTIFY_EVENTS=() T1_RESTARTS=0 T1_WARNINGS=0 @@ -488,33 +472,28 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 done - # ── Docker daemon health check — first check every cycle ──────────────────────────────── - # If daemon is hung all container operations will fail — check first, skip cycle if down + # ── Docker daemon health check — first check every run ────────────────────────────────── + # If daemon is hung all container operations will fail — check first, skip run if down if ! check_docker_daemon; then - sleep "$DOCKER_WATCHDOG_INTERVAL" - continue + exit 0 fi # ── Parity check — skip restarts during parity ─────────────────────────────────────────── if is_parity_running; then - log "Parity check in progress — skipping restart actions this cycle" - sleep "$DOCKER_WATCHDOG_INTERVAL" - continue + log "Parity check in progress — skipping restart actions this run" + exit 0 fi - # ── RAM emergency check — system_watchdog.sh managing containers ─────────────────────────── - # If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container - # management this cycle. Docker daemon health checks continue — system still needs - # monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown. + # ── RAM emergency check — resource_manager.sh managing containers ──────────────────────── + # If resource_manager.sh has triggered a hard RAM shutdown, defer all container + # management this run to prevent undoing the emergency stop and re-pressuring RAM. if ! check_system_watchdog_state; then MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 )) - warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers" - warn "Deferring all container restart logic this cycle" - log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming" - sleep "$DOCKER_WATCHDOG_INTERVAL" & - wait $! - continue + warn "RAM emergency active (${MEM_GB}GB free) — resource_manager.sh managing containers" + warn "Deferring all container restart logic this run" + log "Waiting for RAM to recover above ${RM_RAM_RECOVER_GB:-20}GB before resuming" + exit 0 fi # ── Startup grace period ────────────────────────────────────────────────────────────────── @@ -815,29 +794,11 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then echo "" - echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "━━━ $ICON_WATCHDOG Docker Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings" echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings" echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else - log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))" - - # Heartbeat — periodic proof of life even when everything is healthy - if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then - HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 )) - UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL )) - if [[ "$HB_SECONDS" -gt 0 ]] && \ - (( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \ - [[ "$UPTIME_APPROX" -gt 0 ]]; then - HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 )) - info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))" - fi - fi - fi - - # Sleep until next cycle — interruptible by SIGTERM - sleep "$DOCKER_WATCHDOG_INTERVAL" & - wait $! - -done \ No newline at end of file + log "All healthy ✅ ($(date '+%H:%M:%S'))" + fi \ No newline at end of file diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 541f308..057de3a 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -14,7 +14,7 @@ # 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split # 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth # 🌐 Rsync — weekly transfer totals, per-share breakdown, failures -# 🛡️ Watchdog — system watchdog, docker watchdog, fallback state +# 🛡️ Watchdog — resource manager, system watchdog, docker watchdog, fallback state # 🔐 Security — SSL cert expiry per domain # 📊 Emby — weekly stream count, active now, top users # ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync @@ -468,17 +468,21 @@ fi section "🛡️ WATCHDOG" # ── System Watchdog ─────────────────────────────────────────────────────────────────────────── -line "⚙️ System Watchdog" -SYS_PID=$(_get_lock_pid "system_watchdog") -if _is_running "system_watchdog"; then - SYS_AGE=$(_lock_age "system_watchdog") - line " ✅ Running │ PID: $SYS_PID │ Uptime: $(_fmt_uptime "$SYS_AGE") │ ~Cycle: $(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))" +line "⚙️ System Watchdog (cron via watchdog_orchestrator)" +if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then + _sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0) + _sw_ago=$(( NOW - _sw_last )) + if [[ "$_sw_ago" -lt 600 ]]; then + line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago" + else + issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running" + fi else - issue "system_watchdog NOT RUNNING" + issue " system_watchdog has never run (state file missing)" fi if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=\|^mem_shutdown" \ + SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \ "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$SYS_ACTIVE" ]]; then while IFS=: read -r key count; do @@ -519,13 +523,17 @@ line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB REPORT+=("") # ── Docker Watchdog ─────────────────────────────────────────────────────────────────────────── -line "🐳 Docker Watchdog" -DOCKER_PID=$(_get_lock_pid "docker_watchdog") -if _is_running "docker_watchdog"; then - DOCKER_AGE=$(_lock_age "docker_watchdog") - line " ✅ Running │ PID: $DOCKER_PID │ Uptime: $(_fmt_uptime "$DOCKER_AGE") │ ~Cycle: $(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))" +line "🐳 Docker Watchdog (cron via watchdog_orchestrator)" +if [[ -f "$WATCHDOG_STATE_FILE" ]]; then + _dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0) + _dw_ago=$(( NOW - _dw_last )) + if [[ "$_dw_ago" -lt 600 ]]; then + line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago" + else + issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running" + fi else - issue "docker_watchdog NOT RUNNING" + issue " docker_watchdog has never run (state file missing)" fi if [[ -f "$WATCHDOG_STATE_FILE" ]]; then @@ -554,6 +562,26 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then fi fi +# ── Resource Manager ───────────────────────────────────────────────────────────────────────── +line "🎛️ Resource Manager" +if [[ -f "$RM_STATE_FILE" ]]; then + _rm_last=$(stat -c %Y "$RM_STATE_FILE" 2>/dev/null || echo 0) + _rm_ago=$(( NOW - _rm_last )) + _rm_level=$(grep "^current_level:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2) + _rm_level="${_rm_level:-0}" + if [[ "$_rm_level" -gt 0 ]]; then + issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago" + elif [[ "$_rm_ago" -lt 600 ]]; then + line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago" + else + issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running" + fi +else + line " ℹ️ State file not found (resource_manager may not have run yet)" +fi + +REPORT+=("") + if command -v docker >/dev/null 2>&1; then RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh new file mode 100755 index 0000000..e1cc2a4 --- /dev/null +++ b/Orchestrators/watchdog_orchestrator.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# ============================================================================================== +# ============================ Watchdog Orchestrator =========================================== +# ============================================================================================== +# Runs the three-layer watchdog system in the correct sequence each cycle. +# Schedule: * * * * * (every minute via User Scripts plugin) +# +# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── +# 1. resource_manager.sh — reduce system pressure intelligently +# 2. docker_watchdog.sh — heal containers with freed resources +# 3. system_watchdog.sh — reboot if all else fails (last line of defense) +# +# ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── +# Resource Manager first — frees RAM and CPU before healing attempts container restarts. +# Containers restarted into a resource-pressured system just fail again. +# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise. +# System Watchdog last — only triggers if prior layers could not resolve the issue. +# Rebooting without first reducing pressure may reboot into the same state. +# +# ── STARTUP GRACE ───────────────────────────────────────────────────────────────────────────── +# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. +# Prevents false positives from containers still starting at array launch. +# Each sub-script enforces this independently — orchestrator exits early to avoid log noise. +# +# ── OVERLAP PROTECTION ──────────────────────────────────────────────────────────────────────── +# acquire_lock() — exits immediately if a prior cycle is still in progress. +# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.). +# +# ── REPLACES ────────────────────────────────────────────────────────────────────────────────── +# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh. +# Those scripts are now single-pass — this orchestrator provides the cadence. +# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS. +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate +# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle +# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# watchdog_orchestrator.sh — normal run (called by cron every minute) +# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts +# watchdog_orchestrator.sh --status — show script paths and current grace state +# watchdog_orchestrator.sh --log — verbose output from all sub-scripts +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +source "$ECOSYSTEM_ROOT/load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +# Skip immediately if another cycle is still running — no pile-up +acquire_lock + +detect_hosts + +RESOURCE_MANAGER="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_manager.sh" +DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh" +SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts" + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + + UPTIME_S=$(awk '{print int($1)}' /proc/uptime) + if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then + warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)" + else + log "Past startup grace — $(format_duration $UPTIME_S) uptime" + fi + + echo "" + echo "── Sub-scripts ──" + for pair in \ + "Resource Manager:$RESOURCE_MANAGER" \ + "Docker Watchdog:$DOCKER_WATCHDOG" \ + "System Watchdog:$SYSTEM_WATCHDOG"; do + label="${pair%%:*}" + script="${pair#*:}" + if [[ -f "$script" ]]; then + [[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN" + echo " $icon $label — ${script##*/}" + else + echo " $ICON_ERROR $label — NOT FOUND: $script" + fi + done + + echo "" + echo " Schedule: * * * * * (every minute via User Scripts)" + echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Startup Grace ━━━ +# ============================================================================================== +UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) +if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then + log "Startup grace — ${UPTIME_SECONDS}s / ${WATCHDOG_STARTUP_GRACE}s — skipping cycle" + exit 0 +fi + +# ============================================================================================== +# ━━━ Run Watchdog Cycle ━━━ +# ============================================================================================== +CYCLE_START=$(date +%s) +PASS=() +FAIL=() + +run_watchdog() { + local name="$1" script="$2" + + if [[ ! -f "$script" ]]; then + error "$name — not found: $script" + FAIL+=("$name:missing") + return 1 + fi + + [[ ! -x "$script" ]] && chmod +x "$script" + + local extra_args=() + [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run") + [[ "$VERBOSE" == true ]] && extra_args+=("--log") + + log "$ICON_START $name" + if bash "$script" "${extra_args[@]}"; then + PASS+=("$name") + return 0 + else + error "$name — non-zero exit" + FAIL+=("$name") + return 1 + fi +} + +run_watchdog "Resource Manager" "$RESOURCE_MANAGER" +run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG" +run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG" + +CYCLE_END=$(date +%s) +DURATION=$(( CYCLE_END - CYCLE_START )) + +# ============================================================================================== +# ━━━ Heartbeat ━━━ +# ============================================================================================== +if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then + HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 )) + HB_COUNT_FILE="/tmp/watchdog_orch_hb.count" + HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0) + HB_COUNT=$(( HB_COUNT + 1 )) + echo "$HB_COUNT" > "$HB_COUNT_FILE" + # Each cron run = ~60s — use count × 60 as uptime approximation + HB_ELAPSED=$(( HB_COUNT * 60 )) + if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then + HB_HR=$(( HB_ELAPSED / 3600 )) + warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))" + fi +fi + +# ============================================================================================== +# ━━━ Summary — only shown on failures or --log ━━━ +# ============================================================================================== +if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━" + for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done + for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done + echo "$ICON_TIME Duration: $(format_duration $DURATION)" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + if [[ "${#FAIL[@]}" -gt 0 ]]; then + notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \ + "Watchdog Orchestrator" "warning" + fi +fi diff --git a/Monitors/continuous_scripts_status.sh b/Tools/continuous_scripts_status.sh similarity index 100% rename from Monitors/continuous_scripts_status.sh rename to Tools/continuous_scripts_status.sh diff --git a/common.sh b/common.sh index 05dbdc6..7b99e5e 100644 --- a/common.sh +++ b/common.sh @@ -572,6 +572,8 @@ detect_hosts() { _alias_array "PARTNERSHIP_AUTH_WEBUIS" _alias_array "PARTNERSHIP_MIRROR_BACKUPS" _alias_array "PARTNERSHIP_OWN_CONTAINERS" + _alias_array "RM_PAUSE_CONTAINERS" + _alias_array "RM_STOP_CONTAINERS" # ── Set array aliases — associative arrays ──────────────────────────────── # Associative arrays cannot be copied with eval — must be rebuilt key by key diff --git a/master.conf b/master.conf index 28ac553..c3e2b4b 100644 --- a/master.conf +++ b/master.conf @@ -278,7 +278,9 @@ # Scripts launched by array_started.sh when the array comes online. # Launched in order — each as a background process. # One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. -# Continuous scripts (watchdogs, failover) run until array stops. +# Continuous scripts (failover) run until array stops. +# Watchdogs (resource_manager, docker_watchdog, system_watchdog) are cronned via +# watchdog_orchestrator.sh — NOT launched here. ARRAY_START_SCRIPTS=( "git_pull_execute.sh" # pull latest scripts before anything starts "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts @@ -286,9 +288,7 @@ "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers - "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop - "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop - "Fallback/fallback.sh" # mutual failover — HOST2 back online + "Fallback/fallback.sh" # mutual failover — continuous ) # ━━━ Intermediate Sync Maintenance ━━━ @@ -1114,12 +1114,85 @@ EMBY_REPORT_DAYS=7 # days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show +# ============================================================================================== +# ── WATCHDOG ORCHESTRATOR ───────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Runs resource_manager → docker_watchdog → system_watchdog in sequence each cron cycle. +# Schedule: * * * * * (every minute via User Scripts plugin) +# NOT in ARRAY_START_SCRIPTS — has its own cron entry. + + WATCHDOG_ORCHESTRATOR_HEARTBEAT=true + WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1 + +# ============================================================================================== +# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Pressure reduction layer — keeps the system comfortable before things break. +# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop. +# +# ── PRESSURE LEVELS ─────────────────────────────────────────────────────────────────────────── +# Level 1 (soft) — throttle SABnzbd + qBit download speeds +# Level 2 (medium) — further throttle + docker pause background containers +# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer +# +# ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── +# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (in master_host*.conf) +# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (in master_host*.conf) + + RM_ENABLED=true + RM_STATE_FILE="/tmp/resource_manager_state.db" + +# ━━━ Pressure Thresholds ━━━ +# Graduated RAM response — resource_manager acts before system_watchdog reboots. +# RM_RAM_SOFT_GB > RM_RAM_MEDIUM_GB > RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always + RM_RAM_SOFT_GB=12 # throttle start — reduce background load + RM_RAM_MEDIUM_GB=8 # pause background containers + RM_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) + RM_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers + +# Load average thresholds — multiplier × core count + RM_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained + RM_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained + +# Consecutive runs at lower pressure before de-escalating + RM_RECOVER_CYCLES=3 + +# ━━━ SABnzbd Throttle ━━━ +# Speed values: "50M" = 50 MB/s, "0" = unlimited + RM_SABNZBD_ENABLED=true + RM_SABNZBD_SPEED_SOFT="50M" + RM_SABNZBD_SPEED_MEDIUM="10M" + +# ━━━ qBittorrent Throttle ━━━ +# KB/s — 0 = unlimited + RM_QBIT_ENABLED=true + RM_QBIT_DL_SOFT=51200 # 50 MB/s + RM_QBIT_DL_MEDIUM=10240 # 10 MB/s + +# ━━━ Critical Containers ━━━ +# Never paused or stopped regardless of pressure level. +# Keep DNS, auth, media serving, and live TV always running. + RM_CRITICAL_CONTAINERS=( + "NginxProxyManager" # reverse proxy — internet access + "Authelia" # auth — nothing accessible without it + "Authelia-Secondary" + "Mariadb-Authelia" # Authelia dependency + "Mariadb-Authelia-Secondary" + "Redis-Authelia" # Authelia dependency + "Redis-Authelia-Secondary" + "AdGuard-Home" # DNS — all LAN resolution + "Emby" # media server — Live TV buffering + "Dispatcharr" # Live TV scheduler — loses state if stopped + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + ) + # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Continuous system health monitoring — last line of defense before a crash. -# Started by array_started.sh — runs until array stops. -# Re-sources all three conf files each cycle — config changes take effect on next cycle. +# Single-pass system health check — last line of defense before a crash. +# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh. +# Re-sources config at each orchestrator run — config changes take effect immediately. # # ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── # CRITICAL — bypass ALL strikes, reboot immediately @@ -1132,16 +1205,9 @@ # STANDARD — strike system (N consecutive failures → reboot) # RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers # -# ── RAM TIERS ───────────────────────────────────────────────────────────────────────────────── -# MEM_WARN_GB — warn + notify only (informational) -# MEM_SHUTDOWN_GB — stop non-essential containers, recover above MEM_RECOVER_GB -# MEM_GB — strike system → reboot (or bypass with OOM) -# -# ── CONTAINER SHUTDOWN ──────────────────────────────────────────────────────────────────────── -# At MEM_SHUTDOWN_GB: stop all containers NOT in MEM_SHUTDOWN_EXCLUDED list -# Excluded containers stay running — DNS, auth, Emby, Dispatcharr -# Stopped containers stay stopped until RAM recovers above MEM_RECOVER_GB -# Strike list applied — doesn't flip-flop every cycle +# ── RAM ─────────────────────────────────────────────────────────────────────────────────────── +# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass) +# Warn/shutdown/recover RAM tiers are handled by resource_manager.sh # ━━━ State Files ━━━ SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅ @@ -1167,26 +1233,10 @@ SYSTEM_WATCHDOG_HEARTBEAT=true SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 -# ━━━ RAM Tiers ━━━ -# Three-level RAM response — graduated action instead of single threshold. -# HOST1 has 128GB, HOST2 has 64GB — adjust accordingly. -# MEM_WARN_GB > MEM_SHUTDOWN_GB > MEM_GB always - SYS_WATCHDOG_MEM_WARN_GB=10 # warn + notify — informational only - SYS_WATCHDOG_MEM_SHUTDOWN_GB=6 # stop non-essential containers +# ━━━ RAM Reboot Threshold ━━━ +# Reboot trigger only — warn/shutdown/recover handled by resource_manager.sh +# RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) SYS_WATCHDOG_MEM_GB=4 # strike system → reboot - SYS_WATCHDOG_MEM_RECOVER_GB=30 # RAM must recover above this before restarting containers - -# Containers excluded from RAM emergency shutdown. -# These stay running regardless of RAM pressure. -# DNS and auth must stay up, Emby and Dispatcharr for Live TV continuity. - SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=( - "NginxProxyManager" # DNS / reverse proxy — internet access - "Authelia" # auth — without this nothing is accessible - "Mariadb" # Authelia dependency - "Redis" # Authelia dependency - "Emby" # media server — Live TV buffering - "Dispatcharr" # Live TV scheduler — loses state if stopped - ) # ━━━ OOM Bypass Settings ━━━ # OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold diff --git a/master_host1.conf b/master_host1.conf index 5b156a1..2a819bd 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -670,6 +670,32 @@ # Enable only if HOST1 has no CPU-intensive workloads. HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false +# ============================================================================================== +# ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Containers to manage under pressure — see master.conf RM_CRITICAL_CONTAINERS for exclusions. + +# docker pause at medium pressure (RAM < RM_RAM_MEDIUM_GB or load > medium threshold) +# Suspended in-place — instant to pause/unpause, no state lost, no restart delay. + HOST1_RM_PAUSE_CONTAINERS=( + "Huntarr" # arr search automation — safe to suspend + "Cleanuparr" # download cleanup — safe to suspend + "Healarr" # arr health checks — safe to suspend + "Soularr" # Slskd automation — background only + "ChannelTube" # YouTube archiver — background only + "Pinchflat" # YouTube archiver — background only + ) + +# docker stop at hard pressure (RAM < RM_RAM_HARD_GB) +# Full stop — these are optional/heavy services that free significant RAM when stopped. +# resource_manager.sh restarts them when pressure fully clears (RAM >= RM_RAM_RECOVER_GB). + HOST1_RM_STOP_CONTAINERS=( + "LocalAI" # GPU/CPU heavy — largest RAM consumer when idle + "7DaysToDie" # game server — optional + "V-Rising" # game server — optional + "Code-Server" # IDE — not needed during pressure events + ) + # ============================================================================================== # ──────────────────────── End Of HOST1 Variables ────────────────────────────────────────────── # ============================================================================================== \ No newline at end of file diff --git a/unRAID_Essentials/resource_manager.sh b/unRAID_Essentials/resource_manager.sh new file mode 100755 index 0000000..5771136 --- /dev/null +++ b/unRAID_Essentials/resource_manager.sh @@ -0,0 +1,523 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Resource Manager =========================================== +# ============================================================================================== +# Pressure reduction layer — detects rising system load and reduces it intelligently. +# Called by watchdog_orchestrator.sh every minute — single-pass, not a continuous loop. +# +# ── RESPONSIBILITY ──────────────────────────────────────────────────────────────────────────── +# Reduce system pressure before things break. NOT fixing broken containers (docker_watchdog) +# and NOT rebooting (system_watchdog). The middle layer that keeps the system comfortable. +# +# "Pressure is rising — reduce load intelligently." +# +# ── THREE-LEVEL PRESSURE RESPONSE ───────────────────────────────────────────────────────────── +# +# Level 1 — SOFT (RAM < RM_RAM_SOFT_GB OR load > RM_LOAD_SOFT_MULTIPLIER × cores): +# Throttle SABnzbd download speed to RM_SABNZBD_SPEED_SOFT +# Throttle qBittorrent download to RM_QBIT_DL_SOFT KB/s +# +# Level 2 — MEDIUM (RAM < RM_RAM_MEDIUM_GB OR load > RM_LOAD_MEDIUM_MULTIPLIER × cores): +# Further throttle SABnzbd + qBittorrent to medium limits +# docker pause RM_PAUSE_CONTAINERS — suspend without losing state, instant reversible +# +# Level 3 — HARD (RAM < RM_RAM_HARD_GB): +# docker stop RM_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.) +# Write mem_shutdown_active=true — signals docker_watchdog to defer container restarts +# +# ── RECOVERY ────────────────────────────────────────────────────────────────────────────────── +# Pressure must stay below current action threshold for RM_RECOVER_CYCLES consecutive runs +# before restoring. De-escalates one level at a time to avoid re-triggering immediately. +# Level 3 de-escalation additionally requires RAM >= RM_RAM_RECOVER_GB before un-stopping. +# +# ── COORDINATION WITH DOCKER WATCHDOG ───────────────────────────────────────────────────────── +# At level 3: writes mem_shutdown_active=true to RM_STATE_FILE. +# docker_watchdog.sh reads this and defers all container restart logic. +# Cleared when pressure fully resolves and containers are restarted. +# This prevents docker_watchdog from restarting containers that RM just stopped to free RAM. +# +# ── NOT RESPONSIBLE FOR ─────────────────────────────────────────────────────────────────────── +# Restarting broken containers — docker_watchdog.sh +# Rebooting the system — system_watchdog.sh +# Reacting to single data points — RM_RECOVER_CYCLES prevents flip-flopping +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# RM_ENABLED, RM_STATE_FILE +# RM_RAM_SOFT_GB, RM_RAM_MEDIUM_GB, RM_RAM_HARD_GB, RM_RAM_RECOVER_GB +# RM_LOAD_SOFT_MULTIPLIER, RM_LOAD_MEDIUM_MULTIPLIER +# RM_RECOVER_CYCLES +# RM_SABNZBD_ENABLED, RM_SABNZBD_SPEED_SOFT, RM_SABNZBD_SPEED_MEDIUM +# RM_QBIT_ENABLED, RM_QBIT_DL_SOFT, RM_QBIT_DL_MEDIUM +# RM_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure +# +# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── +# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts) +# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts) +# HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY +# HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# resource_manager.sh — normal run (via watchdog_orchestrator.sh) +# resource_manager.sh --dry-run — show what would happen without acting +# resource_manager.sh --status — current pressure level and active actions +# resource_manager.sh --log — verbose per-check output +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +if [[ "${RM_ENABLED:-true}" != "true" ]]; then + log "Resource Manager disabled (RM_ENABLED=false)" + exit 0 +fi + +acquire_lock + +detect_hosts + +DOCKER_TIMEOUT=15 + +touch "$RM_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $RM_STATE_FILE" + exit 1 +} + +# ============================================================================================== +# ━━━ State Helpers ━━━ +# ============================================================================================== +# rm_state_get/set use : separator for RM-internal state +# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags + +rm_state_get() { + grep -E "^${1}:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2- +} + +rm_state_set() { + local key="$1" val="$2" + grep -vE "^${key}:" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp" + echo "${key}:${val}" >> "${RM_STATE_FILE}.tmp" + mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE" +} + +rm_state_get_eq() { + grep -E "^${1}=" "$RM_STATE_FILE" 2>/dev/null | cut -d= -f2- +} + +rm_state_set_eq() { + local key="$1" val="$2" + grep -vE "^${key}=" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp" + echo "${key}=${val}" >> "${RM_STATE_FILE}.tmp" + mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE" +} + +# ============================================================================================== +# ━━━ Load State ━━━ +# ============================================================================================== +CURRENT_LEVEL=$(rm_state_get "rm_action_level"); CURRENT_LEVEL=${CURRENT_LEVEL:-0} +RECOVER_CYCLES=$(rm_state_get "rm_recover_cycles"); RECOVER_CYCLES=${RECOVER_CYCLES:-0} +PAUSED_LIST=$(rm_state_get "rm_paused_containers"); PAUSED_LIST=${PAUSED_LIST:-""} +STOPPED_LIST=$(rm_state_get "rm_stopped_containers"); STOPPED_LIST=${STOPPED_LIST:-""} + +# ============================================================================================== +# ━━━ Pressure Calculation ━━━ +# ============================================================================================== +TOTAL_CORES=$(nproc) +MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) +MEM_GB=$(( MEM_KB / 1024 / 1024 )) +LOAD=$(awk '{print $1}' /proc/loadavg) +LOAD_INT=$(printf "%.0f" "$LOAD") +RM_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_SOFT_MULTIPLIER:-2.0}}") +RM_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_MEDIUM_MULTIPLIER:-3.0}}") + +TARGET_LEVEL=0 +TARGET_REASON="" +if [[ "$MEM_GB" -lt "${RM_RAM_HARD_GB:-6}" ]]; then + TARGET_LEVEL=3 + TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RM_RAM_HARD_GB}GB" +elif [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]]; then + TARGET_LEVEL=2 + [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RM_RAM_MEDIUM_GB}GB" + [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RM_LOAD_MEDIUM_THRESH}" +elif [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]]; then + TARGET_LEVEL=1 + [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RM_RAM_SOFT_GB}GB" + [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RM_LOAD_SOFT_THRESH}" +fi + +LEVEL_NAMES=("normal" "soft" "medium" "hard") + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY RESOURCE MANAGER STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + echo "── Current State ──" + echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})" + echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})" + echo " Recover cycles: $RECOVER_CYCLES / ${RM_RECOVER_CYCLES:-3}" + [[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST" + [[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST" + MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active") + [[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)" + echo "" + echo "── System Pressure ──" + echo " RAM free: ${MEM_GB}GB (soft:<${RM_RAM_SOFT_GB} medium:<${RM_RAM_MEDIUM_GB} hard:<${RM_RAM_HARD_GB} recover:>=${RM_RAM_RECOVER_GB})" + echo " Load avg: ${LOAD} (soft:>=${RM_LOAD_SOFT_THRESH} medium:>=${RM_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})" + echo "" + echo "── Configuration ──" + echo " SABnzbd throttle: ${RM_SABNZBD_ENABLED:-true} soft=${RM_SABNZBD_SPEED_SOFT} medium=${RM_SABNZBD_SPEED_MEDIUM}" + echo " qBit throttle: ${RM_QBIT_ENABLED:-true} soft=${RM_QBIT_DL_SOFT}KB/s medium=${RM_QBIT_DL_MEDIUM}KB/s" + echo "" + echo "── Container Lists (this host) ──" + echo " Pause at medium: ${RM_PAUSE_CONTAINERS[*]:-none configured}" + echo " Stop at hard: ${RM_STOP_CONTAINERS[*]:-none configured}" + echo " Critical (never touched): ${RM_CRITICAL_CONTAINERS[*]:-none}" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Critical Container Guard ━━━ +# ============================================================================================== +# Returns 0 if container is safe to pause/stop, 1 if it is critical +is_critical() { + local container="$1" + for c in "${RM_CRITICAL_CONTAINERS[@]:-}"; do + [[ "$c" == "$container" ]] && return 1 + done + return 0 +} + +# ============================================================================================== +# ━━━ SABnzbd API ━━━ +# ============================================================================================== +sabnzbd_set_speed() { + local speed="$1" + [[ "${RM_SABNZBD_ENABLED:-true}" != "true" ]] && return 0 + [[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0 + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set SABnzbd speed to $speed" + return 0 + fi + + curl -sf --max-time 10 \ + "${SABNZBD_URL}/api?mode=config&name=speedlimit&value=${speed}&apikey=${SABNZBD_API_KEY}" \ + >/dev/null 2>&1 && log "SABnzbd speed → $speed" || warn "SABnzbd API call failed" +} + +# ============================================================================================== +# ━━━ qBittorrent API ━━━ +# ============================================================================================== +QBIT_COOKIE="/tmp/rm_qbit_cookie.txt" + +qbit_login() { + [[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0 + [[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0 + + curl -sf --max-time 10 -c "$QBIT_COOKIE" \ + -X POST "${QBIT_URL}/api/v2/auth/login" \ + -d "username=${QBIT_USERNAME}&password=${QBIT_PASSWORD}" >/dev/null 2>&1 +} + +qbit_set_dl_limit() { + local kbps="$1" # KB/s — 0 = unlimited + [[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0 + [[ -z "$QBIT_URL" ]] && return 0 + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set qBit download limit to ${kbps}KB/s" + return 0 + fi + + local bps=$(( kbps * 1024 )) + qbit_login + curl -sf --max-time 10 -b "$QBIT_COOKIE" \ + -X POST "${QBIT_URL}/api/v2/transfer/setDownloadLimit" \ + -d "limit=${bps}" >/dev/null 2>&1 && log "qBit download limit → ${kbps}KB/s" || warn "qBit API call failed" +} + +# ============================================================================================== +# ━━━ Container Actions ━━━ +# ============================================================================================== + +# Pause a list of containers — returns newline-separated list of actually-paused containers +pause_containers() { + local actually_paused=() + for container in "$@"; do + [[ -z "$container" ]] && continue + is_critical "$container" || { log "$container — critical, skipping pause"; continue; } + + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null) + if [[ "$status" != "running" ]]; then + log "$container — not running (status: ${status:-unknown}), skipping pause" + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would docker pause $container" + actually_paused+=("$container") + continue + fi + + if timeout "$DOCKER_TIMEOUT" docker pause "$container" >/dev/null 2>&1; then + warn "Paused $container (medium pressure)" + actually_paused+=("$container") + else + error "Failed to pause $container" + fi + done + printf '%s,' "${actually_paused[@]}" | sed 's/,$//' +} + +# Unpause a comma-separated list of containers +unpause_containers() { + local IFS=',' + for container in $1; do + [[ -z "$container" ]] && continue + + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null) + if [[ "$status" != "paused" ]]; then + log "$container — not paused (status: ${status:-unknown}), skipping unpause" + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would docker unpause $container" + continue + fi + + if timeout "$DOCKER_TIMEOUT" docker unpause "$container" >/dev/null 2>&1; then + warn "Unpaused $container (pressure reduced)" + else + error "Failed to unpause $container" + fi + done +} + +# Stop a list of containers — returns comma-separated list of actually-stopped containers +stop_containers() { + local actually_stopped=() + for container in "$@"; do + [[ -z "$container" ]] && continue + is_critical "$container" || { log "$container — critical, skipping stop"; continue; } + + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null) + if [[ "$status" != "running" && "$status" != "paused" ]]; then + log "$container — not running (status: ${status:-unknown}), skipping stop" + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would docker stop $container" + actually_stopped+=("$container") + continue + fi + + if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then + warn "Stopped $container (hard pressure)" + actually_stopped+=("$container") + else + error "Failed to stop $container" + fi + done + printf '%s,' "${actually_stopped[@]}" | sed 's/,$//' +} + +# Start a comma-separated list of containers (only those RM stopped) +start_containers() { + local IFS=',' + for container in $1; do + [[ -z "$container" ]] && continue + + local status + status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null) + if [[ "$status" == "running" ]]; then + log "$container — already running" + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would docker start $container" + continue + fi + + if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then + warn "Started $container (pressure cleared)" + else + error "Failed to start $container" + fi + done +} + +# ============================================================================================== +# ━━━ Apply Level Actions ━━━ +# ============================================================================================== + +apply_level_1() { + log "Applying level 1 (soft) — throttling downloaders" + sabnzbd_set_speed "${RM_SABNZBD_SPEED_SOFT:-50M}" + qbit_set_dl_limit "${RM_QBIT_DL_SOFT:-51200}" +} + +apply_level_2() { + log "Applying level 2 (medium) — throttling + pausing background containers" + sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" + qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}" + + if [[ ${#RM_PAUSE_CONTAINERS[@]} -gt 0 ]]; then + local newly_paused + newly_paused=$(pause_containers "${RM_PAUSE_CONTAINERS[@]}") + # Merge with existing paused list (avoid duplicates on re-escalation) + if [[ -n "$newly_paused" ]]; then + if [[ -n "$PAUSED_LIST" ]]; then + PAUSED_LIST="${PAUSED_LIST},${newly_paused}" + else + PAUSED_LIST="$newly_paused" + fi + fi + fi +} + +apply_level_3() { + log "Applying level 3 (hard) — stopping optional containers" + sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2 + qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}" + + if [[ ${#RM_STOP_CONTAINERS[@]} -gt 0 ]]; then + local newly_stopped + newly_stopped=$(stop_containers "${RM_STOP_CONTAINERS[@]}") + if [[ -n "$newly_stopped" ]]; then + if [[ -n "$STOPPED_LIST" ]]; then + STOPPED_LIST="${STOPPED_LIST},${newly_stopped}" + else + STOPPED_LIST="$newly_stopped" + fi + fi + fi + + # Signal docker_watchdog to defer container restarts + rm_state_set_eq "mem_shutdown_active" "true" + warn "mem_shutdown_active=true — docker_watchdog will defer restarts" +} + +# ============================================================================================== +# ━━━ Restore Level Actions ━━━ +# ============================================================================================== + +restore_level_3() { + log "Restoring from level 3 — starting stopped containers" + if [[ -n "$STOPPED_LIST" ]]; then + start_containers "$STOPPED_LIST" + STOPPED_LIST="" + fi + rm_state_set_eq "mem_shutdown_active" "false" + warn "mem_shutdown_active=false — docker_watchdog restoring normal operation" +} + +restore_level_2() { + log "Restoring from level 2 — unpausing containers" + if [[ -n "$PAUSED_LIST" ]]; then + unpause_containers "$PAUSED_LIST" + PAUSED_LIST="" + fi +} + +restore_level_1() { + log "Restoring from level 1 — removing downloader throttle" + sabnzbd_set_speed "0" + qbit_set_dl_limit 0 +} + +# ============================================================================================== +# ━━━ Pressure Decision ━━━ +# ============================================================================================== +echo "" +echo "━━━ $ICON_GEAR Resource Manager — $MY_ID — $(date '+%H:%M:%S') ━━━" +echo " RAM: ${MEM_GB}GB free Load: ${LOAD} Action: ${CURRENT_LEVEL} → ${TARGET_LEVEL} (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})" + +if [[ "$TARGET_LEVEL" -gt "$CURRENT_LEVEL" ]]; then + # ── Escalate ──────────────────────────────────────────────────────────────────────────── + warn "Pressure escalating to level $TARGET_LEVEL — $TARGET_REASON" + notify "Resource Manager: pressure level $TARGET_LEVEL on $(hostname) ($MY_ID) — $TARGET_REASON" \ + "Resource Manager" "warning" + + for (( lvl = CURRENT_LEVEL + 1; lvl <= TARGET_LEVEL; lvl++ )); do + case "$lvl" in + 1) apply_level_1 ;; + 2) apply_level_2 ;; + 3) apply_level_3 ;; + esac + done + + rm_state_set "rm_action_level" "$TARGET_LEVEL" + rm_state_set "rm_recover_cycles" 0 + +elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then + # ── Tracking recovery ─────────────────────────────────────────────────────────────────── + RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 )) + rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES" + log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RM_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions" + + if [[ "$RECOVER_CYCLES" -ge "${RM_RECOVER_CYCLES:-3}" ]]; then + # Level 3 de-escalation requires RAM above recover threshold + if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RM_RAM_RECOVER_GB:-20}" ]]; then + warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RM_RAM_RECOVER_GB}GB" + else + warn "Pressure sustained below level $CURRENT_LEVEL — restoring" + case "$CURRENT_LEVEL" in + 3) restore_level_3 ;; + 2) restore_level_2 ;; + 1) restore_level_1 ;; + esac + + NEW_LEVEL=$(( CURRENT_LEVEL - 1 )) + rm_state_set "rm_action_level" "$NEW_LEVEL" + rm_state_set "rm_recover_cycles" 0 + rm_state_set "rm_paused_containers" "$PAUSED_LIST" + rm_state_set "rm_stopped_containers" "$STOPPED_LIST" + + if [[ "$NEW_LEVEL" -gt 0 ]]; then + warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RM_RECOVER_CYCLES:-3} more cycles to fully clear" + else + log "All pressure cleared — system at normal operation ✅" + notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \ + "Resource Manager" "normal" + fi + fi + fi + +else + # ── Steady state ──────────────────────────────────────────────────────────────────────── + if [[ "$CURRENT_LEVEL" -gt 0 ]]; then + log "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery" + else + log "System at normal pressure ✅" + fi + rm_state_set "rm_recover_cycles" 0 +fi + +# ============================================================================================== +# ━━━ Persist State ━━━ +# ============================================================================================== +rm_state_set "rm_paused_containers" "$PAUSED_LIST" +rm_state_set "rm_stopped_containers" "$STOPPED_LIST" +# Touch state file each run so docker_watchdog stale guard sees fresh mtime +touch "$RM_STATE_FILE" 2>/dev/null diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 2992d39..57fb9a9 100755 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -88,12 +88,13 @@ validate_unraid_cmd \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -acquire_lock "continuous" +acquire_lock detect_hosts TOTAL_CORES=$(nproc) DOCKER_TIMEOUT=10 +SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) # Ensure state files exist for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \ @@ -129,10 +130,8 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%" echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%" echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%" - echo "$ICON_MEM RAM warn: < ${SYS_WATCHDOG_MEM_WARN_GB}GB" - echo "$ICON_MEM RAM shutdown: < ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB" - echo "$ICON_MEM RAM recover: > ${SYS_WATCHDOG_MEM_RECOVER_GB}GB" echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)" + echo " (RAM warn/shutdown/recover managed by resource_manager.sh)" echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%" echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)" echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}" @@ -141,10 +140,6 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr" echo "" - echo "── Container Shutdown Excluded ──" - for c in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do - echo " $ICON_RUNNING $c" - done echo "" echo "── Check Toggles ──" echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM" @@ -319,56 +314,6 @@ run_strike_check() { return 1 } -# ============================================================================================== -# ── CONTAINER SHUTDOWN (RAM EMERGENCY) ──────────────────────────────────────────────────────── -# ============================================================================================== - -shutdown_non_essential_containers() { - warn "RAM emergency — stopping non-essential containers" - local stopped=() - - # Build exclusion map - declare -A EXCLUDED_MAP - for exc in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do - [[ -n "$exc" ]] && EXCLUDED_MAP["$exc"]=1 - done - - # Stop all running containers not in exclusion list - while IFS= read -r container; do - [[ -z "$container" ]] && continue - if [[ -n "${EXCLUDED_MAP[$container]:-}" ]]; then - log "$container — excluded from RAM shutdown, leaving running" - continue - fi - if [[ "$DRY_RUN" == false ]]; then - timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \ - warn "Stopped $container (RAM emergency)" && \ - stopped+=("$container") || \ - error "Failed to stop $container" - else - warn "DRY RUN — would stop $container (RAM emergency)" - stopped+=("$container") - fi - done < <(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) - - if [[ ${#stopped[@]} -gt 0 ]]; then - set_state_val "mem_shutdown_active" "true" - notify "RAM emergency on $(hostname) ($MY_ID) — stopped ${#stopped[@]} containers. Excluded: ${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[*]}" \ - "System Watchdog" "warning" - warn "Stopped ${#stopped[@]} containers — waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB" - fi -} - -restart_non_essential_containers() { - warn "RAM recovered — restarting containers that were stopped in emergency" - if [[ "$DRY_RUN" == false ]]; then - set_state_val "mem_shutdown_active" "false" - fi - # docker_watchdog.sh will detect stopped required containers and restart them - # We just clear the state flag here - warn "Cleared RAM emergency state — docker_watchdog.sh will restart required containers" -} - # ============================================================================================== # ── DO REBOOT ───────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -457,36 +402,11 @@ do_reboot() { } # ============================================================================================== -# ━━━ Clean Shutdown ━━━ +# ━━━ Single-Pass Health Check ━━━ # ============================================================================================== -WATCHDOG_RUNNING=true - -cleanup() { - echo "" - warn "System watchdog received shutdown signal — stopping cleanly" - WATCHDOG_RUNNING=false - exit 0 -} - -trap cleanup SIGTERM SIGINT - -# ============================================================================================== -# ━━━ Continuous Monitoring Loop ━━━ -# ============================================================================================== -warn "System watchdog started — $MY_ID — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" +warn "System watchdog — $MY_ID — $(date '+%H:%M:%S')" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -CYCLE=0 - -while [[ "$WATCHDOG_RUNNING" == true ]]; do - (( CYCLE++ )) - - # Re-source config each cycle — picks up config changes without restart - source "$SCRIPT_DIR/../load_config.sh" - detect_hosts - SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) - TOTAL_CORES=$(nproc) - TRIGGERS=() CRITICAL_TRIGGERS=() URGENT_OOM_CONFIRMED=false @@ -584,12 +504,12 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do # ── Act on CRITICAL triggers immediately ───────────────────────────────────────────────── if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then echo "" - echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT — Cycle $CYCLE ━━━" + echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT ━━━" for t in "${CRITICAL_TRIGGERS[@]}"; do error " CRITICAL: $t" done do_reboot "critical" "${CRITICAL_TRIGGERS[@]}" - continue + exit 0 fi # ========================================================================================== @@ -638,60 +558,27 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do TRIGGERS+=("tmp=${TMP_USED}%") fi - # ── RAM tiers ───────────────────────────────────────────────────────────────────────────── + # ── RAM — reboot tier only (warn/shutdown/recover handled by resource_manager.sh) ────────── if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 )) - MEM_SHUTDOWN_ACTIVE=$(get_state_val "mem_shutdown_active") if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then - # Tier 2 check — bypass if OOM confirms crisis + # Tier 2 — bypass strike system if OOM confirms active crisis if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \ [[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then - error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this cycle — URGENT bypass" + error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this run — URGENT bypass" OOM_VICTIMS=$(get_oom_victims) URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}") [[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS") do_reboot "urgent" "${URGENT_TRIGGERS[@]}" - continue + exit 0 fi # Standard strike path run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \ TRIGGERS+=("low_ram=${MEM_GB}GB") - - elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_SHUTDOWN_GB" ]]; then - reset_strikes "ram" - # Container shutdown tier — but only once per event - if [[ "$MEM_SHUTDOWN_ACTIVE" != "true" ]]; then - warn "RAM ${MEM_GB}GB — below shutdown threshold ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB" - run_strike_check "ram_shutdown" true "RAM shutdown tier ${MEM_GB}GB" && \ - shutdown_non_essential_containers - else - # Already shutdown — check if recovered - if [[ "$MEM_GB" -ge "$SYS_WATCHDOG_MEM_RECOVER_GB" ]]; then - warn "RAM recovered to ${MEM_GB}GB — clearing emergency state" - restart_non_essential_containers - reset_strikes "ram_shutdown" - else - warn "RAM ${MEM_GB}GB — still in emergency shutdown (recover threshold: ${SYS_WATCHDOG_MEM_RECOVER_GB}GB)" - fi - fi - - elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_WARN_GB" ]]; then - reset_strikes "ram" - reset_strikes "ram_shutdown" - warn "RAM ${MEM_GB}GB — below warning threshold ${SYS_WATCHDOG_MEM_WARN_GB}GB" - local prev_ram_warn - prev_ram_warn=$(get_strikes "ram_warn_notified") - if [[ "${prev_ram_warn:-0}" -eq 0 ]]; then - notify "RAM warning on $(hostname) ($MY_ID) — ${MEM_GB}GB free (threshold: ${SYS_WATCHDOG_MEM_WARN_GB}GB)" \ - "System Watchdog" "warning" - set_strikes "ram_warn_notified" 1 - fi else reset_strikes "ram" - reset_strikes "ram_shutdown" - set_strikes "ram_warn_notified" 0 log "RAM ${MEM_GB}GB free ✅" fi fi @@ -852,38 +739,17 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do # ========================================================================================== if [[ ${#TRIGGERS[@]} -gt 0 ]]; then echo "" - echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "━━━ $ICON_REBOOT_SMART System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" for t in "${TRIGGERS[@]}"; do echo " $ICON_REBOOT_SMART $t" done - [[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this cycle: $OOM_DELTA" + [[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this run: $OOM_DELTA" echo "" do_reboot "standard" "${TRIGGERS[@]}" + exit 0 else - log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))" - - # Heartbeat — periodic proof of life - if [[ "${SYSTEM_WATCHDOG_HEARTBEAT:-true}" == true ]]; then - HB_SECONDS=$(( ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 )) - UPTIME_APPROX=$(( CYCLE * SYSTEM_WATCHDOG_INTERVAL )) - if [[ "$HB_SECONDS" -gt 0 ]] && \ - (( UPTIME_APPROX % HB_SECONDS < SYSTEM_WATCHDOG_INTERVAL )) && \ - [[ "$UPTIME_APPROX" -gt 0 ]]; then - HB_HR=$(( UPTIME_APPROX / 3600 )) - warn "♥ system_watchdog alive — $MY_ID — ~${HB_HR}hr uptime ($(date '+%H:%M:%S'))" - fi - fi + log "System healthy ✅ ($(date '+%H:%M:%S'))" fi - # ── State file heartbeat — keep mtime fresh every cycle ─────────────────────────────────── - # docker_watchdog.sh uses state file mtime to detect stale RAM emergency flags. - # If all checks pass with no set_state_val calls (e.g. KERNEL_OOPS + MDSTAT both disabled), - # mtime would not update and stale guard would incorrectly resume docker_watchdog.sh. - # Writing watchdog_cycle each tick guarantees mtime stays current while watchdog runs. - set_state_val "watchdog_cycle" "$CYCLE" - - # Sleep until next cycle — interruptible by SIGTERM - sleep "$SYSTEM_WATCHDOG_INTERVAL" & - wait $! - -done \ No newline at end of file + # Keep state file mtime fresh — docker_watchdog stale guard checks this + set_state_val "watchdog_cycle" "$(date +%s)" \ No newline at end of file