diff --git a/Docker_Essentials/docker_watchdog.sh b/Docker_Essentials/docker_watchdog.sh index ef03814..a0360e8 100755 --- a/Docker_Essentials/docker_watchdog.sh +++ b/Docker_Essentials/docker_watchdog.sh @@ -351,10 +351,10 @@ _dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE") # 1 = RAM emergency active — defer container management this cycle check_system_watchdog_state() { - # Returns 0 = normal operation | 1 = defer, resource_manager RAM emergency active - local state_file="$RM_STATE_FILE" + # Returns 0 = normal operation | 1 = defer, resource_watchdog RAM emergency active + local state_file="$RW_STATE_FILE" - # No state file = resource_manager not yet run — assume normal + # No state file = resource_watchdog not yet run — assume normal [[ ! -f "$state_file" ]] && return 0 local mem_shutdown @@ -364,7 +364,7 @@ check_system_watchdog_state() { # ── Stale state guard ───────────────────────────────────────────────────────────────────── # If mem_shutdown_active=true but state file hasn't been updated in > 2 hours, - # resource_manager.sh may not be running — don't defer indefinitely on stale state. + # resource_watchdog.sh may not be running — don't defer indefinitely on stale state. local state_mtime now age_seconds stale_limit=7200 # 2 hours state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0) now=$(date +%s) @@ -372,7 +372,7 @@ check_system_watchdog_state() { if [[ "$age_seconds" -gt "$stale_limit" ]]; then warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale" - warn "resource_manager.sh may not be running — resuming normal container management" + warn "resource_watchdog.sh may not be running — resuming normal container management" return 0 # Resume normal — don't defer indefinitely on stale state fi @@ -484,15 +484,15 @@ CYCLE_START=$(date +%s) exit 0 fi - # ── RAM emergency check — resource_manager.sh managing containers ──────────────────────── - # If resource_manager.sh has triggered a hard RAM shutdown, defer all container + # ── RAM emergency check — resource_watchdog.sh managing containers ──────────────────────── + # If resource_watchdog.sh has triggered a hard RAM shutdown, defer all container # management this run to prevent undoing the emergency stop and re-pressuring RAM. if ! check_system_watchdog_state; then MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 )) - warn "RAM emergency active (${MEM_GB}GB free) — resource_manager.sh managing containers" + warn "RAM emergency active (${MEM_GB}GB free) — resource_watchdog.sh managing containers" warn "Deferring all container restart logic this run" - log "Waiting for RAM to recover above ${RM_RAM_RECOVER_GB:-20}GB before resuming" + log "Waiting for RAM to recover above ${RW_RAM_RECOVER_GB:-20}GB before resuming" exit 0 fi diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 057de3a..9d83f70 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -14,7 +14,7 @@ # 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split # 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth # 🌐 Rsync — weekly transfer totals, per-share breakdown, failures -# 🛡️ Watchdog — resource manager, system watchdog, docker watchdog, fallback state +# 🛡️ Watchdog — resource watchdog, system watchdog, docker watchdog, fallback state # 🔐 Security — SSL cert expiry per domain # 📊 Emby — weekly stream count, active now, top users # ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync @@ -562,12 +562,12 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then fi fi -# ── Resource Manager ───────────────────────────────────────────────────────────────────────── -line "🎛️ Resource Manager" -if [[ -f "$RM_STATE_FILE" ]]; then - _rm_last=$(stat -c %Y "$RM_STATE_FILE" 2>/dev/null || echo 0) +# ── Resource Watchdog ───────────────────────────────────────────────────────────────────────── +line "🎛️ Resource Watchdog" +if [[ -f "$RW_STATE_FILE" ]]; then + _rm_last=$(stat -c %Y "$RW_STATE_FILE" 2>/dev/null || echo 0) _rm_ago=$(( NOW - _rm_last )) - _rm_level=$(grep "^current_level:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2) + _rm_level=$(grep "^current_level:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2) _rm_level="${_rm_level:-0}" if [[ "$_rm_level" -gt 0 ]]; then issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago" @@ -577,7 +577,7 @@ if [[ -f "$RM_STATE_FILE" ]]; then issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running" fi else - line " ℹ️ State file not found (resource_manager may not have run yet)" + line " ℹ️ State file not found (resource_watchdog may not have run yet)" fi REPORT+=("") diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index e1cc2a4..a8b5942 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -6,12 +6,12 @@ # Schedule: * * * * * (every minute via User Scripts plugin) # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# 1. resource_manager.sh — reduce system pressure intelligently +# 1. resource_watchdog.sh — reduce system pressure intelligently # 2. docker_watchdog.sh — heal containers with freed resources # 3. system_watchdog.sh — reboot if all else fails (last line of defense) # # ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── -# Resource Manager first — frees RAM and CPU before healing attempts container restarts. +# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts. # Containers restarted into a resource-pressured system just fail again. # Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise. # System Watchdog last — only triggers if prior layers could not resolve the issue. @@ -63,7 +63,7 @@ acquire_lock detect_hosts -RESOURCE_MANAGER="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_manager.sh" +RESOURCE_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_watchdog.sh" DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh" SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh" @@ -88,7 +88,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "── Sub-scripts ──" for pair in \ - "Resource Manager:$RESOURCE_MANAGER" \ + "Resource Watchdog:$RESOURCE_WATCHDOG" \ "Docker Watchdog:$DOCKER_WATCHDOG" \ "System Watchdog:$SYSTEM_WATCHDOG"; do label="${pair%%:*}" @@ -150,7 +150,7 @@ run_watchdog() { fi } -run_watchdog "Resource Manager" "$RESOURCE_MANAGER" +run_watchdog "Resource Watchdog" "$RESOURCE_WATCHDOG" run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG" run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG" diff --git a/common.sh b/common.sh index 7b99e5e..1dc1eab 100644 --- a/common.sh +++ b/common.sh @@ -572,8 +572,8 @@ detect_hosts() { _alias_array "PARTNERSHIP_AUTH_WEBUIS" _alias_array "PARTNERSHIP_MIRROR_BACKUPS" _alias_array "PARTNERSHIP_OWN_CONTAINERS" - _alias_array "RM_PAUSE_CONTAINERS" - _alias_array "RM_STOP_CONTAINERS" + _alias_array "RW_PAUSE_CONTAINERS" + _alias_array "RW_STOP_CONTAINERS" # ── Set array aliases — associative arrays ──────────────────────────────── # Associative arrays cannot be copied with eval — must be rebuilt key by key diff --git a/master.conf b/master.conf index c3e2b4b..183e965 100644 --- a/master.conf +++ b/master.conf @@ -279,7 +279,7 @@ # Launched in order — each as a background process. # One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. # Continuous scripts (failover) run until array stops. -# Watchdogs (resource_manager, docker_watchdog, system_watchdog) are cronned via +# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via # watchdog_orchestrator.sh — NOT launched here. ARRAY_START_SCRIPTS=( "git_pull_execute.sh" # pull latest scripts before anything starts @@ -1117,7 +1117,7 @@ # ============================================================================================== # ── WATCHDOG ORCHESTRATOR ───────────────────────────────────────────────────────────────────── # ============================================================================================== -# Runs resource_manager → docker_watchdog → system_watchdog in sequence each cron cycle. +# Runs resource_watchdog → docker_watchdog → system_watchdog in sequence each cron cycle. # Schedule: * * * * * (every minute via User Scripts plugin) # NOT in ARRAY_START_SCRIPTS — has its own cron entry. @@ -1136,43 +1136,43 @@ # Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer # # ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── -# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (in master_host*.conf) -# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (in master_host*.conf) +# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in master_host*.conf) +# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in master_host*.conf) - RM_ENABLED=true - RM_STATE_FILE="/tmp/resource_manager_state.db" + RW_ENABLED=true + RW_STATE_FILE="/tmp/resource_watchdog_state.db" # ━━━ Pressure Thresholds ━━━ -# Graduated RAM response — resource_manager acts before system_watchdog reboots. -# RM_RAM_SOFT_GB > RM_RAM_MEDIUM_GB > RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always - RM_RAM_SOFT_GB=12 # throttle start — reduce background load - RM_RAM_MEDIUM_GB=8 # pause background containers - RM_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) - RM_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers +# Graduated RAM response — resource_watchdog acts before system_watchdog reboots. +# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always + RW_RAM_SOFT_GB=12 # throttle start — reduce background load + RW_RAM_MEDIUM_GB=8 # pause background containers + RW_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) + RW_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers # Load average thresholds — multiplier × core count - RM_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained - RM_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained + RW_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained + RW_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained # Consecutive runs at lower pressure before de-escalating - RM_RECOVER_CYCLES=3 + RW_RECOVER_CYCLES=3 # ━━━ SABnzbd Throttle ━━━ # Speed values: "50M" = 50 MB/s, "0" = unlimited - RM_SABNZBD_ENABLED=true - RM_SABNZBD_SPEED_SOFT="50M" - RM_SABNZBD_SPEED_MEDIUM="10M" + RW_SABNZBD_ENABLED=true + RW_SABNZBD_SPEED_SOFT="50M" + RW_SABNZBD_SPEED_MEDIUM="10M" # ━━━ qBittorrent Throttle ━━━ # KB/s — 0 = unlimited - RM_QBIT_ENABLED=true - RM_QBIT_DL_SOFT=51200 # 50 MB/s - RM_QBIT_DL_MEDIUM=10240 # 10 MB/s + RW_QBIT_ENABLED=true + RW_QBIT_DL_SOFT=51200 # 50 MB/s + RW_QBIT_DL_MEDIUM=10240 # 10 MB/s # ━━━ Critical Containers ━━━ # Never paused or stopped regardless of pressure level. # Keep DNS, auth, media serving, and live TV always running. - RM_CRITICAL_CONTAINERS=( + RW_CRITICAL_CONTAINERS=( "NginxProxyManager" # reverse proxy — internet access "Authelia" # auth — nothing accessible without it "Authelia-Secondary" @@ -1207,7 +1207,7 @@ # # ── RAM ─────────────────────────────────────────────────────────────────────────────────────── # SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass) -# Warn/shutdown/recover RAM tiers are handled by resource_manager.sh +# Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh # ━━━ State Files ━━━ SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅ @@ -1234,8 +1234,8 @@ SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # ━━━ RAM Reboot Threshold ━━━ -# Reboot trigger only — warn/shutdown/recover handled by resource_manager.sh -# RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) +# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh +# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) SYS_WATCHDOG_MEM_GB=4 # strike system → reboot # ━━━ OOM Bypass Settings ━━━ diff --git a/master_host1.conf b/master_host1.conf index 2a819bd..81ab920 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -673,11 +673,11 @@ # ============================================================================================== # ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Containers to manage under pressure — see master.conf RM_CRITICAL_CONTAINERS for exclusions. +# Containers to manage under pressure — see master.conf RW_CRITICAL_CONTAINERS for exclusions. -# docker pause at medium pressure (RAM < RM_RAM_MEDIUM_GB or load > medium threshold) +# docker pause at medium pressure (RAM < RW_RAM_MEDIUM_GB or load > medium threshold) # Suspended in-place — instant to pause/unpause, no state lost, no restart delay. - HOST1_RM_PAUSE_CONTAINERS=( + HOST1_RW_PAUSE_CONTAINERS=( "Huntarr" # arr search automation — safe to suspend "Cleanuparr" # download cleanup — safe to suspend "Healarr" # arr health checks — safe to suspend @@ -686,10 +686,10 @@ "Pinchflat" # YouTube archiver — background only ) -# docker stop at hard pressure (RAM < RM_RAM_HARD_GB) +# docker stop at hard pressure (RAM < RW_RAM_HARD_GB) # Full stop — these are optional/heavy services that free significant RAM when stopped. -# resource_manager.sh restarts them when pressure fully clears (RAM >= RM_RAM_RECOVER_GB). - HOST1_RM_STOP_CONTAINERS=( +# resource_watchdog.sh restarts them when pressure fully clears (RAM >= RW_RAM_RECOVER_GB). + HOST1_RW_STOP_CONTAINERS=( "LocalAI" # GPU/CPU heavy — largest RAM consumer when idle "7DaysToDie" # game server — optional "V-Rising" # game server — optional diff --git a/unRAID_Essentials/resource_manager.sh b/unRAID_Essentials/resource_watchdog.sh similarity index 79% rename from unRAID_Essentials/resource_manager.sh rename to unRAID_Essentials/resource_watchdog.sh index 5771136..0cb5188 100755 --- a/unRAID_Essentials/resource_manager.sh +++ b/unRAID_Essentials/resource_watchdog.sh @@ -13,25 +13,25 @@ # # ── THREE-LEVEL PRESSURE RESPONSE ───────────────────────────────────────────────────────────── # -# Level 1 — SOFT (RAM < RM_RAM_SOFT_GB OR load > RM_LOAD_SOFT_MULTIPLIER × cores): -# Throttle SABnzbd download speed to RM_SABNZBD_SPEED_SOFT -# Throttle qBittorrent download to RM_QBIT_DL_SOFT KB/s +# Level 1 — SOFT (RAM < RW_RAM_SOFT_GB OR load > RW_LOAD_SOFT_MULTIPLIER × cores): +# Throttle SABnzbd download speed to RW_SABNZBD_SPEED_SOFT +# Throttle qBittorrent download to RW_QBIT_DL_SOFT KB/s # -# Level 2 — MEDIUM (RAM < RM_RAM_MEDIUM_GB OR load > RM_LOAD_MEDIUM_MULTIPLIER × cores): +# Level 2 — MEDIUM (RAM < RW_RAM_MEDIUM_GB OR load > RW_LOAD_MEDIUM_MULTIPLIER × cores): # Further throttle SABnzbd + qBittorrent to medium limits -# docker pause RM_PAUSE_CONTAINERS — suspend without losing state, instant reversible +# docker pause RW_PAUSE_CONTAINERS — suspend without losing state, instant reversible # -# Level 3 — HARD (RAM < RM_RAM_HARD_GB): -# docker stop RM_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.) +# Level 3 — HARD (RAM < RW_RAM_HARD_GB): +# docker stop RW_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.) # Write mem_shutdown_active=true — signals docker_watchdog to defer container restarts # # ── RECOVERY ────────────────────────────────────────────────────────────────────────────────── -# Pressure must stay below current action threshold for RM_RECOVER_CYCLES consecutive runs +# Pressure must stay below current action threshold for RW_RECOVER_CYCLES consecutive runs # before restoring. De-escalates one level at a time to avoid re-triggering immediately. -# Level 3 de-escalation additionally requires RAM >= RM_RAM_RECOVER_GB before un-stopping. +# Level 3 de-escalation additionally requires RAM >= RW_RAM_RECOVER_GB before un-stopping. # # ── COORDINATION WITH DOCKER WATCHDOG ───────────────────────────────────────────────────────── -# At level 3: writes mem_shutdown_active=true to RM_STATE_FILE. +# At level 3: writes mem_shutdown_active=true to RW_STATE_FILE. # docker_watchdog.sh reads this and defers all container restart logic. # Cleared when pressure fully resolves and containers are restarted. # This prevents docker_watchdog from restarting containers that RM just stopped to free RAM. @@ -39,28 +39,28 @@ # ── NOT RESPONSIBLE FOR ─────────────────────────────────────────────────────────────────────── # Restarting broken containers — docker_watchdog.sh # Rebooting the system — system_watchdog.sh -# Reacting to single data points — RM_RECOVER_CYCLES prevents flip-flopping +# Reacting to single data points — RW_RECOVER_CYCLES prevents flip-flopping # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# RM_ENABLED, RM_STATE_FILE -# RM_RAM_SOFT_GB, RM_RAM_MEDIUM_GB, RM_RAM_HARD_GB, RM_RAM_RECOVER_GB -# RM_LOAD_SOFT_MULTIPLIER, RM_LOAD_MEDIUM_MULTIPLIER -# RM_RECOVER_CYCLES -# RM_SABNZBD_ENABLED, RM_SABNZBD_SPEED_SOFT, RM_SABNZBD_SPEED_MEDIUM -# RM_QBIT_ENABLED, RM_QBIT_DL_SOFT, RM_QBIT_DL_MEDIUM -# RM_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure +# RW_ENABLED, RW_STATE_FILE +# RW_RAM_SOFT_GB, RW_RAM_MEDIUM_GB, RW_RAM_HARD_GB, RW_RAM_RECOVER_GB +# RW_LOAD_SOFT_MULTIPLIER, RW_LOAD_MEDIUM_MULTIPLIER +# RW_RECOVER_CYCLES +# RW_SABNZBD_ENABLED, RW_SABNZBD_SPEED_SOFT, RW_SABNZBD_SPEED_MEDIUM +# RW_QBIT_ENABLED, RW_QBIT_DL_SOFT, RW_QBIT_DL_MEDIUM +# RW_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure # # ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts) -# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts) +# HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts) +# HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts) # HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY # HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# resource_manager.sh — normal run (via watchdog_orchestrator.sh) -# resource_manager.sh --dry-run — show what would happen without acting -# resource_manager.sh --status — current pressure level and active actions -# resource_manager.sh --log — verbose per-check output +# resource_watchdog.sh — normal run (via watchdog_orchestrator.sh) +# resource_watchdog.sh --dry-run — show what would happen without acting +# resource_watchdog.sh --status — current pressure level and active actions +# resource_watchdog.sh --log — verbose per-check output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -77,8 +77,8 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -if [[ "${RM_ENABLED:-true}" != "true" ]]; then - log "Resource Manager disabled (RM_ENABLED=false)" +if [[ "${RW_ENABLED:-true}" != "true" ]]; then + log "Resource Manager disabled (RW_ENABLED=false)" exit 0 fi @@ -88,8 +88,8 @@ detect_hosts DOCKER_TIMEOUT=15 -touch "$RM_STATE_FILE" 2>/dev/null || { - error "Cannot create state file: $RM_STATE_FILE" +touch "$RW_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $RW_STATE_FILE" exit 1 } @@ -100,25 +100,25 @@ touch "$RM_STATE_FILE" 2>/dev/null || { # rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags rm_state_get() { - grep -E "^${1}:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2- + grep -E "^${1}:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2- } rm_state_set() { local key="$1" val="$2" - grep -vE "^${key}:" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp" - echo "${key}:${val}" >> "${RM_STATE_FILE}.tmp" - mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE" + grep -vE "^${key}:" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp" + echo "${key}:${val}" >> "${RW_STATE_FILE}.tmp" + mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE" } rm_state_get_eq() { - grep -E "^${1}=" "$RM_STATE_FILE" 2>/dev/null | cut -d= -f2- + grep -E "^${1}=" "$RW_STATE_FILE" 2>/dev/null | cut -d= -f2- } rm_state_set_eq() { local key="$1" val="$2" - grep -vE "^${key}=" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp" - echo "${key}=${val}" >> "${RM_STATE_FILE}.tmp" - mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE" + grep -vE "^${key}=" "$RW_STATE_FILE" 2>/dev/null > "${RW_STATE_FILE}.tmp" + echo "${key}=${val}" >> "${RW_STATE_FILE}.tmp" + mv "${RW_STATE_FILE}.tmp" "$RW_STATE_FILE" } # ============================================================================================== @@ -137,22 +137,22 @@ MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 )) LOAD=$(awk '{print $1}' /proc/loadavg) LOAD_INT=$(printf "%.0f" "$LOAD") -RM_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_SOFT_MULTIPLIER:-2.0}}") -RM_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_MEDIUM_MULTIPLIER:-3.0}}") +RW_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_SOFT_MULTIPLIER:-2.0}}") +RW_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RW_LOAD_MEDIUM_MULTIPLIER:-3.0}}") TARGET_LEVEL=0 TARGET_REASON="" -if [[ "$MEM_GB" -lt "${RM_RAM_HARD_GB:-6}" ]]; then +if [[ "$MEM_GB" -lt "${RW_RAM_HARD_GB:-6}" ]]; then TARGET_LEVEL=3 - TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RM_RAM_HARD_GB}GB" -elif [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]]; then + TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RW_RAM_HARD_GB}GB" +elif [[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]]; then TARGET_LEVEL=2 - [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RM_RAM_MEDIUM_GB}GB" - [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RM_LOAD_MEDIUM_THRESH}" -elif [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]]; then + [[ "$MEM_GB" -lt "${RW_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RW_RAM_MEDIUM_GB}GB" + [[ "$LOAD_INT" -ge "$RW_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RW_LOAD_MEDIUM_THRESH}" +elif [[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]]; then TARGET_LEVEL=1 - [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RM_RAM_SOFT_GB}GB" - [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RM_LOAD_SOFT_THRESH}" + [[ "$MEM_GB" -lt "${RW_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RW_RAM_SOFT_GB}GB" + [[ "$LOAD_INT" -ge "$RW_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RW_LOAD_SOFT_THRESH}" fi LEVEL_NAMES=("normal" "soft" "medium" "hard") @@ -168,24 +168,24 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "── Current State ──" echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})" echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})" - echo " Recover cycles: $RECOVER_CYCLES / ${RM_RECOVER_CYCLES:-3}" + echo " Recover cycles: $RECOVER_CYCLES / ${RW_RECOVER_CYCLES:-3}" [[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST" [[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST" MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active") [[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)" echo "" echo "── System Pressure ──" - echo " RAM free: ${MEM_GB}GB (soft:<${RM_RAM_SOFT_GB} medium:<${RM_RAM_MEDIUM_GB} hard:<${RM_RAM_HARD_GB} recover:>=${RM_RAM_RECOVER_GB})" - echo " Load avg: ${LOAD} (soft:>=${RM_LOAD_SOFT_THRESH} medium:>=${RM_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})" + echo " RAM free: ${MEM_GB}GB (soft:<${RW_RAM_SOFT_GB} medium:<${RW_RAM_MEDIUM_GB} hard:<${RW_RAM_HARD_GB} recover:>=${RW_RAM_RECOVER_GB})" + echo " Load avg: ${LOAD} (soft:>=${RW_LOAD_SOFT_THRESH} medium:>=${RW_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})" echo "" echo "── Configuration ──" - echo " SABnzbd throttle: ${RM_SABNZBD_ENABLED:-true} soft=${RM_SABNZBD_SPEED_SOFT} medium=${RM_SABNZBD_SPEED_MEDIUM}" - echo " qBit throttle: ${RM_QBIT_ENABLED:-true} soft=${RM_QBIT_DL_SOFT}KB/s medium=${RM_QBIT_DL_MEDIUM}KB/s" + echo " SABnzbd throttle: ${RW_SABNZBD_ENABLED:-true} soft=${RW_SABNZBD_SPEED_SOFT} medium=${RW_SABNZBD_SPEED_MEDIUM}" + echo " qBit throttle: ${RW_QBIT_ENABLED:-true} soft=${RW_QBIT_DL_SOFT}KB/s medium=${RW_QBIT_DL_MEDIUM}KB/s" echo "" echo "── Container Lists (this host) ──" - echo " Pause at medium: ${RM_PAUSE_CONTAINERS[*]:-none configured}" - echo " Stop at hard: ${RM_STOP_CONTAINERS[*]:-none configured}" - echo " Critical (never touched): ${RM_CRITICAL_CONTAINERS[*]:-none}" + echo " Pause at medium: ${RW_PAUSE_CONTAINERS[*]:-none configured}" + echo " Stop at hard: ${RW_STOP_CONTAINERS[*]:-none configured}" + echo " Critical (never touched): ${RW_CRITICAL_CONTAINERS[*]:-none}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi @@ -196,7 +196,7 @@ fi # Returns 0 if container is safe to pause/stop, 1 if it is critical is_critical() { local container="$1" - for c in "${RM_CRITICAL_CONTAINERS[@]:-}"; do + for c in "${RW_CRITICAL_CONTAINERS[@]:-}"; do [[ "$c" == "$container" ]] && return 1 done return 0 @@ -207,7 +207,7 @@ is_critical() { # ============================================================================================== sabnzbd_set_speed() { local speed="$1" - [[ "${RM_SABNZBD_ENABLED:-true}" != "true" ]] && return 0 + [[ "${RW_SABNZBD_ENABLED:-true}" != "true" ]] && return 0 [[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0 if [[ "$DRY_RUN" == true ]]; then @@ -226,7 +226,7 @@ sabnzbd_set_speed() { QBIT_COOKIE="/tmp/rm_qbit_cookie.txt" qbit_login() { - [[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0 + [[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0 [[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0 curl -sf --max-time 10 -c "$QBIT_COOKIE" \ @@ -236,7 +236,7 @@ qbit_login() { qbit_set_dl_limit() { local kbps="$1" # KB/s — 0 = unlimited - [[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0 + [[ "${RW_QBIT_ENABLED:-true}" != "true" ]] && return 0 [[ -z "$QBIT_URL" ]] && return 0 if [[ "$DRY_RUN" == true ]]; then @@ -373,18 +373,18 @@ start_containers() { apply_level_1() { log "Applying level 1 (soft) — throttling downloaders" - sabnzbd_set_speed "${RM_SABNZBD_SPEED_SOFT:-50M}" - qbit_set_dl_limit "${RM_QBIT_DL_SOFT:-51200}" + sabnzbd_set_speed "${RW_SABNZBD_SPEED_SOFT:-50M}" + qbit_set_dl_limit "${RW_QBIT_DL_SOFT:-51200}" } apply_level_2() { log "Applying level 2 (medium) — throttling + pausing background containers" - sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" - qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}" + sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}" + qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}" - if [[ ${#RM_PAUSE_CONTAINERS[@]} -gt 0 ]]; then + if [[ ${#RW_PAUSE_CONTAINERS[@]} -gt 0 ]]; then local newly_paused - newly_paused=$(pause_containers "${RM_PAUSE_CONTAINERS[@]}") + newly_paused=$(pause_containers "${RW_PAUSE_CONTAINERS[@]}") # Merge with existing paused list (avoid duplicates on re-escalation) if [[ -n "$newly_paused" ]]; then if [[ -n "$PAUSED_LIST" ]]; then @@ -398,12 +398,12 @@ apply_level_2() { apply_level_3() { log "Applying level 3 (hard) — stopping optional containers" - sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2 - qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}" + sabnzbd_set_speed "${RW_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2 + qbit_set_dl_limit "${RW_QBIT_DL_MEDIUM:-10240}" - if [[ ${#RM_STOP_CONTAINERS[@]} -gt 0 ]]; then + if [[ ${#RW_STOP_CONTAINERS[@]} -gt 0 ]]; then local newly_stopped - newly_stopped=$(stop_containers "${RM_STOP_CONTAINERS[@]}") + newly_stopped=$(stop_containers "${RW_STOP_CONTAINERS[@]}") if [[ -n "$newly_stopped" ]]; then if [[ -n "$STOPPED_LIST" ]]; then STOPPED_LIST="${STOPPED_LIST},${newly_stopped}" @@ -474,12 +474,12 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then # ── Tracking recovery ─────────────────────────────────────────────────────────────────── RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 )) rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES" - log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RM_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions" + log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RW_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions" - if [[ "$RECOVER_CYCLES" -ge "${RM_RECOVER_CYCLES:-3}" ]]; then + if [[ "$RECOVER_CYCLES" -ge "${RW_RECOVER_CYCLES:-3}" ]]; then # Level 3 de-escalation requires RAM above recover threshold - if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RM_RAM_RECOVER_GB:-20}" ]]; then - warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RM_RAM_RECOVER_GB}GB" + if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RW_RAM_RECOVER_GB:-20}" ]]; then + warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RW_RAM_RECOVER_GB}GB" else warn "Pressure sustained below level $CURRENT_LEVEL — restoring" case "$CURRENT_LEVEL" in @@ -495,7 +495,7 @@ elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then rm_state_set "rm_stopped_containers" "$STOPPED_LIST" if [[ "$NEW_LEVEL" -gt 0 ]]; then - warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RM_RECOVER_CYCLES:-3} more cycles to fully clear" + warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RW_RECOVER_CYCLES:-3} more cycles to fully clear" else log "All pressure cleared — system at normal operation ✅" notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \ @@ -520,4 +520,4 @@ fi rm_state_set "rm_paused_containers" "$PAUSED_LIST" rm_state_set "rm_stopped_containers" "$STOPPED_LIST" # Touch state file each run so docker_watchdog stale guard sees fresh mtime -touch "$RM_STATE_FILE" 2>/dev/null +touch "$RW_STATE_FILE" 2>/dev/null diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index 57fb9a9..008b616 100755 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -131,7 +131,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%" echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%" echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)" - echo " (RAM warn/shutdown/recover managed by resource_manager.sh)" + echo " (RAM warn/shutdown/recover managed by resource_watchdog.sh)" echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%" echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)" echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}" @@ -558,7 +558,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ TRIGGERS+=("tmp=${TMP_USED}%") fi - # ── RAM — reboot tier only (warn/shutdown/recover handled by resource_manager.sh) ────────── + # ── RAM — reboot tier only (warn/shutdown/recover handled by resource_watchdog.sh) ────────── if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 ))