feat: watchdog architecture v2 — resource manager + single-pass orchestrator
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
f16c962ac0
commit
309546e615
@@ -97,8 +97,8 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
fi
|
||||
success "Running as root"
|
||||
|
||||
# Continuous mode — skip gracefully if healthy instance already running
|
||||
acquire_lock "continuous"
|
||||
# Skip if another instance is running — no pile-up during long operations
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||||
detect_hosts
|
||||
@@ -332,8 +332,11 @@ is_parity_running() {
|
||||
|
||||
WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt
|
||||
WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying
|
||||
WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers
|
||||
WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session
|
||||
# Loaded from state file — persists across single-pass runs
|
||||
WATCHDOG_DAEMON_STRIKES=$(get_strikes "daemon_strikes" "$WATCHDOG_STATE_FILE")
|
||||
WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES//[^0-9]/}"; WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES:-0}"
|
||||
_dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE")
|
||||
[[ "$_dr_raw" == "true" ]] && WATCHDOG_DAEMON_RESTARTED=true || WATCHDOG_DAEMON_RESTARTED=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
|
||||
@@ -348,10 +351,10 @@ WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart thi
|
||||
# 1 = RAM emergency active — defer container management this cycle
|
||||
|
||||
check_system_watchdog_state() {
|
||||
# Returns 0 = normal operation | 1 = defer, RAM emergency active
|
||||
local state_file="$SYS_WATCHDOG_STATE_FILE"
|
||||
# Returns 0 = normal operation | 1 = defer, resource_manager RAM emergency active
|
||||
local state_file="$RM_STATE_FILE"
|
||||
|
||||
# No state file = system_watchdog not running or not yet written — assume normal
|
||||
# No state file = resource_manager not yet run — assume normal
|
||||
[[ ! -f "$state_file" ]] && return 0
|
||||
|
||||
local mem_shutdown
|
||||
@@ -361,7 +364,7 @@ check_system_watchdog_state() {
|
||||
|
||||
# ── Stale state guard ─────────────────────────────────────────────────────────────────────
|
||||
# If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
|
||||
# system_watchdog.sh may have died — don't be silenced forever by a stale flag.
|
||||
# resource_manager.sh may not be running — don't defer indefinitely on stale state.
|
||||
local state_mtime now age_seconds stale_limit=7200 # 2 hours
|
||||
state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
@@ -369,8 +372,7 @@ check_system_watchdog_state() {
|
||||
|
||||
if [[ "$age_seconds" -gt "$stale_limit" ]]; then
|
||||
warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
|
||||
warn "system_watchdog.sh may not be running — resuming normal container management"
|
||||
warn "If RAM is still low this will be caught on next system_watchdog.sh cycle"
|
||||
warn "resource_manager.sh may not be running — resuming normal container management"
|
||||
return 0 # Resume normal — don't defer indefinitely on stale state
|
||||
fi
|
||||
|
||||
@@ -385,11 +387,14 @@ check_docker_daemon() {
|
||||
queue_notify "Docker daemon recovered on $(hostname)" "normal"
|
||||
WATCHDOG_DAEMON_STRIKES=0
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
|
||||
set_strikes "daemon_strikes" "$WATCHDOG_DAEMON_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"
|
||||
|
||||
if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
|
||||
@@ -415,6 +420,7 @@ check_docker_daemon() {
|
||||
|
||||
# Restart daemon — unRAID uses rc.d scripts, not systemd
|
||||
WATCHDOG_DAEMON_RESTARTED=true
|
||||
set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE"
|
||||
if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then
|
||||
info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
|
||||
sleep "$WATCHDOG_DAEMON_RESTART_WAIT"
|
||||
@@ -424,6 +430,8 @@ check_docker_daemon() {
|
||||
notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
|
||||
WATCHDOG_DAEMON_STRIKES=0
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||
return 0
|
||||
else
|
||||
error "Docker daemon did not recover after restart"
|
||||
@@ -440,38 +448,14 @@ check_docker_daemon() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CLEAN SHUTDOWN ────────────────────────────────────────────────────────────────────────────
|
||||
# ━━━ Single-Pass Monitoring Run ━━━
|
||||
# ==============================================================================================
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
warn "Docker watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Continuous Monitoring Loop ━━━
|
||||
# ==============================================================================================
|
||||
info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
info "$ICON_WATCHDOG Docker watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
((CYCLE++))
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
# ── Re-source config each cycle ──────────────────────────────────────────────────────────
|
||||
# Picks up config changes (new containers, threshold adjustments) without restart.
|
||||
# detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source.
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
detect_hosts
|
||||
|
||||
# ── Per-cycle state — cleared each iteration ──────────────────────────────────────────────
|
||||
# ── Per-run state ─────────────────────────────────────────────────────────────────────────
|
||||
NOTIFY_EVENTS=()
|
||||
T1_RESTARTS=0
|
||||
T1_WARNINGS=0
|
||||
@@ -488,33 +472,28 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# ── Docker daemon health check — first check every cycle ────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip cycle if down
|
||||
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||
if ! check_docker_daemon; then
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Parity check — skip restarts during parity ───────────────────────────────────────────
|
||||
if is_parity_running; then
|
||||
log "Parity check in progress — skipping restart actions this cycle"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
log "Parity check in progress — skipping restart actions this run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── RAM emergency check — system_watchdog.sh managing containers ───────────────────────────
|
||||
# If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container
|
||||
# management this cycle. Docker daemon health checks continue — system still needs
|
||||
# monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown.
|
||||
# ── RAM emergency check — resource_manager.sh managing containers ────────────────────────
|
||||
# If resource_manager.sh has triggered a hard RAM shutdown, defer all container
|
||||
# management this run to prevent undoing the emergency stop and re-pressuring RAM.
|
||||
if ! check_system_watchdog_state; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers"
|
||||
warn "Deferring all container restart logic this cycle"
|
||||
log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
continue
|
||||
warn "RAM emergency active (${MEM_GB}GB free) — resource_manager.sh managing containers"
|
||||
warn "Deferring all container restart logic this run"
|
||||
log "Waiting for RAM to recover above ${RM_RAM_RECOVER_GB:-20}GB before resuming"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Startup grace period ──────────────────────────────────────────────────────────────────
|
||||
@@ -815,29 +794,11 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
|
||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "━━━ $ICON_WATCHDOG Docker Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||
echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
|
||||
|
||||
# Heartbeat — periodic proof of life even when everything is healthy
|
||||
if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
||||
(( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \
|
||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
||||
HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 ))
|
||||
info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
|
||||
log "All healthy ✅ ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
@@ -14,7 +14,7 @@
|
||||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth
|
||||
# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures
|
||||
# 🛡️ Watchdog — system watchdog, docker watchdog, fallback state
|
||||
# 🛡️ Watchdog — resource manager, system watchdog, docker watchdog, fallback state
|
||||
# 🔐 Security — SSL cert expiry per domain
|
||||
# 📊 Emby — weekly stream count, active now, top users
|
||||
# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync
|
||||
@@ -468,17 +468,21 @@ fi
|
||||
section "🛡️ WATCHDOG"
|
||||
|
||||
# ── System Watchdog ───────────────────────────────────────────────────────────────────────────
|
||||
line "⚙️ System Watchdog"
|
||||
SYS_PID=$(_get_lock_pid "system_watchdog")
|
||||
if _is_running "system_watchdog"; then
|
||||
SYS_AGE=$(_lock_age "system_watchdog")
|
||||
line " ✅ Running │ PID: $SYS_PID │ Uptime: $(_fmt_uptime "$SYS_AGE") │ ~Cycle: $(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))"
|
||||
line "⚙️ System Watchdog (cron via watchdog_orchestrator)"
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
_sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||||
_sw_ago=$(( NOW - _sw_last ))
|
||||
if [[ "$_sw_ago" -lt 600 ]]; then
|
||||
line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago"
|
||||
else
|
||||
issue "system_watchdog NOT RUNNING"
|
||||
issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running"
|
||||
fi
|
||||
else
|
||||
issue " system_watchdog has never run (state file missing)"
|
||||
fi
|
||||
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=\|^mem_shutdown" \
|
||||
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \
|
||||
"$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$SYS_ACTIVE" ]]; then
|
||||
while IFS=: read -r key count; do
|
||||
@@ -519,13 +523,17 @@ line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB
|
||||
REPORT+=("")
|
||||
|
||||
# ── Docker Watchdog ───────────────────────────────────────────────────────────────────────────
|
||||
line "🐳 Docker Watchdog"
|
||||
DOCKER_PID=$(_get_lock_pid "docker_watchdog")
|
||||
if _is_running "docker_watchdog"; then
|
||||
DOCKER_AGE=$(_lock_age "docker_watchdog")
|
||||
line " ✅ Running │ PID: $DOCKER_PID │ Uptime: $(_fmt_uptime "$DOCKER_AGE") │ ~Cycle: $(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))"
|
||||
line "🐳 Docker Watchdog (cron via watchdog_orchestrator)"
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
_dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||||
_dw_ago=$(( NOW - _dw_last ))
|
||||
if [[ "$_dw_ago" -lt 600 ]]; then
|
||||
line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago"
|
||||
else
|
||||
issue "docker_watchdog NOT RUNNING"
|
||||
issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running"
|
||||
fi
|
||||
else
|
||||
issue " docker_watchdog has never run (state file missing)"
|
||||
fi
|
||||
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
@@ -554,6 +562,26 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Resource Manager ─────────────────────────────────────────────────────────────────────────
|
||||
line "🎛️ Resource Manager"
|
||||
if [[ -f "$RM_STATE_FILE" ]]; then
|
||||
_rm_last=$(stat -c %Y "$RM_STATE_FILE" 2>/dev/null || echo 0)
|
||||
_rm_ago=$(( NOW - _rm_last ))
|
||||
_rm_level=$(grep "^current_level:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2)
|
||||
_rm_level="${_rm_level:-0}"
|
||||
if [[ "$_rm_level" -gt 0 ]]; then
|
||||
issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||||
elif [[ "$_rm_ago" -lt 600 ]]; then
|
||||
line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||||
else
|
||||
issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running"
|
||||
fi
|
||||
else
|
||||
line " ℹ️ State file not found (resource_manager may not have run yet)"
|
||||
fi
|
||||
|
||||
REPORT+=("")
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||||
|
||||
Executable
+192
@@ -0,0 +1,192 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================ Watchdog Orchestrator ===========================================
|
||||
# ==============================================================================================
|
||||
# Runs the three-layer watchdog system in the correct sequence each cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. resource_manager.sh — reduce system pressure intelligently
|
||||
# 2. docker_watchdog.sh — heal containers with freed resources
|
||||
# 3. system_watchdog.sh — reboot if all else fails (last line of defense)
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# Resource Manager first — frees RAM and CPU before healing attempts container restarts.
|
||||
# Containers restarted into a resource-pressured system just fail again.
|
||||
# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
|
||||
# System Watchdog last — only triggers if prior layers could not resolve the issue.
|
||||
# Rebooting without first reducing pressure may reboot into the same state.
|
||||
#
|
||||
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
|
||||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
|
||||
# Prevents false positives from containers still starting at array launch.
|
||||
# Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
|
||||
#
|
||||
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock() — exits immediately if a prior cycle is still in progress.
|
||||
# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||||
#
|
||||
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
|
||||
# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
|
||||
# Those scripts are now single-pass — this orchestrator provides the cadence.
|
||||
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_orchestrator.sh — normal run (called by cron every minute)
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
|
||||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Skip immediately if another cycle is still running — no pile-up
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
RESOURCE_MANAGER="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_manager.sh"
|
||||
DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
|
||||
SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||||
else
|
||||
log "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Sub-scripts ──"
|
||||
for pair in \
|
||||
"Resource Manager:$RESOURCE_MANAGER" \
|
||||
"Docker Watchdog:$DOCKER_WATCHDOG" \
|
||||
"System Watchdog:$SYSTEM_WATCHDOG"; do
|
||||
label="${pair%%:*}"
|
||||
script="${pair#*:}"
|
||||
if [[ -f "$script" ]]; then
|
||||
[[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${script##*/}"
|
||||
else
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $script"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Schedule: * * * * * (every minute via User Scripts)"
|
||||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Startup Grace ━━━
|
||||
# ==============================================================================================
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
log "Startup grace — ${UPTIME_SECONDS}s / ${WATCHDOG_STARTUP_GRACE}s — skipping cycle"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Watchdog Cycle ━━━
|
||||
# ==============================================================================================
|
||||
CYCLE_START=$(date +%s)
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
run_watchdog() {
|
||||
local name="$1" script="$2"
|
||||
|
||||
if [[ ! -f "$script" ]]; then
|
||||
error "$name — not found: $script"
|
||||
FAIL+=("$name:missing")
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ ! -x "$script" ]] && chmod +x "$script"
|
||||
|
||||
local extra_args=()
|
||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||
|
||||
log "$ICON_START $name"
|
||||
if bash "$script" "${extra_args[@]}"; then
|
||||
PASS+=("$name")
|
||||
return 0
|
||||
else
|
||||
error "$name — non-zero exit"
|
||||
FAIL+=("$name")
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_watchdog "Resource Manager" "$RESOURCE_MANAGER"
|
||||
run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG"
|
||||
run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG"
|
||||
|
||||
CYCLE_END=$(date +%s)
|
||||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Heartbeat ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
HB_COUNT_FILE="/tmp/watchdog_orch_hb.count"
|
||||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||||
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — only shown on failures or --log ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
fi
|
||||
fi
|
||||
@@ -572,6 +572,8 @@ detect_hosts() {
|
||||
_alias_array "PARTNERSHIP_AUTH_WEBUIS"
|
||||
_alias_array "PARTNERSHIP_MIRROR_BACKUPS"
|
||||
_alias_array "PARTNERSHIP_OWN_CONTAINERS"
|
||||
_alias_array "RM_PAUSE_CONTAINERS"
|
||||
_alias_array "RM_STOP_CONTAINERS"
|
||||
|
||||
# ── Set array aliases — associative arrays ────────────────────────────────
|
||||
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
||||
|
||||
+86
-36
@@ -278,7 +278,9 @@
|
||||
# Scripts launched by array_started.sh when the array comes online.
|
||||
# Launched in order — each as a background process.
|
||||
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||||
# Continuous scripts (watchdogs, failover) run until array stops.
|
||||
# Continuous scripts (failover) run until array stops.
|
||||
# Watchdogs (resource_manager, docker_watchdog, system_watchdog) are cronned via
|
||||
# watchdog_orchestrator.sh — NOT launched here.
|
||||
ARRAY_START_SCRIPTS=(
|
||||
"git_pull_execute.sh" # pull latest scripts before anything starts
|
||||
"Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
||||
@@ -286,9 +288,7 @@
|
||||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||||
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||||
"unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop
|
||||
"Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop
|
||||
"Fallback/fallback.sh" # mutual failover — HOST2 back online
|
||||
"Fallback/fallback.sh" # mutual failover — continuous
|
||||
)
|
||||
|
||||
# ━━━ Intermediate Sync Maintenance ━━━
|
||||
@@ -1114,12 +1114,85 @@
|
||||
EMBY_REPORT_DAYS=7 # days to include in the report period
|
||||
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||||
|
||||
# ==============================================================================================
|
||||
# ── WATCHDOG ORCHESTRATOR ─────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Runs resource_manager → docker_watchdog → system_watchdog in sequence each cron cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||||
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Pressure reduction layer — keeps the system comfortable before things break.
|
||||
# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop.
|
||||
#
|
||||
# ── PRESSURE LEVELS ───────────────────────────────────────────────────────────────────────────
|
||||
# Level 1 (soft) — throttle SABnzbd + qBit download speeds
|
||||
# Level 2 (medium) — further throttle + docker pause background containers
|
||||
# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer
|
||||
#
|
||||
# ── PER-HOST CONTAINER LISTS ──────────────────────────────────────────────────────────────────
|
||||
# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (in master_host*.conf)
|
||||
# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (in master_host*.conf)
|
||||
|
||||
RM_ENABLED=true
|
||||
RM_STATE_FILE="/tmp/resource_manager_state.db"
|
||||
|
||||
# ━━━ Pressure Thresholds ━━━
|
||||
# Graduated RAM response — resource_manager acts before system_watchdog reboots.
|
||||
# RM_RAM_SOFT_GB > RM_RAM_MEDIUM_GB > RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always
|
||||
RM_RAM_SOFT_GB=12 # throttle start — reduce background load
|
||||
RM_RAM_MEDIUM_GB=8 # pause background containers
|
||||
RM_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB)
|
||||
RM_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers
|
||||
|
||||
# Load average thresholds — multiplier × core count
|
||||
RM_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained
|
||||
RM_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained
|
||||
|
||||
# Consecutive runs at lower pressure before de-escalating
|
||||
RM_RECOVER_CYCLES=3
|
||||
|
||||
# ━━━ SABnzbd Throttle ━━━
|
||||
# Speed values: "50M" = 50 MB/s, "0" = unlimited
|
||||
RM_SABNZBD_ENABLED=true
|
||||
RM_SABNZBD_SPEED_SOFT="50M"
|
||||
RM_SABNZBD_SPEED_MEDIUM="10M"
|
||||
|
||||
# ━━━ qBittorrent Throttle ━━━
|
||||
# KB/s — 0 = unlimited
|
||||
RM_QBIT_ENABLED=true
|
||||
RM_QBIT_DL_SOFT=51200 # 50 MB/s
|
||||
RM_QBIT_DL_MEDIUM=10240 # 10 MB/s
|
||||
|
||||
# ━━━ Critical Containers ━━━
|
||||
# Never paused or stopped regardless of pressure level.
|
||||
# Keep DNS, auth, media serving, and live TV always running.
|
||||
RM_CRITICAL_CONTAINERS=(
|
||||
"NginxProxyManager" # reverse proxy — internet access
|
||||
"Authelia" # auth — nothing accessible without it
|
||||
"Authelia-Secondary"
|
||||
"Mariadb-Authelia" # Authelia dependency
|
||||
"Mariadb-Authelia-Secondary"
|
||||
"Redis-Authelia" # Authelia dependency
|
||||
"Redis-Authelia-Secondary"
|
||||
"AdGuard-Home" # DNS — all LAN resolution
|
||||
"Emby" # media server — Live TV buffering
|
||||
"Dispatcharr" # Live TV scheduler — loses state if stopped
|
||||
"Dispatcharr-Basic"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Continuous system health monitoring — last line of defense before a crash.
|
||||
# Started by array_started.sh — runs until array stops.
|
||||
# Re-sources all three conf files each cycle — config changes take effect on next cycle.
|
||||
# Single-pass system health check — last line of defense before a crash.
|
||||
# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh.
|
||||
# Re-sources config at each orchestrator run — config changes take effect immediately.
|
||||
#
|
||||
# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
|
||||
# CRITICAL — bypass ALL strikes, reboot immediately
|
||||
@@ -1132,16 +1205,9 @@
|
||||
# STANDARD — strike system (N consecutive failures → reboot)
|
||||
# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
|
||||
#
|
||||
# ── RAM TIERS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# MEM_WARN_GB — warn + notify only (informational)
|
||||
# MEM_SHUTDOWN_GB — stop non-essential containers, recover above MEM_RECOVER_GB
|
||||
# MEM_GB — strike system → reboot (or bypass with OOM)
|
||||
#
|
||||
# ── CONTAINER SHUTDOWN ────────────────────────────────────────────────────────────────────────
|
||||
# At MEM_SHUTDOWN_GB: stop all containers NOT in MEM_SHUTDOWN_EXCLUDED list
|
||||
# Excluded containers stay running — DNS, auth, Emby, Dispatcharr
|
||||
# Stopped containers stay stopped until RAM recovers above MEM_RECOVER_GB
|
||||
# Strike list applied — doesn't flip-flop every cycle
|
||||
# ── RAM ───────────────────────────────────────────────────────────────────────────────────────
|
||||
# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass)
|
||||
# Warn/shutdown/recover RAM tiers are handled by resource_manager.sh
|
||||
|
||||
# ━━━ State Files ━━━
|
||||
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅
|
||||
@@ -1167,26 +1233,10 @@
|
||||
SYSTEM_WATCHDOG_HEARTBEAT=true
|
||||
SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1
|
||||
|
||||
# ━━━ RAM Tiers ━━━
|
||||
# Three-level RAM response — graduated action instead of single threshold.
|
||||
# HOST1 has 128GB, HOST2 has 64GB — adjust accordingly.
|
||||
# MEM_WARN_GB > MEM_SHUTDOWN_GB > MEM_GB always
|
||||
SYS_WATCHDOG_MEM_WARN_GB=10 # warn + notify — informational only
|
||||
SYS_WATCHDOG_MEM_SHUTDOWN_GB=6 # stop non-essential containers
|
||||
# ━━━ RAM Reboot Threshold ━━━
|
||||
# Reboot trigger only — warn/shutdown/recover handled by resource_manager.sh
|
||||
# RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots)
|
||||
SYS_WATCHDOG_MEM_GB=4 # strike system → reboot
|
||||
SYS_WATCHDOG_MEM_RECOVER_GB=30 # RAM must recover above this before restarting containers
|
||||
|
||||
# Containers excluded from RAM emergency shutdown.
|
||||
# These stay running regardless of RAM pressure.
|
||||
# DNS and auth must stay up, Emby and Dispatcharr for Live TV continuity.
|
||||
SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=(
|
||||
"NginxProxyManager" # DNS / reverse proxy — internet access
|
||||
"Authelia" # auth — without this nothing is accessible
|
||||
"Mariadb" # Authelia dependency
|
||||
"Redis" # Authelia dependency
|
||||
"Emby" # media server — Live TV buffering
|
||||
"Dispatcharr" # Live TV scheduler — loses state if stopped
|
||||
)
|
||||
|
||||
# ━━━ OOM Bypass Settings ━━━
|
||||
# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
|
||||
|
||||
@@ -670,6 +670,32 @@
|
||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Containers to manage under pressure — see master.conf RM_CRITICAL_CONTAINERS for exclusions.
|
||||
|
||||
# docker pause at medium pressure (RAM < RM_RAM_MEDIUM_GB or load > medium threshold)
|
||||
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||
HOST1_RM_PAUSE_CONTAINERS=(
|
||||
"Huntarr" # arr search automation — safe to suspend
|
||||
"Cleanuparr" # download cleanup — safe to suspend
|
||||
"Healarr" # arr health checks — safe to suspend
|
||||
"Soularr" # Slskd automation — background only
|
||||
"ChannelTube" # YouTube archiver — background only
|
||||
"Pinchflat" # YouTube archiver — background only
|
||||
)
|
||||
|
||||
# docker stop at hard pressure (RAM < RM_RAM_HARD_GB)
|
||||
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||
# resource_manager.sh restarts them when pressure fully clears (RAM >= RM_RAM_RECOVER_GB).
|
||||
HOST1_RM_STOP_CONTAINERS=(
|
||||
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||
"7DaysToDie" # game server — optional
|
||||
"V-Rising" # game server — optional
|
||||
"Code-Server" # IDE — not needed during pressure events
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
Executable
+523
@@ -0,0 +1,523 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Resource Manager ===========================================
|
||||
# ==============================================================================================
|
||||
# Pressure reduction layer — detects rising system load and reduces it intelligently.
|
||||
# Called by watchdog_orchestrator.sh every minute — single-pass, not a continuous loop.
|
||||
#
|
||||
# ── RESPONSIBILITY ────────────────────────────────────────────────────────────────────────────
|
||||
# Reduce system pressure before things break. NOT fixing broken containers (docker_watchdog)
|
||||
# and NOT rebooting (system_watchdog). The middle layer that keeps the system comfortable.
|
||||
#
|
||||
# "Pressure is rising — reduce load intelligently."
|
||||
#
|
||||
# ── THREE-LEVEL PRESSURE RESPONSE ─────────────────────────────────────────────────────────────
|
||||
#
|
||||
# Level 1 — SOFT (RAM < RM_RAM_SOFT_GB OR load > RM_LOAD_SOFT_MULTIPLIER × cores):
|
||||
# Throttle SABnzbd download speed to RM_SABNZBD_SPEED_SOFT
|
||||
# Throttle qBittorrent download to RM_QBIT_DL_SOFT KB/s
|
||||
#
|
||||
# Level 2 — MEDIUM (RAM < RM_RAM_MEDIUM_GB OR load > RM_LOAD_MEDIUM_MULTIPLIER × cores):
|
||||
# Further throttle SABnzbd + qBittorrent to medium limits
|
||||
# docker pause RM_PAUSE_CONTAINERS — suspend without losing state, instant reversible
|
||||
#
|
||||
# Level 3 — HARD (RAM < RM_RAM_HARD_GB):
|
||||
# docker stop RM_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.)
|
||||
# Write mem_shutdown_active=true — signals docker_watchdog to defer container restarts
|
||||
#
|
||||
# ── RECOVERY ──────────────────────────────────────────────────────────────────────────────────
|
||||
# Pressure must stay below current action threshold for RM_RECOVER_CYCLES consecutive runs
|
||||
# before restoring. De-escalates one level at a time to avoid re-triggering immediately.
|
||||
# Level 3 de-escalation additionally requires RAM >= RM_RAM_RECOVER_GB before un-stopping.
|
||||
#
|
||||
# ── COORDINATION WITH DOCKER WATCHDOG ─────────────────────────────────────────────────────────
|
||||
# At level 3: writes mem_shutdown_active=true to RM_STATE_FILE.
|
||||
# docker_watchdog.sh reads this and defers all container restart logic.
|
||||
# Cleared when pressure fully resolves and containers are restarted.
|
||||
# This prevents docker_watchdog from restarting containers that RM just stopped to free RAM.
|
||||
#
|
||||
# ── NOT RESPONSIBLE FOR ───────────────────────────────────────────────────────────────────────
|
||||
# Restarting broken containers — docker_watchdog.sh
|
||||
# Rebooting the system — system_watchdog.sh
|
||||
# Reacting to single data points — RM_RECOVER_CYCLES prevents flip-flopping
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# RM_ENABLED, RM_STATE_FILE
|
||||
# RM_RAM_SOFT_GB, RM_RAM_MEDIUM_GB, RM_RAM_HARD_GB, RM_RAM_RECOVER_GB
|
||||
# RM_LOAD_SOFT_MULTIPLIER, RM_LOAD_MEDIUM_MULTIPLIER
|
||||
# RM_RECOVER_CYCLES
|
||||
# RM_SABNZBD_ENABLED, RM_SABNZBD_SPEED_SOFT, RM_SABNZBD_SPEED_MEDIUM
|
||||
# RM_QBIT_ENABLED, RM_QBIT_DL_SOFT, RM_QBIT_DL_MEDIUM
|
||||
# RM_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts)
|
||||
# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts)
|
||||
# HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY
|
||||
# HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# resource_manager.sh — normal run (via watchdog_orchestrator.sh)
|
||||
# resource_manager.sh --dry-run — show what would happen without acting
|
||||
# resource_manager.sh --status — current pressure level and active actions
|
||||
# resource_manager.sh --log — verbose per-check output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${RM_ENABLED:-true}" != "true" ]]; then
|
||||
log "Resource Manager disabled (RM_ENABLED=false)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
touch "$RM_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $RM_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ State Helpers ━━━
|
||||
# ==============================================================================================
|
||||
# rm_state_get/set use : separator for RM-internal state
|
||||
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
||||
|
||||
rm_state_get() {
|
||||
grep -E "^${1}:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2-
|
||||
}
|
||||
|
||||
rm_state_set() {
|
||||
local key="$1" val="$2"
|
||||
grep -vE "^${key}:" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp"
|
||||
echo "${key}:${val}" >> "${RM_STATE_FILE}.tmp"
|
||||
mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE"
|
||||
}
|
||||
|
||||
rm_state_get_eq() {
|
||||
grep -E "^${1}=" "$RM_STATE_FILE" 2>/dev/null | cut -d= -f2-
|
||||
}
|
||||
|
||||
rm_state_set_eq() {
|
||||
local key="$1" val="$2"
|
||||
grep -vE "^${key}=" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp"
|
||||
echo "${key}=${val}" >> "${RM_STATE_FILE}.tmp"
|
||||
mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Load State ━━━
|
||||
# ==============================================================================================
|
||||
CURRENT_LEVEL=$(rm_state_get "rm_action_level"); CURRENT_LEVEL=${CURRENT_LEVEL:-0}
|
||||
RECOVER_CYCLES=$(rm_state_get "rm_recover_cycles"); RECOVER_CYCLES=${RECOVER_CYCLES:-0}
|
||||
PAUSED_LIST=$(rm_state_get "rm_paused_containers"); PAUSED_LIST=${PAUSED_LIST:-""}
|
||||
STOPPED_LIST=$(rm_state_get "rm_stopped_containers"); STOPPED_LIST=${STOPPED_LIST:-""}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pressure Calculation ━━━
|
||||
# ==============================================================================================
|
||||
TOTAL_CORES=$(nproc)
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
RM_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_SOFT_MULTIPLIER:-2.0}}")
|
||||
RM_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_MEDIUM_MULTIPLIER:-3.0}}")
|
||||
|
||||
TARGET_LEVEL=0
|
||||
TARGET_REASON=""
|
||||
if [[ "$MEM_GB" -lt "${RM_RAM_HARD_GB:-6}" ]]; then
|
||||
TARGET_LEVEL=3
|
||||
TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RM_RAM_HARD_GB}GB"
|
||||
elif [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]]; then
|
||||
TARGET_LEVEL=2
|
||||
[[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RM_RAM_MEDIUM_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RM_LOAD_MEDIUM_THRESH}"
|
||||
elif [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]]; then
|
||||
TARGET_LEVEL=1
|
||||
[[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RM_RAM_SOFT_GB}GB"
|
||||
[[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RM_LOAD_SOFT_THRESH}"
|
||||
fi
|
||||
|
||||
LEVEL_NAMES=("normal" "soft" "medium" "hard")
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY RESOURCE MANAGER STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "── Current State ──"
|
||||
echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})"
|
||||
echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||
echo " Recover cycles: $RECOVER_CYCLES / ${RM_RECOVER_CYCLES:-3}"
|
||||
[[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST"
|
||||
[[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST"
|
||||
MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active")
|
||||
[[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)"
|
||||
echo ""
|
||||
echo "── System Pressure ──"
|
||||
echo " RAM free: ${MEM_GB}GB (soft:<${RM_RAM_SOFT_GB} medium:<${RM_RAM_MEDIUM_GB} hard:<${RM_RAM_HARD_GB} recover:>=${RM_RAM_RECOVER_GB})"
|
||||
echo " Load avg: ${LOAD} (soft:>=${RM_LOAD_SOFT_THRESH} medium:>=${RM_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})"
|
||||
echo ""
|
||||
echo "── Configuration ──"
|
||||
echo " SABnzbd throttle: ${RM_SABNZBD_ENABLED:-true} soft=${RM_SABNZBD_SPEED_SOFT} medium=${RM_SABNZBD_SPEED_MEDIUM}"
|
||||
echo " qBit throttle: ${RM_QBIT_ENABLED:-true} soft=${RM_QBIT_DL_SOFT}KB/s medium=${RM_QBIT_DL_MEDIUM}KB/s"
|
||||
echo ""
|
||||
echo "── Container Lists (this host) ──"
|
||||
echo " Pause at medium: ${RM_PAUSE_CONTAINERS[*]:-none configured}"
|
||||
echo " Stop at hard: ${RM_STOP_CONTAINERS[*]:-none configured}"
|
||||
echo " Critical (never touched): ${RM_CRITICAL_CONTAINERS[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Container Guard ━━━
|
||||
# ==============================================================================================
|
||||
# Returns 0 if container is safe to pause/stop, 1 if it is critical
|
||||
is_critical() {
|
||||
local container="$1"
|
||||
for c in "${RM_CRITICAL_CONTAINERS[@]:-}"; do
|
||||
[[ "$c" == "$container" ]] && return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd API ━━━
|
||||
# ==============================================================================================
|
||||
sabnzbd_set_speed() {
|
||||
local speed="$1"
|
||||
[[ "${RM_SABNZBD_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set SABnzbd speed to $speed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
curl -sf --max-time 10 \
|
||||
"${SABNZBD_URL}/api?mode=config&name=speedlimit&value=${speed}&apikey=${SABNZBD_API_KEY}" \
|
||||
>/dev/null 2>&1 && log "SABnzbd speed → $speed" || warn "SABnzbd API call failed"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ qBittorrent API ━━━
|
||||
# ==============================================================================================
|
||||
QBIT_COOKIE="/tmp/rm_qbit_cookie.txt"
|
||||
|
||||
qbit_login() {
|
||||
[[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0
|
||||
|
||||
curl -sf --max-time 10 -c "$QBIT_COOKIE" \
|
||||
-X POST "${QBIT_URL}/api/v2/auth/login" \
|
||||
-d "username=${QBIT_USERNAME}&password=${QBIT_PASSWORD}" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
qbit_set_dl_limit() {
|
||||
local kbps="$1" # KB/s — 0 = unlimited
|
||||
[[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||
[[ -z "$QBIT_URL" ]] && return 0
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set qBit download limit to ${kbps}KB/s"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local bps=$(( kbps * 1024 ))
|
||||
qbit_login
|
||||
curl -sf --max-time 10 -b "$QBIT_COOKIE" \
|
||||
-X POST "${QBIT_URL}/api/v2/transfer/setDownloadLimit" \
|
||||
-d "limit=${bps}" >/dev/null 2>&1 && log "qBit download limit → ${kbps}KB/s" || warn "qBit API call failed"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Container Actions ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
# Pause a list of containers — returns newline-separated list of actually-paused containers
|
||||
pause_containers() {
|
||||
local actually_paused=()
|
||||
for container in "$@"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
is_critical "$container" || { log "$container — critical, skipping pause"; continue; }
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" != "running" ]]; then
|
||||
log "$container — not running (status: ${status:-unknown}), skipping pause"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker pause $container"
|
||||
actually_paused+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker pause "$container" >/dev/null 2>&1; then
|
||||
warn "Paused $container (medium pressure)"
|
||||
actually_paused+=("$container")
|
||||
else
|
||||
error "Failed to pause $container"
|
||||
fi
|
||||
done
|
||||
printf '%s,' "${actually_paused[@]}" | sed 's/,$//'
|
||||
}
|
||||
|
||||
# Unpause a comma-separated list of containers
|
||||
unpause_containers() {
|
||||
local IFS=','
|
||||
for container in $1; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" != "paused" ]]; then
|
||||
log "$container — not paused (status: ${status:-unknown}), skipping unpause"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker unpause $container"
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker unpause "$container" >/dev/null 2>&1; then
|
||||
warn "Unpaused $container (pressure reduced)"
|
||||
else
|
||||
error "Failed to unpause $container"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Stop a list of containers — returns comma-separated list of actually-stopped containers
|
||||
stop_containers() {
|
||||
local actually_stopped=()
|
||||
for container in "$@"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
is_critical "$container" || { log "$container — critical, skipping stop"; continue; }
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" != "running" && "$status" != "paused" ]]; then
|
||||
log "$container — not running (status: ${status:-unknown}), skipping stop"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker stop $container"
|
||||
actually_stopped+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||||
warn "Stopped $container (hard pressure)"
|
||||
actually_stopped+=("$container")
|
||||
else
|
||||
error "Failed to stop $container"
|
||||
fi
|
||||
done
|
||||
printf '%s,' "${actually_stopped[@]}" | sed 's/,$//'
|
||||
}
|
||||
|
||||
# Start a comma-separated list of containers (only those RM stopped)
|
||||
start_containers() {
|
||||
local IFS=','
|
||||
for container in $1; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||
if [[ "$status" == "running" ]]; then
|
||||
log "$container — already running"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would docker start $container"
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||||
warn "Started $container (pressure cleared)"
|
||||
else
|
||||
error "Failed to start $container"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Apply Level Actions ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
apply_level_1() {
|
||||
log "Applying level 1 (soft) — throttling downloaders"
|
||||
sabnzbd_set_speed "${RM_SABNZBD_SPEED_SOFT:-50M}"
|
||||
qbit_set_dl_limit "${RM_QBIT_DL_SOFT:-51200}"
|
||||
}
|
||||
|
||||
apply_level_2() {
|
||||
log "Applying level 2 (medium) — throttling + pausing background containers"
|
||||
sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}"
|
||||
qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}"
|
||||
|
||||
if [[ ${#RM_PAUSE_CONTAINERS[@]} -gt 0 ]]; then
|
||||
local newly_paused
|
||||
newly_paused=$(pause_containers "${RM_PAUSE_CONTAINERS[@]}")
|
||||
# Merge with existing paused list (avoid duplicates on re-escalation)
|
||||
if [[ -n "$newly_paused" ]]; then
|
||||
if [[ -n "$PAUSED_LIST" ]]; then
|
||||
PAUSED_LIST="${PAUSED_LIST},${newly_paused}"
|
||||
else
|
||||
PAUSED_LIST="$newly_paused"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
apply_level_3() {
|
||||
log "Applying level 3 (hard) — stopping optional containers"
|
||||
sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2
|
||||
qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}"
|
||||
|
||||
if [[ ${#RM_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||||
local newly_stopped
|
||||
newly_stopped=$(stop_containers "${RM_STOP_CONTAINERS[@]}")
|
||||
if [[ -n "$newly_stopped" ]]; then
|
||||
if [[ -n "$STOPPED_LIST" ]]; then
|
||||
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
||||
else
|
||||
STOPPED_LIST="$newly_stopped"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Signal docker_watchdog to defer container restarts
|
||||
rm_state_set_eq "mem_shutdown_active" "true"
|
||||
warn "mem_shutdown_active=true — docker_watchdog will defer restarts"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Restore Level Actions ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
restore_level_3() {
|
||||
log "Restoring from level 3 — starting stopped containers"
|
||||
if [[ -n "$STOPPED_LIST" ]]; then
|
||||
start_containers "$STOPPED_LIST"
|
||||
STOPPED_LIST=""
|
||||
fi
|
||||
rm_state_set_eq "mem_shutdown_active" "false"
|
||||
warn "mem_shutdown_active=false — docker_watchdog restoring normal operation"
|
||||
}
|
||||
|
||||
restore_level_2() {
|
||||
log "Restoring from level 2 — unpausing containers"
|
||||
if [[ -n "$PAUSED_LIST" ]]; then
|
||||
unpause_containers "$PAUSED_LIST"
|
||||
PAUSED_LIST=""
|
||||
fi
|
||||
}
|
||||
|
||||
restore_level_1() {
|
||||
log "Restoring from level 1 — removing downloader throttle"
|
||||
sabnzbd_set_speed "0"
|
||||
qbit_set_dl_limit 0
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pressure Decision ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Resource Manager — $MY_ID — $(date '+%H:%M:%S') ━━━"
|
||||
echo " RAM: ${MEM_GB}GB free Load: ${LOAD} Action: ${CURRENT_LEVEL} → ${TARGET_LEVEL} (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||
|
||||
if [[ "$TARGET_LEVEL" -gt "$CURRENT_LEVEL" ]]; then
|
||||
# ── Escalate ────────────────────────────────────────────────────────────────────────────
|
||||
warn "Pressure escalating to level $TARGET_LEVEL — $TARGET_REASON"
|
||||
notify "Resource Manager: pressure level $TARGET_LEVEL on $(hostname) ($MY_ID) — $TARGET_REASON" \
|
||||
"Resource Manager" "warning"
|
||||
|
||||
for (( lvl = CURRENT_LEVEL + 1; lvl <= TARGET_LEVEL; lvl++ )); do
|
||||
case "$lvl" in
|
||||
1) apply_level_1 ;;
|
||||
2) apply_level_2 ;;
|
||||
3) apply_level_3 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
rm_state_set "rm_action_level" "$TARGET_LEVEL"
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
|
||||
elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
||||
# ── Tracking recovery ───────────────────────────────────────────────────────────────────
|
||||
RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 ))
|
||||
rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES"
|
||||
log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RM_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions"
|
||||
|
||||
if [[ "$RECOVER_CYCLES" -ge "${RM_RECOVER_CYCLES:-3}" ]]; then
|
||||
# Level 3 de-escalation requires RAM above recover threshold
|
||||
if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RM_RAM_RECOVER_GB:-20}" ]]; then
|
||||
warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RM_RAM_RECOVER_GB}GB"
|
||||
else
|
||||
warn "Pressure sustained below level $CURRENT_LEVEL — restoring"
|
||||
case "$CURRENT_LEVEL" in
|
||||
3) restore_level_3 ;;
|
||||
2) restore_level_2 ;;
|
||||
1) restore_level_1 ;;
|
||||
esac
|
||||
|
||||
NEW_LEVEL=$(( CURRENT_LEVEL - 1 ))
|
||||
rm_state_set "rm_action_level" "$NEW_LEVEL"
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||
|
||||
if [[ "$NEW_LEVEL" -gt 0 ]]; then
|
||||
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RM_RECOVER_CYCLES:-3} more cycles to fully clear"
|
||||
else
|
||||
log "All pressure cleared — system at normal operation ✅"
|
||||
notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \
|
||||
"Resource Manager" "normal"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
# ── Steady state ────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||||
log "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||||
else
|
||||
log "System at normal pressure ✅"
|
||||
fi
|
||||
rm_state_set "rm_recover_cycles" 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Persist State ━━━
|
||||
# ==============================================================================================
|
||||
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
||||
touch "$RM_STATE_FILE" 2>/dev/null
|
||||
@@ -88,12 +88,13 @@ validate_unraid_cmd \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "continuous"
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
TOTAL_CORES=$(nproc)
|
||||
DOCKER_TIMEOUT=10
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
|
||||
# Ensure state files exist
|
||||
for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \
|
||||
@@ -129,10 +130,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
||||
echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%"
|
||||
echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%"
|
||||
echo "$ICON_MEM RAM warn: < ${SYS_WATCHDOG_MEM_WARN_GB}GB"
|
||||
echo "$ICON_MEM RAM shutdown: < ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB"
|
||||
echo "$ICON_MEM RAM recover: > ${SYS_WATCHDOG_MEM_RECOVER_GB}GB"
|
||||
echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)"
|
||||
echo " (RAM warn/shutdown/recover managed by resource_manager.sh)"
|
||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%"
|
||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)"
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||
@@ -141,10 +140,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr"
|
||||
echo ""
|
||||
echo "── Container Shutdown Excluded ──"
|
||||
for c in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do
|
||||
echo " $ICON_RUNNING $c"
|
||||
done
|
||||
echo ""
|
||||
echo "── Check Toggles ──"
|
||||
echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM"
|
||||
@@ -319,56 +314,6 @@ run_strike_check() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CONTAINER SHUTDOWN (RAM EMERGENCY) ────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
shutdown_non_essential_containers() {
|
||||
warn "RAM emergency — stopping non-essential containers"
|
||||
local stopped=()
|
||||
|
||||
# Build exclusion map
|
||||
declare -A EXCLUDED_MAP
|
||||
for exc in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do
|
||||
[[ -n "$exc" ]] && EXCLUDED_MAP["$exc"]=1
|
||||
done
|
||||
|
||||
# Stop all running containers not in exclusion list
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ -n "${EXCLUDED_MAP[$container]:-}" ]]; then
|
||||
log "$container — excluded from RAM shutdown, leaving running"
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
|
||||
warn "Stopped $container (RAM emergency)" && \
|
||||
stopped+=("$container") || \
|
||||
error "Failed to stop $container"
|
||||
else
|
||||
warn "DRY RUN — would stop $container (RAM emergency)"
|
||||
stopped+=("$container")
|
||||
fi
|
||||
done < <(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
if [[ ${#stopped[@]} -gt 0 ]]; then
|
||||
set_state_val "mem_shutdown_active" "true"
|
||||
notify "RAM emergency on $(hostname) ($MY_ID) — stopped ${#stopped[@]} containers. Excluded: ${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[*]}" \
|
||||
"System Watchdog" "warning"
|
||||
warn "Stopped ${#stopped[@]} containers — waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB"
|
||||
fi
|
||||
}
|
||||
|
||||
restart_non_essential_containers() {
|
||||
warn "RAM recovered — restarting containers that were stopped in emergency"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
set_state_val "mem_shutdown_active" "false"
|
||||
fi
|
||||
# docker_watchdog.sh will detect stopped required containers and restart them
|
||||
# We just clear the state flag here
|
||||
warn "Cleared RAM emergency state — docker_watchdog.sh will restart required containers"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -457,36 +402,11 @@ do_reboot() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Clean Shutdown ━━━
|
||||
# ━━━ Single-Pass Health Check ━━━
|
||||
# ==============================================================================================
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
warn "System watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Continuous Monitoring Loop ━━━
|
||||
# ==============================================================================================
|
||||
warn "System watchdog started — $MY_ID — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
warn "System watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
(( CYCLE++ ))
|
||||
|
||||
# Re-source config each cycle — picks up config changes without restart
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
detect_hosts
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
TRIGGERS=()
|
||||
CRITICAL_TRIGGERS=()
|
||||
URGENT_OOM_CONFIRMED=false
|
||||
@@ -584,12 +504,12 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
# ── Act on CRITICAL triggers immediately ─────────────────────────────────────────────────
|
||||
if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT — Cycle $CYCLE ━━━"
|
||||
echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT ━━━"
|
||||
for t in "${CRITICAL_TRIGGERS[@]}"; do
|
||||
error " CRITICAL: $t"
|
||||
done
|
||||
do_reboot "critical" "${CRITICAL_TRIGGERS[@]}"
|
||||
continue
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==========================================================================================
|
||||
@@ -638,60 +558,27 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
TRIGGERS+=("tmp=${TMP_USED}%")
|
||||
fi
|
||||
|
||||
# ── RAM tiers ─────────────────────────────────────────────────────────────────────────────
|
||||
# ── RAM — reboot tier only (warn/shutdown/recover handled by resource_manager.sh) ──────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
MEM_SHUTDOWN_ACTIVE=$(get_state_val "mem_shutdown_active")
|
||||
|
||||
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
||||
# Tier 2 check — bypass if OOM confirms crisis
|
||||
# Tier 2 — bypass strike system if OOM confirms active crisis
|
||||
if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \
|
||||
[[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then
|
||||
error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this cycle — URGENT bypass"
|
||||
error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this run — URGENT bypass"
|
||||
OOM_VICTIMS=$(get_oom_victims)
|
||||
URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}")
|
||||
[[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS")
|
||||
do_reboot "urgent" "${URGENT_TRIGGERS[@]}"
|
||||
continue
|
||||
exit 0
|
||||
fi
|
||||
# Standard strike path
|
||||
run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \
|
||||
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||
|
||||
elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_SHUTDOWN_GB" ]]; then
|
||||
reset_strikes "ram"
|
||||
# Container shutdown tier — but only once per event
|
||||
if [[ "$MEM_SHUTDOWN_ACTIVE" != "true" ]]; then
|
||||
warn "RAM ${MEM_GB}GB — below shutdown threshold ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB"
|
||||
run_strike_check "ram_shutdown" true "RAM shutdown tier ${MEM_GB}GB" && \
|
||||
shutdown_non_essential_containers
|
||||
else
|
||||
# Already shutdown — check if recovered
|
||||
if [[ "$MEM_GB" -ge "$SYS_WATCHDOG_MEM_RECOVER_GB" ]]; then
|
||||
warn "RAM recovered to ${MEM_GB}GB — clearing emergency state"
|
||||
restart_non_essential_containers
|
||||
reset_strikes "ram_shutdown"
|
||||
else
|
||||
warn "RAM ${MEM_GB}GB — still in emergency shutdown (recover threshold: ${SYS_WATCHDOG_MEM_RECOVER_GB}GB)"
|
||||
fi
|
||||
fi
|
||||
|
||||
elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_WARN_GB" ]]; then
|
||||
reset_strikes "ram"
|
||||
reset_strikes "ram_shutdown"
|
||||
warn "RAM ${MEM_GB}GB — below warning threshold ${SYS_WATCHDOG_MEM_WARN_GB}GB"
|
||||
local prev_ram_warn
|
||||
prev_ram_warn=$(get_strikes "ram_warn_notified")
|
||||
if [[ "${prev_ram_warn:-0}" -eq 0 ]]; then
|
||||
notify "RAM warning on $(hostname) ($MY_ID) — ${MEM_GB}GB free (threshold: ${SYS_WATCHDOG_MEM_WARN_GB}GB)" \
|
||||
"System Watchdog" "warning"
|
||||
set_strikes "ram_warn_notified" 1
|
||||
fi
|
||||
else
|
||||
reset_strikes "ram"
|
||||
reset_strikes "ram_shutdown"
|
||||
set_strikes "ram_warn_notified" 0
|
||||
log "RAM ${MEM_GB}GB free ✅"
|
||||
fi
|
||||
fi
|
||||
@@ -852,38 +739,17 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
# ==========================================================================================
|
||||
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
for t in "${TRIGGERS[@]}"; do
|
||||
echo " $ICON_REBOOT_SMART $t"
|
||||
done
|
||||
[[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this cycle: $OOM_DELTA"
|
||||
[[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this run: $OOM_DELTA"
|
||||
echo ""
|
||||
do_reboot "standard" "${TRIGGERS[@]}"
|
||||
exit 0
|
||||
else
|
||||
log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
|
||||
|
||||
# Heartbeat — periodic proof of life
|
||||
if [[ "${SYSTEM_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
UPTIME_APPROX=$(( CYCLE * SYSTEM_WATCHDOG_INTERVAL ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
||||
(( UPTIME_APPROX % HB_SECONDS < SYSTEM_WATCHDOG_INTERVAL )) && \
|
||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
||||
HB_HR=$(( UPTIME_APPROX / 3600 ))
|
||||
warn "♥ system_watchdog alive — $MY_ID — ~${HB_HR}hr uptime ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
log "System healthy ✅ ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
|
||||
# ── State file heartbeat — keep mtime fresh every cycle ───────────────────────────────────
|
||||
# docker_watchdog.sh uses state file mtime to detect stale RAM emergency flags.
|
||||
# If all checks pass with no set_state_val calls (e.g. KERNEL_OOPS + MDSTAT both disabled),
|
||||
# mtime would not update and stale guard would incorrectly resume docker_watchdog.sh.
|
||||
# Writing watchdog_cycle each tick guarantees mtime stays current while watchdog runs.
|
||||
set_state_val "watchdog_cycle" "$CYCLE"
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$SYSTEM_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
# Keep state file mtime fresh — docker_watchdog stale guard checks this
|
||||
set_state_val "watchdog_cycle" "$(date +%s)"
|
||||
Reference in New Issue
Block a user