feat: watchdog architecture v2 — resource manager + single-pass orchestrator
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
f16c962ac0
commit
309546e615
@@ -97,8 +97,8 @@ if [[ "$EUID" -ne 0 ]]; then
|
|||||||
fi
|
fi
|
||||||
success "Running as root"
|
success "Running as root"
|
||||||
|
|
||||||
# Continuous mode — skip gracefully if healthy instance already running
|
# Skip if another instance is running — no pile-up during long operations
|
||||||
acquire_lock "continuous"
|
acquire_lock
|
||||||
|
|
||||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||||||
detect_hosts
|
detect_hosts
|
||||||
@@ -332,8 +332,11 @@ is_parity_running() {
|
|||||||
|
|
||||||
WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt
|
WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt
|
||||||
WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying
|
WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying
|
||||||
WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers
|
# Loaded from state file — persists across single-pass runs
|
||||||
WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session
|
WATCHDOG_DAEMON_STRIKES=$(get_strikes "daemon_strikes" "$WATCHDOG_STATE_FILE")
|
||||||
|
WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES//[^0-9]/}"; WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES:-0}"
|
||||||
|
_dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE")
|
||||||
|
[[ "$_dr_raw" == "true" ]] && WATCHDOG_DAEMON_RESTARTED=true || WATCHDOG_DAEMON_RESTARTED=false
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
|
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
|
||||||
@@ -348,10 +351,10 @@ WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart thi
|
|||||||
# 1 = RAM emergency active — defer container management this cycle
|
# 1 = RAM emergency active — defer container management this cycle
|
||||||
|
|
||||||
check_system_watchdog_state() {
|
check_system_watchdog_state() {
|
||||||
# Returns 0 = normal operation | 1 = defer, RAM emergency active
|
# Returns 0 = normal operation | 1 = defer, resource_manager RAM emergency active
|
||||||
local state_file="$SYS_WATCHDOG_STATE_FILE"
|
local state_file="$RM_STATE_FILE"
|
||||||
|
|
||||||
# No state file = system_watchdog not running or not yet written — assume normal
|
# No state file = resource_manager not yet run — assume normal
|
||||||
[[ ! -f "$state_file" ]] && return 0
|
[[ ! -f "$state_file" ]] && return 0
|
||||||
|
|
||||||
local mem_shutdown
|
local mem_shutdown
|
||||||
@@ -361,7 +364,7 @@ check_system_watchdog_state() {
|
|||||||
|
|
||||||
# ── Stale state guard ─────────────────────────────────────────────────────────────────────
|
# ── Stale state guard ─────────────────────────────────────────────────────────────────────
|
||||||
# If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
|
# If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
|
||||||
# system_watchdog.sh may have died — don't be silenced forever by a stale flag.
|
# resource_manager.sh may not be running — don't defer indefinitely on stale state.
|
||||||
local state_mtime now age_seconds stale_limit=7200 # 2 hours
|
local state_mtime now age_seconds stale_limit=7200 # 2 hours
|
||||||
state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
|
state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
|
||||||
now=$(date +%s)
|
now=$(date +%s)
|
||||||
@@ -369,8 +372,7 @@ check_system_watchdog_state() {
|
|||||||
|
|
||||||
if [[ "$age_seconds" -gt "$stale_limit" ]]; then
|
if [[ "$age_seconds" -gt "$stale_limit" ]]; then
|
||||||
warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
|
warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
|
||||||
warn "system_watchdog.sh may not be running — resuming normal container management"
|
warn "resource_manager.sh may not be running — resuming normal container management"
|
||||||
warn "If RAM is still low this will be caught on next system_watchdog.sh cycle"
|
|
||||||
return 0 # Resume normal — don't defer indefinitely on stale state
|
return 0 # Resume normal — don't defer indefinitely on stale state
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -385,11 +387,14 @@ check_docker_daemon() {
|
|||||||
queue_notify "Docker daemon recovered on $(hostname)" "normal"
|
queue_notify "Docker daemon recovered on $(hostname)" "normal"
|
||||||
WATCHDOG_DAEMON_STRIKES=0
|
WATCHDOG_DAEMON_STRIKES=0
|
||||||
WATCHDOG_DAEMON_RESTARTED=false
|
WATCHDOG_DAEMON_RESTARTED=false
|
||||||
|
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||||
|
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||||
fi
|
fi
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
|
WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
|
||||||
|
set_strikes "daemon_strikes" "$WATCHDOG_DAEMON_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||||
warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"
|
warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"
|
||||||
|
|
||||||
if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
|
if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
|
||||||
@@ -415,6 +420,7 @@ check_docker_daemon() {
|
|||||||
|
|
||||||
# Restart daemon — unRAID uses rc.d scripts, not systemd
|
# Restart daemon — unRAID uses rc.d scripts, not systemd
|
||||||
WATCHDOG_DAEMON_RESTARTED=true
|
WATCHDOG_DAEMON_RESTARTED=true
|
||||||
|
set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE"
|
||||||
if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then
|
if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then
|
||||||
info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
|
info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
|
||||||
sleep "$WATCHDOG_DAEMON_RESTART_WAIT"
|
sleep "$WATCHDOG_DAEMON_RESTART_WAIT"
|
||||||
@@ -424,6 +430,8 @@ check_docker_daemon() {
|
|||||||
notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
|
notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
|
||||||
WATCHDOG_DAEMON_STRIKES=0
|
WATCHDOG_DAEMON_STRIKES=0
|
||||||
WATCHDOG_DAEMON_RESTARTED=false
|
WATCHDOG_DAEMON_RESTARTED=false
|
||||||
|
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||||
|
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
error "Docker daemon did not recover after restart"
|
error "Docker daemon did not recover after restart"
|
||||||
@@ -440,38 +448,14 @@ check_docker_daemon() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── CLEAN SHUTDOWN ────────────────────────────────────────────────────────────────────────────
|
# ━━━ Single-Pass Monitoring Run ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
WATCHDOG_RUNNING=true
|
info "$ICON_WATCHDOG Docker watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
echo ""
|
|
||||||
warn "Docker watchdog received shutdown signal — stopping cleanly"
|
|
||||||
WATCHDOG_RUNNING=false
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
trap cleanup SIGTERM SIGINT
|
|
||||||
|
|
||||||
# ==============================================================================================
|
|
||||||
# ━━━ Continuous Monitoring Loop ━━━
|
|
||||||
# ==============================================================================================
|
|
||||||
info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
CYCLE=0
|
CYCLE_START=$(date +%s)
|
||||||
|
|
||||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
# ── Per-run state ─────────────────────────────────────────────────────────────────────────
|
||||||
((CYCLE++))
|
|
||||||
CYCLE_START=$(date +%s)
|
|
||||||
|
|
||||||
# ── Re-source config each cycle ──────────────────────────────────────────────────────────
|
|
||||||
# Picks up config changes (new containers, threshold adjustments) without restart.
|
|
||||||
# detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source.
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
|
||||||
detect_hosts
|
|
||||||
|
|
||||||
# ── Per-cycle state — cleared each iteration ──────────────────────────────────────────────
|
|
||||||
NOTIFY_EVENTS=()
|
NOTIFY_EVENTS=()
|
||||||
T1_RESTARTS=0
|
T1_RESTARTS=0
|
||||||
T1_WARNINGS=0
|
T1_WARNINGS=0
|
||||||
@@ -488,33 +472,28 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|||||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||||
done
|
done
|
||||||
|
|
||||||
# ── Docker daemon health check — first check every cycle ────────────────────────────────
|
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||||
# If daemon is hung all container operations will fail — check first, skip cycle if down
|
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||||
if ! check_docker_daemon; then
|
if ! check_docker_daemon; then
|
||||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
exit 0
|
||||||
continue
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Parity check — skip restarts during parity ───────────────────────────────────────────
|
# ── Parity check — skip restarts during parity ───────────────────────────────────────────
|
||||||
if is_parity_running; then
|
if is_parity_running; then
|
||||||
log "Parity check in progress — skipping restart actions this cycle"
|
log "Parity check in progress — skipping restart actions this run"
|
||||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
exit 0
|
||||||
continue
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── RAM emergency check — system_watchdog.sh managing containers ───────────────────────────
|
# ── RAM emergency check — resource_manager.sh managing containers ────────────────────────
|
||||||
# If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container
|
# If resource_manager.sh has triggered a hard RAM shutdown, defer all container
|
||||||
# management this cycle. Docker daemon health checks continue — system still needs
|
# management this run to prevent undoing the emergency stop and re-pressuring RAM.
|
||||||
# monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown.
|
|
||||||
if ! check_system_watchdog_state; then
|
if ! check_system_watchdog_state; then
|
||||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||||
warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers"
|
warn "RAM emergency active (${MEM_GB}GB free) — resource_manager.sh managing containers"
|
||||||
warn "Deferring all container restart logic this cycle"
|
warn "Deferring all container restart logic this run"
|
||||||
log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming"
|
log "Waiting for RAM to recover above ${RM_RAM_RECOVER_GB:-20}GB before resuming"
|
||||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
exit 0
|
||||||
wait $!
|
|
||||||
continue
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Startup grace period ──────────────────────────────────────────────────────────────────
|
# ── Startup grace period ──────────────────────────────────────────────────────────────────
|
||||||
@@ -815,29 +794,11 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|||||||
|
|
||||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_WATCHDOG Docker Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||||
echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
else
|
else
|
||||||
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
|
log "All healthy ✅ ($(date '+%H:%M:%S'))"
|
||||||
|
fi
|
||||||
# Heartbeat — periodic proof of life even when everything is healthy
|
|
||||||
if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
|
||||||
HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
|
||||||
UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
|
|
||||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
|
||||||
(( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \
|
|
||||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
|
||||||
HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 ))
|
|
||||||
info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Sleep until next cycle — interruptible by SIGTERM
|
|
||||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
|
||||||
wait $!
|
|
||||||
|
|
||||||
done
|
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||||||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth
|
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth
|
||||||
# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures
|
# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures
|
||||||
# 🛡️ Watchdog — system watchdog, docker watchdog, fallback state
|
# 🛡️ Watchdog — resource manager, system watchdog, docker watchdog, fallback state
|
||||||
# 🔐 Security — SSL cert expiry per domain
|
# 🔐 Security — SSL cert expiry per domain
|
||||||
# 📊 Emby — weekly stream count, active now, top users
|
# 📊 Emby — weekly stream count, active now, top users
|
||||||
# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync
|
# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync
|
||||||
@@ -468,17 +468,21 @@ fi
|
|||||||
section "🛡️ WATCHDOG"
|
section "🛡️ WATCHDOG"
|
||||||
|
|
||||||
# ── System Watchdog ───────────────────────────────────────────────────────────────────────────
|
# ── System Watchdog ───────────────────────────────────────────────────────────────────────────
|
||||||
line "⚙️ System Watchdog"
|
line "⚙️ System Watchdog (cron via watchdog_orchestrator)"
|
||||||
SYS_PID=$(_get_lock_pid "system_watchdog")
|
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||||
if _is_running "system_watchdog"; then
|
_sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||||||
SYS_AGE=$(_lock_age "system_watchdog")
|
_sw_ago=$(( NOW - _sw_last ))
|
||||||
line " ✅ Running │ PID: $SYS_PID │ Uptime: $(_fmt_uptime "$SYS_AGE") │ ~Cycle: $(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))"
|
if [[ "$_sw_ago" -lt 600 ]]; then
|
||||||
|
line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago"
|
||||||
|
else
|
||||||
|
issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
issue "system_watchdog NOT RUNNING"
|
issue " system_watchdog has never run (state file missing)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||||
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=\|^mem_shutdown" \
|
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \
|
||||||
"$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
"$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||||
if [[ -n "$SYS_ACTIVE" ]]; then
|
if [[ -n "$SYS_ACTIVE" ]]; then
|
||||||
while IFS=: read -r key count; do
|
while IFS=: read -r key count; do
|
||||||
@@ -519,13 +523,17 @@ line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB
|
|||||||
REPORT+=("")
|
REPORT+=("")
|
||||||
|
|
||||||
# ── Docker Watchdog ───────────────────────────────────────────────────────────────────────────
|
# ── Docker Watchdog ───────────────────────────────────────────────────────────────────────────
|
||||||
line "🐳 Docker Watchdog"
|
line "🐳 Docker Watchdog (cron via watchdog_orchestrator)"
|
||||||
DOCKER_PID=$(_get_lock_pid "docker_watchdog")
|
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||||
if _is_running "docker_watchdog"; then
|
_dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||||||
DOCKER_AGE=$(_lock_age "docker_watchdog")
|
_dw_ago=$(( NOW - _dw_last ))
|
||||||
line " ✅ Running │ PID: $DOCKER_PID │ Uptime: $(_fmt_uptime "$DOCKER_AGE") │ ~Cycle: $(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))"
|
if [[ "$_dw_ago" -lt 600 ]]; then
|
||||||
|
line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago"
|
||||||
|
else
|
||||||
|
issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
issue "docker_watchdog NOT RUNNING"
|
issue " docker_watchdog has never run (state file missing)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||||
@@ -554,6 +562,26 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Resource Manager ─────────────────────────────────────────────────────────────────────────
|
||||||
|
line "🎛️ Resource Manager"
|
||||||
|
if [[ -f "$RM_STATE_FILE" ]]; then
|
||||||
|
_rm_last=$(stat -c %Y "$RM_STATE_FILE" 2>/dev/null || echo 0)
|
||||||
|
_rm_ago=$(( NOW - _rm_last ))
|
||||||
|
_rm_level=$(grep "^current_level:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2)
|
||||||
|
_rm_level="${_rm_level:-0}"
|
||||||
|
if [[ "$_rm_level" -gt 0 ]]; then
|
||||||
|
issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||||||
|
elif [[ "$_rm_ago" -lt 600 ]]; then
|
||||||
|
line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||||||
|
else
|
||||||
|
issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
line " ℹ️ State file not found (resource_manager may not have run yet)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
REPORT+=("")
|
||||||
|
|
||||||
if command -v docker >/dev/null 2>&1; then
|
if command -v docker >/dev/null 2>&1; then
|
||||||
RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||||||
TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||||||
|
|||||||
Executable
+192
@@ -0,0 +1,192 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ==============================================================================================
|
||||||
|
# ============================ Watchdog Orchestrator ===========================================
|
||||||
|
# ==============================================================================================
|
||||||
|
# Runs the three-layer watchdog system in the correct sequence each cycle.
|
||||||
|
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||||
|
#
|
||||||
|
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||||
|
# 1. resource_manager.sh — reduce system pressure intelligently
|
||||||
|
# 2. docker_watchdog.sh — heal containers with freed resources
|
||||||
|
# 3. system_watchdog.sh — reboot if all else fails (last line of defense)
|
||||||
|
#
|
||||||
|
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||||
|
# Resource Manager first — frees RAM and CPU before healing attempts container restarts.
|
||||||
|
# Containers restarted into a resource-pressured system just fail again.
|
||||||
|
# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
|
||||||
|
# System Watchdog last — only triggers if prior layers could not resolve the issue.
|
||||||
|
# Rebooting without first reducing pressure may reboot into the same state.
|
||||||
|
#
|
||||||
|
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
|
||||||
|
# Prevents false positives from containers still starting at array launch.
|
||||||
|
# Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
|
||||||
|
#
|
||||||
|
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
|
||||||
|
# acquire_lock() — exits immediately if a prior cycle is still in progress.
|
||||||
|
# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||||||
|
#
|
||||||
|
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
|
||||||
|
# Those scripts are now single-pass — this orchestrator provides the cadence.
|
||||||
|
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
|
||||||
|
#
|
||||||
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||||
|
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||||||
|
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||||||
|
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||||||
|
#
|
||||||
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# watchdog_orchestrator.sh — normal run (called by cron every minute)
|
||||||
|
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
|
||||||
|
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||||||
|
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Setup ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Skip immediately if another cycle is still running — no pile-up
|
||||||
|
acquire_lock
|
||||||
|
|
||||||
|
detect_hosts
|
||||||
|
|
||||||
|
RESOURCE_MANAGER="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_manager.sh"
|
||||||
|
DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
|
||||||
|
SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
|
||||||
|
|
||||||
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Status ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||||||
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||||
|
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||||
|
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||||||
|
else
|
||||||
|
log "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "── Sub-scripts ──"
|
||||||
|
for pair in \
|
||||||
|
"Resource Manager:$RESOURCE_MANAGER" \
|
||||||
|
"Docker Watchdog:$DOCKER_WATCHDOG" \
|
||||||
|
"System Watchdog:$SYSTEM_WATCHDOG"; do
|
||||||
|
label="${pair%%:*}"
|
||||||
|
script="${pair#*:}"
|
||||||
|
if [[ -f "$script" ]]; then
|
||||||
|
[[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||||
|
echo " $icon $label — ${script##*/}"
|
||||||
|
else
|
||||||
|
echo " $ICON_ERROR $label — NOT FOUND: $script"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo " Schedule: * * * * * (every minute via User Scripts)"
|
||||||
|
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Startup Grace ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||||
|
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||||
|
log "Startup grace — ${UPTIME_SECONDS}s / ${WATCHDOG_STARTUP_GRACE}s — skipping cycle"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Run Watchdog Cycle ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
CYCLE_START=$(date +%s)
|
||||||
|
PASS=()
|
||||||
|
FAIL=()
|
||||||
|
|
||||||
|
run_watchdog() {
|
||||||
|
local name="$1" script="$2"
|
||||||
|
|
||||||
|
if [[ ! -f "$script" ]]; then
|
||||||
|
error "$name — not found: $script"
|
||||||
|
FAIL+=("$name:missing")
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ ! -x "$script" ]] && chmod +x "$script"
|
||||||
|
|
||||||
|
local extra_args=()
|
||||||
|
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||||
|
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||||
|
|
||||||
|
log "$ICON_START $name"
|
||||||
|
if bash "$script" "${extra_args[@]}"; then
|
||||||
|
PASS+=("$name")
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
error "$name — non-zero exit"
|
||||||
|
FAIL+=("$name")
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
run_watchdog "Resource Manager" "$RESOURCE_MANAGER"
|
||||||
|
run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG"
|
||||||
|
run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG"
|
||||||
|
|
||||||
|
CYCLE_END=$(date +%s)
|
||||||
|
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Heartbeat ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||||||
|
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||||
|
HB_COUNT_FILE="/tmp/watchdog_orch_hb.count"
|
||||||
|
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||||||
|
HB_COUNT=$(( HB_COUNT + 1 ))
|
||||||
|
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||||||
|
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||||||
|
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||||||
|
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||||||
|
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||||||
|
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Summary — only shown on failures or --log ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||||
|
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||||
|
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||||||
|
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||||
|
"Watchdog Orchestrator" "warning"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
@@ -572,6 +572,8 @@ detect_hosts() {
|
|||||||
_alias_array "PARTNERSHIP_AUTH_WEBUIS"
|
_alias_array "PARTNERSHIP_AUTH_WEBUIS"
|
||||||
_alias_array "PARTNERSHIP_MIRROR_BACKUPS"
|
_alias_array "PARTNERSHIP_MIRROR_BACKUPS"
|
||||||
_alias_array "PARTNERSHIP_OWN_CONTAINERS"
|
_alias_array "PARTNERSHIP_OWN_CONTAINERS"
|
||||||
|
_alias_array "RM_PAUSE_CONTAINERS"
|
||||||
|
_alias_array "RM_STOP_CONTAINERS"
|
||||||
|
|
||||||
# ── Set array aliases — associative arrays ────────────────────────────────
|
# ── Set array aliases — associative arrays ────────────────────────────────
|
||||||
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
||||||
|
|||||||
+86
-36
@@ -278,7 +278,9 @@
|
|||||||
# Scripts launched by array_started.sh when the array comes online.
|
# Scripts launched by array_started.sh when the array comes online.
|
||||||
# Launched in order — each as a background process.
|
# Launched in order — each as a background process.
|
||||||
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||||||
# Continuous scripts (watchdogs, failover) run until array stops.
|
# Continuous scripts (failover) run until array stops.
|
||||||
|
# Watchdogs (resource_manager, docker_watchdog, system_watchdog) are cronned via
|
||||||
|
# watchdog_orchestrator.sh — NOT launched here.
|
||||||
ARRAY_START_SCRIPTS=(
|
ARRAY_START_SCRIPTS=(
|
||||||
"git_pull_execute.sh" # pull latest scripts before anything starts
|
"git_pull_execute.sh" # pull latest scripts before anything starts
|
||||||
"Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
"Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
||||||
@@ -286,9 +288,7 @@
|
|||||||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||||||
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||||||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||||||
"unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop
|
"Fallback/fallback.sh" # mutual failover — continuous
|
||||||
"Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop
|
|
||||||
"Fallback/fallback.sh" # mutual failover — HOST2 back online
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# ━━━ Intermediate Sync Maintenance ━━━
|
# ━━━ Intermediate Sync Maintenance ━━━
|
||||||
@@ -1114,12 +1114,85 @@
|
|||||||
EMBY_REPORT_DAYS=7 # days to include in the report period
|
EMBY_REPORT_DAYS=7 # days to include in the report period
|
||||||
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── WATCHDOG ORCHESTRATOR ─────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Runs resource_manager → docker_watchdog → system_watchdog in sequence each cron cycle.
|
||||||
|
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||||
|
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||||||
|
|
||||||
|
WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
|
||||||
|
WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Pressure reduction layer — keeps the system comfortable before things break.
|
||||||
|
# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop.
|
||||||
|
#
|
||||||
|
# ── PRESSURE LEVELS ───────────────────────────────────────────────────────────────────────────
|
||||||
|
# Level 1 (soft) — throttle SABnzbd + qBit download speeds
|
||||||
|
# Level 2 (medium) — further throttle + docker pause background containers
|
||||||
|
# Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer
|
||||||
|
#
|
||||||
|
# ── PER-HOST CONTAINER LISTS ──────────────────────────────────────────────────────────────────
|
||||||
|
# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (in master_host*.conf)
|
||||||
|
# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (in master_host*.conf)
|
||||||
|
|
||||||
|
RM_ENABLED=true
|
||||||
|
RM_STATE_FILE="/tmp/resource_manager_state.db"
|
||||||
|
|
||||||
|
# ━━━ Pressure Thresholds ━━━
|
||||||
|
# Graduated RAM response — resource_manager acts before system_watchdog reboots.
|
||||||
|
# RM_RAM_SOFT_GB > RM_RAM_MEDIUM_GB > RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always
|
||||||
|
RM_RAM_SOFT_GB=12 # throttle start — reduce background load
|
||||||
|
RM_RAM_MEDIUM_GB=8 # pause background containers
|
||||||
|
RM_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB)
|
||||||
|
RM_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers
|
||||||
|
|
||||||
|
# Load average thresholds — multiplier × core count
|
||||||
|
RM_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained
|
||||||
|
RM_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained
|
||||||
|
|
||||||
|
# Consecutive runs at lower pressure before de-escalating
|
||||||
|
RM_RECOVER_CYCLES=3
|
||||||
|
|
||||||
|
# ━━━ SABnzbd Throttle ━━━
|
||||||
|
# Speed values: "50M" = 50 MB/s, "0" = unlimited
|
||||||
|
RM_SABNZBD_ENABLED=true
|
||||||
|
RM_SABNZBD_SPEED_SOFT="50M"
|
||||||
|
RM_SABNZBD_SPEED_MEDIUM="10M"
|
||||||
|
|
||||||
|
# ━━━ qBittorrent Throttle ━━━
|
||||||
|
# KB/s — 0 = unlimited
|
||||||
|
RM_QBIT_ENABLED=true
|
||||||
|
RM_QBIT_DL_SOFT=51200 # 50 MB/s
|
||||||
|
RM_QBIT_DL_MEDIUM=10240 # 10 MB/s
|
||||||
|
|
||||||
|
# ━━━ Critical Containers ━━━
|
||||||
|
# Never paused or stopped regardless of pressure level.
|
||||||
|
# Keep DNS, auth, media serving, and live TV always running.
|
||||||
|
RM_CRITICAL_CONTAINERS=(
|
||||||
|
"NginxProxyManager" # reverse proxy — internet access
|
||||||
|
"Authelia" # auth — nothing accessible without it
|
||||||
|
"Authelia-Secondary"
|
||||||
|
"Mariadb-Authelia" # Authelia dependency
|
||||||
|
"Mariadb-Authelia-Secondary"
|
||||||
|
"Redis-Authelia" # Authelia dependency
|
||||||
|
"Redis-Authelia-Secondary"
|
||||||
|
"AdGuard-Home" # DNS — all LAN resolution
|
||||||
|
"Emby" # media server — Live TV buffering
|
||||||
|
"Dispatcharr" # Live TV scheduler — loses state if stopped
|
||||||
|
"Dispatcharr-Basic"
|
||||||
|
"Dispatcharr-Iptv-Users"
|
||||||
|
)
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# Continuous system health monitoring — last line of defense before a crash.
|
# Single-pass system health check — last line of defense before a crash.
|
||||||
# Started by array_started.sh — runs until array stops.
|
# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh.
|
||||||
# Re-sources all three conf files each cycle — config changes take effect on next cycle.
|
# Re-sources config at each orchestrator run — config changes take effect immediately.
|
||||||
#
|
#
|
||||||
# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
|
# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
|
||||||
# CRITICAL — bypass ALL strikes, reboot immediately
|
# CRITICAL — bypass ALL strikes, reboot immediately
|
||||||
@@ -1132,16 +1205,9 @@
|
|||||||
# STANDARD — strike system (N consecutive failures → reboot)
|
# STANDARD — strike system (N consecutive failures → reboot)
|
||||||
# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
|
# RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
|
||||||
#
|
#
|
||||||
# ── RAM TIERS ─────────────────────────────────────────────────────────────────────────────────
|
# ── RAM ───────────────────────────────────────────────────────────────────────────────────────
|
||||||
# MEM_WARN_GB — warn + notify only (informational)
|
# SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass)
|
||||||
# MEM_SHUTDOWN_GB — stop non-essential containers, recover above MEM_RECOVER_GB
|
# Warn/shutdown/recover RAM tiers are handled by resource_manager.sh
|
||||||
# MEM_GB — strike system → reboot (or bypass with OOM)
|
|
||||||
#
|
|
||||||
# ── CONTAINER SHUTDOWN ────────────────────────────────────────────────────────────────────────
|
|
||||||
# At MEM_SHUTDOWN_GB: stop all containers NOT in MEM_SHUTDOWN_EXCLUDED list
|
|
||||||
# Excluded containers stay running — DNS, auth, Emby, Dispatcharr
|
|
||||||
# Stopped containers stay stopped until RAM recovers above MEM_RECOVER_GB
|
|
||||||
# Strike list applied — doesn't flip-flop every cycle
|
|
||||||
|
|
||||||
# ━━━ State Files ━━━
|
# ━━━ State Files ━━━
|
||||||
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅
|
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp — resets on reboot ✅
|
||||||
@@ -1167,26 +1233,10 @@
|
|||||||
SYSTEM_WATCHDOG_HEARTBEAT=true
|
SYSTEM_WATCHDOG_HEARTBEAT=true
|
||||||
SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1
|
SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1
|
||||||
|
|
||||||
# ━━━ RAM Tiers ━━━
|
# ━━━ RAM Reboot Threshold ━━━
|
||||||
# Three-level RAM response — graduated action instead of single threshold.
|
# Reboot trigger only — warn/shutdown/recover handled by resource_manager.sh
|
||||||
# HOST1 has 128GB, HOST2 has 64GB — adjust accordingly.
|
# RM_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots)
|
||||||
# MEM_WARN_GB > MEM_SHUTDOWN_GB > MEM_GB always
|
|
||||||
SYS_WATCHDOG_MEM_WARN_GB=10 # warn + notify — informational only
|
|
||||||
SYS_WATCHDOG_MEM_SHUTDOWN_GB=6 # stop non-essential containers
|
|
||||||
SYS_WATCHDOG_MEM_GB=4 # strike system → reboot
|
SYS_WATCHDOG_MEM_GB=4 # strike system → reboot
|
||||||
SYS_WATCHDOG_MEM_RECOVER_GB=30 # RAM must recover above this before restarting containers
|
|
||||||
|
|
||||||
# Containers excluded from RAM emergency shutdown.
|
|
||||||
# These stay running regardless of RAM pressure.
|
|
||||||
# DNS and auth must stay up, Emby and Dispatcharr for Live TV continuity.
|
|
||||||
SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=(
|
|
||||||
"NginxProxyManager" # DNS / reverse proxy — internet access
|
|
||||||
"Authelia" # auth — without this nothing is accessible
|
|
||||||
"Mariadb" # Authelia dependency
|
|
||||||
"Redis" # Authelia dependency
|
|
||||||
"Emby" # media server — Live TV buffering
|
|
||||||
"Dispatcharr" # Live TV scheduler — loses state if stopped
|
|
||||||
)
|
|
||||||
|
|
||||||
# ━━━ OOM Bypass Settings ━━━
|
# ━━━ OOM Bypass Settings ━━━
|
||||||
# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
|
# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
|
||||||
|
|||||||
@@ -670,6 +670,32 @@
|
|||||||
# Enable only if HOST1 has no CPU-intensive workloads.
|
# Enable only if HOST1 has no CPU-intensive workloads.
|
||||||
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
HOST1_SYS_WATCHDOG_CHECK_RUNAWAY=false
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Containers to manage under pressure — see master.conf RM_CRITICAL_CONTAINERS for exclusions.
|
||||||
|
|
||||||
|
# docker pause at medium pressure (RAM < RM_RAM_MEDIUM_GB or load > medium threshold)
|
||||||
|
# Suspended in-place — instant to pause/unpause, no state lost, no restart delay.
|
||||||
|
HOST1_RM_PAUSE_CONTAINERS=(
|
||||||
|
"Huntarr" # arr search automation — safe to suspend
|
||||||
|
"Cleanuparr" # download cleanup — safe to suspend
|
||||||
|
"Healarr" # arr health checks — safe to suspend
|
||||||
|
"Soularr" # Slskd automation — background only
|
||||||
|
"ChannelTube" # YouTube archiver — background only
|
||||||
|
"Pinchflat" # YouTube archiver — background only
|
||||||
|
)
|
||||||
|
|
||||||
|
# docker stop at hard pressure (RAM < RM_RAM_HARD_GB)
|
||||||
|
# Full stop — these are optional/heavy services that free significant RAM when stopped.
|
||||||
|
# resource_manager.sh restarts them when pressure fully clears (RAM >= RM_RAM_RECOVER_GB).
|
||||||
|
HOST1_RM_STOP_CONTAINERS=(
|
||||||
|
"LocalAI" # GPU/CPU heavy — largest RAM consumer when idle
|
||||||
|
"7DaysToDie" # game server — optional
|
||||||
|
"V-Rising" # game server — optional
|
||||||
|
"Code-Server" # IDE — not needed during pressure events
|
||||||
|
)
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
# ──────────────────────── End Of HOST1 Variables ──────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
Executable
+523
@@ -0,0 +1,523 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ==============================================================================================
|
||||||
|
# ================================= Resource Manager ===========================================
|
||||||
|
# ==============================================================================================
|
||||||
|
# Pressure reduction layer — detects rising system load and reduces it intelligently.
|
||||||
|
# Called by watchdog_orchestrator.sh every minute — single-pass, not a continuous loop.
|
||||||
|
#
|
||||||
|
# ── RESPONSIBILITY ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Reduce system pressure before things break. NOT fixing broken containers (docker_watchdog)
|
||||||
|
# and NOT rebooting (system_watchdog). The middle layer that keeps the system comfortable.
|
||||||
|
#
|
||||||
|
# "Pressure is rising — reduce load intelligently."
|
||||||
|
#
|
||||||
|
# ── THREE-LEVEL PRESSURE RESPONSE ─────────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# Level 1 — SOFT (RAM < RM_RAM_SOFT_GB OR load > RM_LOAD_SOFT_MULTIPLIER × cores):
|
||||||
|
# Throttle SABnzbd download speed to RM_SABNZBD_SPEED_SOFT
|
||||||
|
# Throttle qBittorrent download to RM_QBIT_DL_SOFT KB/s
|
||||||
|
#
|
||||||
|
# Level 2 — MEDIUM (RAM < RM_RAM_MEDIUM_GB OR load > RM_LOAD_MEDIUM_MULTIPLIER × cores):
|
||||||
|
# Further throttle SABnzbd + qBittorrent to medium limits
|
||||||
|
# docker pause RM_PAUSE_CONTAINERS — suspend without losing state, instant reversible
|
||||||
|
#
|
||||||
|
# Level 3 — HARD (RAM < RM_RAM_HARD_GB):
|
||||||
|
# docker stop RM_STOP_CONTAINERS — optional/heavy services (games, LocalAI, etc.)
|
||||||
|
# Write mem_shutdown_active=true — signals docker_watchdog to defer container restarts
|
||||||
|
#
|
||||||
|
# ── RECOVERY ──────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Pressure must stay below current action threshold for RM_RECOVER_CYCLES consecutive runs
|
||||||
|
# before restoring. De-escalates one level at a time to avoid re-triggering immediately.
|
||||||
|
# Level 3 de-escalation additionally requires RAM >= RM_RAM_RECOVER_GB before un-stopping.
|
||||||
|
#
|
||||||
|
# ── COORDINATION WITH DOCKER WATCHDOG ─────────────────────────────────────────────────────────
|
||||||
|
# At level 3: writes mem_shutdown_active=true to RM_STATE_FILE.
|
||||||
|
# docker_watchdog.sh reads this and defers all container restart logic.
|
||||||
|
# Cleared when pressure fully resolves and containers are restarted.
|
||||||
|
# This prevents docker_watchdog from restarting containers that RM just stopped to free RAM.
|
||||||
|
#
|
||||||
|
# ── NOT RESPONSIBLE FOR ───────────────────────────────────────────────────────────────────────
|
||||||
|
# Restarting broken containers — docker_watchdog.sh
|
||||||
|
# Rebooting the system — system_watchdog.sh
|
||||||
|
# Reacting to single data points — RM_RECOVER_CYCLES prevents flip-flopping
|
||||||
|
#
|
||||||
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||||
|
# RM_ENABLED, RM_STATE_FILE
|
||||||
|
# RM_RAM_SOFT_GB, RM_RAM_MEDIUM_GB, RM_RAM_HARD_GB, RM_RAM_RECOVER_GB
|
||||||
|
# RM_LOAD_SOFT_MULTIPLIER, RM_LOAD_MEDIUM_MULTIPLIER
|
||||||
|
# RM_RECOVER_CYCLES
|
||||||
|
# RM_SABNZBD_ENABLED, RM_SABNZBD_SPEED_SOFT, RM_SABNZBD_SPEED_MEDIUM
|
||||||
|
# RM_QBIT_ENABLED, RM_QBIT_DL_SOFT, RM_QBIT_DL_MEDIUM
|
||||||
|
# RM_CRITICAL_CONTAINERS — never paused or stopped regardless of pressure
|
||||||
|
#
|
||||||
|
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||||
|
# HOST*_RM_PAUSE_CONTAINERS — docker pause at medium pressure (aliased by detect_hosts)
|
||||||
|
# HOST*_RM_STOP_CONTAINERS — docker stop at hard pressure (aliased by detect_hosts)
|
||||||
|
# HOST*_SABNZBD_URL, HOST*_SABNZBD_API_KEY
|
||||||
|
# HOST*_QBIT_URL, HOST*_QBIT_USERNAME, HOST*_QBIT_PASSWORD
|
||||||
|
#
|
||||||
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# resource_manager.sh — normal run (via watchdog_orchestrator.sh)
|
||||||
|
# resource_manager.sh --dry-run — show what would happen without acting
|
||||||
|
# resource_manager.sh --status — current pressure level and active actions
|
||||||
|
# resource_manager.sh --log — verbose per-check output
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Setup ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "${RM_ENABLED:-true}" != "true" ]]; then
|
||||||
|
log "Resource Manager disabled (RM_ENABLED=false)"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
acquire_lock
|
||||||
|
|
||||||
|
detect_hosts
|
||||||
|
|
||||||
|
DOCKER_TIMEOUT=15
|
||||||
|
|
||||||
|
touch "$RM_STATE_FILE" 2>/dev/null || {
|
||||||
|
error "Cannot create state file: $RM_STATE_FILE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ State Helpers ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
# rm_state_get/set use : separator for RM-internal state
|
||||||
|
# rm_state_get_eq/set_eq use = separator for docker_watchdog coordination flags
|
||||||
|
|
||||||
|
rm_state_get() {
|
||||||
|
grep -E "^${1}:" "$RM_STATE_FILE" 2>/dev/null | cut -d: -f2-
|
||||||
|
}
|
||||||
|
|
||||||
|
rm_state_set() {
|
||||||
|
local key="$1" val="$2"
|
||||||
|
grep -vE "^${key}:" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp"
|
||||||
|
echo "${key}:${val}" >> "${RM_STATE_FILE}.tmp"
|
||||||
|
mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
rm_state_get_eq() {
|
||||||
|
grep -E "^${1}=" "$RM_STATE_FILE" 2>/dev/null | cut -d= -f2-
|
||||||
|
}
|
||||||
|
|
||||||
|
rm_state_set_eq() {
|
||||||
|
local key="$1" val="$2"
|
||||||
|
grep -vE "^${key}=" "$RM_STATE_FILE" 2>/dev/null > "${RM_STATE_FILE}.tmp"
|
||||||
|
echo "${key}=${val}" >> "${RM_STATE_FILE}.tmp"
|
||||||
|
mv "${RM_STATE_FILE}.tmp" "$RM_STATE_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Load State ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
CURRENT_LEVEL=$(rm_state_get "rm_action_level"); CURRENT_LEVEL=${CURRENT_LEVEL:-0}
|
||||||
|
RECOVER_CYCLES=$(rm_state_get "rm_recover_cycles"); RECOVER_CYCLES=${RECOVER_CYCLES:-0}
|
||||||
|
PAUSED_LIST=$(rm_state_get "rm_paused_containers"); PAUSED_LIST=${PAUSED_LIST:-""}
|
||||||
|
STOPPED_LIST=$(rm_state_get "rm_stopped_containers"); STOPPED_LIST=${STOPPED_LIST:-""}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Pressure Calculation ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
TOTAL_CORES=$(nproc)
|
||||||
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||||
|
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||||
|
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||||
|
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||||
|
RM_LOAD_SOFT_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_SOFT_MULTIPLIER:-2.0}}")
|
||||||
|
RM_LOAD_MEDIUM_THRESH=$(awk "BEGIN{printf \"%.0f\", $TOTAL_CORES * ${RM_LOAD_MEDIUM_MULTIPLIER:-3.0}}")
|
||||||
|
|
||||||
|
TARGET_LEVEL=0
|
||||||
|
TARGET_REASON=""
|
||||||
|
if [[ "$MEM_GB" -lt "${RM_RAM_HARD_GB:-6}" ]]; then
|
||||||
|
TARGET_LEVEL=3
|
||||||
|
TARGET_REASON="RAM ${MEM_GB}GB < hard threshold ${RM_RAM_HARD_GB}GB"
|
||||||
|
elif [[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]]; then
|
||||||
|
TARGET_LEVEL=2
|
||||||
|
[[ "$MEM_GB" -lt "${RM_RAM_MEDIUM_GB:-8}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < medium threshold ${RM_RAM_MEDIUM_GB}GB"
|
||||||
|
[[ "$LOAD_INT" -ge "$RM_LOAD_MEDIUM_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= medium threshold ${RM_LOAD_MEDIUM_THRESH}"
|
||||||
|
elif [[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] || [[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]]; then
|
||||||
|
TARGET_LEVEL=1
|
||||||
|
[[ "$MEM_GB" -lt "${RM_RAM_SOFT_GB:-12}" ]] && TARGET_REASON="RAM ${MEM_GB}GB < soft threshold ${RM_RAM_SOFT_GB}GB"
|
||||||
|
[[ "$LOAD_INT" -ge "$RM_LOAD_SOFT_THRESH" ]] && TARGET_REASON="${TARGET_REASON:+$TARGET_REASON, }load ${LOAD} >= soft threshold ${RM_LOAD_SOFT_THRESH}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
LEVEL_NAMES=("normal" "soft" "medium" "hard")
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Status ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY RESOURCE MANAGER STATUS ━━━━━"
|
||||||
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
|
echo ""
|
||||||
|
echo "── Current State ──"
|
||||||
|
echo " Action level: $CURRENT_LEVEL (${LEVEL_NAMES[$CURRENT_LEVEL]:-unknown})"
|
||||||
|
echo " Target level: $TARGET_LEVEL (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||||
|
echo " Recover cycles: $RECOVER_CYCLES / ${RM_RECOVER_CYCLES:-3}"
|
||||||
|
[[ -n "$PAUSED_LIST" ]] && echo " Paused: $PAUSED_LIST"
|
||||||
|
[[ -n "$STOPPED_LIST" ]] && echo " Stopped: $STOPPED_LIST"
|
||||||
|
MEM_SHUTDOWN_ACTIVE=$(rm_state_get_eq "mem_shutdown_active")
|
||||||
|
[[ "$MEM_SHUTDOWN_ACTIVE" == "true" ]] && warn " docker_watchdog DEFERRED (mem_shutdown_active=true)"
|
||||||
|
echo ""
|
||||||
|
echo "── System Pressure ──"
|
||||||
|
echo " RAM free: ${MEM_GB}GB (soft:<${RM_RAM_SOFT_GB} medium:<${RM_RAM_MEDIUM_GB} hard:<${RM_RAM_HARD_GB} recover:>=${RM_RAM_RECOVER_GB})"
|
||||||
|
echo " Load avg: ${LOAD} (soft:>=${RM_LOAD_SOFT_THRESH} medium:>=${RM_LOAD_MEDIUM_THRESH} cores:${TOTAL_CORES})"
|
||||||
|
echo ""
|
||||||
|
echo "── Configuration ──"
|
||||||
|
echo " SABnzbd throttle: ${RM_SABNZBD_ENABLED:-true} soft=${RM_SABNZBD_SPEED_SOFT} medium=${RM_SABNZBD_SPEED_MEDIUM}"
|
||||||
|
echo " qBit throttle: ${RM_QBIT_ENABLED:-true} soft=${RM_QBIT_DL_SOFT}KB/s medium=${RM_QBIT_DL_MEDIUM}KB/s"
|
||||||
|
echo ""
|
||||||
|
echo "── Container Lists (this host) ──"
|
||||||
|
echo " Pause at medium: ${RM_PAUSE_CONTAINERS[*]:-none configured}"
|
||||||
|
echo " Stop at hard: ${RM_STOP_CONTAINERS[*]:-none configured}"
|
||||||
|
echo " Critical (never touched): ${RM_CRITICAL_CONTAINERS[*]:-none}"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Critical Container Guard ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
# Returns 0 if container is safe to pause/stop, 1 if it is critical
|
||||||
|
is_critical() {
|
||||||
|
local container="$1"
|
||||||
|
for c in "${RM_CRITICAL_CONTAINERS[@]:-}"; do
|
||||||
|
[[ "$c" == "$container" ]] && return 1
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ SABnzbd API ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
sabnzbd_set_speed() {
|
||||||
|
local speed="$1"
|
||||||
|
[[ "${RM_SABNZBD_ENABLED:-true}" != "true" ]] && return 0
|
||||||
|
[[ -z "$SABNZBD_URL" || -z "$SABNZBD_API_KEY" ]] && return 0
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would set SABnzbd speed to $speed"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
curl -sf --max-time 10 \
|
||||||
|
"${SABNZBD_URL}/api?mode=config&name=speedlimit&value=${speed}&apikey=${SABNZBD_API_KEY}" \
|
||||||
|
>/dev/null 2>&1 && log "SABnzbd speed → $speed" || warn "SABnzbd API call failed"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ qBittorrent API ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
QBIT_COOKIE="/tmp/rm_qbit_cookie.txt"
|
||||||
|
|
||||||
|
qbit_login() {
|
||||||
|
[[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||||
|
[[ -z "$QBIT_URL" || -z "$QBIT_USERNAME" || -z "$QBIT_PASSWORD" ]] && return 0
|
||||||
|
|
||||||
|
curl -sf --max-time 10 -c "$QBIT_COOKIE" \
|
||||||
|
-X POST "${QBIT_URL}/api/v2/auth/login" \
|
||||||
|
-d "username=${QBIT_USERNAME}&password=${QBIT_PASSWORD}" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
qbit_set_dl_limit() {
|
||||||
|
local kbps="$1" # KB/s — 0 = unlimited
|
||||||
|
[[ "${RM_QBIT_ENABLED:-true}" != "true" ]] && return 0
|
||||||
|
[[ -z "$QBIT_URL" ]] && return 0
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would set qBit download limit to ${kbps}KB/s"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local bps=$(( kbps * 1024 ))
|
||||||
|
qbit_login
|
||||||
|
curl -sf --max-time 10 -b "$QBIT_COOKIE" \
|
||||||
|
-X POST "${QBIT_URL}/api/v2/transfer/setDownloadLimit" \
|
||||||
|
-d "limit=${bps}" >/dev/null 2>&1 && log "qBit download limit → ${kbps}KB/s" || warn "qBit API call failed"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Container Actions ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
# Pause a list of containers — returns newline-separated list of actually-paused containers
|
||||||
|
pause_containers() {
|
||||||
|
local actually_paused=()
|
||||||
|
for container in "$@"; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
is_critical "$container" || { log "$container — critical, skipping pause"; continue; }
|
||||||
|
|
||||||
|
local status
|
||||||
|
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$status" != "running" ]]; then
|
||||||
|
log "$container — not running (status: ${status:-unknown}), skipping pause"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would docker pause $container"
|
||||||
|
actually_paused+=("$container")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker pause "$container" >/dev/null 2>&1; then
|
||||||
|
warn "Paused $container (medium pressure)"
|
||||||
|
actually_paused+=("$container")
|
||||||
|
else
|
||||||
|
error "Failed to pause $container"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf '%s,' "${actually_paused[@]}" | sed 's/,$//'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Unpause a comma-separated list of containers
|
||||||
|
unpause_containers() {
|
||||||
|
local IFS=','
|
||||||
|
for container in $1; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
|
||||||
|
local status
|
||||||
|
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$status" != "paused" ]]; then
|
||||||
|
log "$container — not paused (status: ${status:-unknown}), skipping unpause"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would docker unpause $container"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker unpause "$container" >/dev/null 2>&1; then
|
||||||
|
warn "Unpaused $container (pressure reduced)"
|
||||||
|
else
|
||||||
|
error "Failed to unpause $container"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Stop a list of containers — returns comma-separated list of actually-stopped containers
|
||||||
|
stop_containers() {
|
||||||
|
local actually_stopped=()
|
||||||
|
for container in "$@"; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
is_critical "$container" || { log "$container — critical, skipping stop"; continue; }
|
||||||
|
|
||||||
|
local status
|
||||||
|
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$status" != "running" && "$status" != "paused" ]]; then
|
||||||
|
log "$container — not running (status: ${status:-unknown}), skipping stop"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would docker stop $container"
|
||||||
|
actually_stopped+=("$container")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1; then
|
||||||
|
warn "Stopped $container (hard pressure)"
|
||||||
|
actually_stopped+=("$container")
|
||||||
|
else
|
||||||
|
error "Failed to stop $container"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf '%s,' "${actually_stopped[@]}" | sed 's/,$//'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Start a comma-separated list of containers (only those RM stopped)
|
||||||
|
start_containers() {
|
||||||
|
local IFS=','
|
||||||
|
for container in $1; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
|
||||||
|
local status
|
||||||
|
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Status}}' "$container" 2>/dev/null)
|
||||||
|
if [[ "$status" == "running" ]]; then
|
||||||
|
log "$container — already running"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would docker start $container"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||||||
|
warn "Started $container (pressure cleared)"
|
||||||
|
else
|
||||||
|
error "Failed to start $container"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Apply Level Actions ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
apply_level_1() {
|
||||||
|
log "Applying level 1 (soft) — throttling downloaders"
|
||||||
|
sabnzbd_set_speed "${RM_SABNZBD_SPEED_SOFT:-50M}"
|
||||||
|
qbit_set_dl_limit "${RM_QBIT_DL_SOFT:-51200}"
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_level_2() {
|
||||||
|
log "Applying level 2 (medium) — throttling + pausing background containers"
|
||||||
|
sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}"
|
||||||
|
qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}"
|
||||||
|
|
||||||
|
if [[ ${#RM_PAUSE_CONTAINERS[@]} -gt 0 ]]; then
|
||||||
|
local newly_paused
|
||||||
|
newly_paused=$(pause_containers "${RM_PAUSE_CONTAINERS[@]}")
|
||||||
|
# Merge with existing paused list (avoid duplicates on re-escalation)
|
||||||
|
if [[ -n "$newly_paused" ]]; then
|
||||||
|
if [[ -n "$PAUSED_LIST" ]]; then
|
||||||
|
PAUSED_LIST="${PAUSED_LIST},${newly_paused}"
|
||||||
|
else
|
||||||
|
PAUSED_LIST="$newly_paused"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_level_3() {
|
||||||
|
log "Applying level 3 (hard) — stopping optional containers"
|
||||||
|
sabnzbd_set_speed "${RM_SABNZBD_SPEED_MEDIUM:-10M}" # already at medium from level 2
|
||||||
|
qbit_set_dl_limit "${RM_QBIT_DL_MEDIUM:-10240}"
|
||||||
|
|
||||||
|
if [[ ${#RM_STOP_CONTAINERS[@]} -gt 0 ]]; then
|
||||||
|
local newly_stopped
|
||||||
|
newly_stopped=$(stop_containers "${RM_STOP_CONTAINERS[@]}")
|
||||||
|
if [[ -n "$newly_stopped" ]]; then
|
||||||
|
if [[ -n "$STOPPED_LIST" ]]; then
|
||||||
|
STOPPED_LIST="${STOPPED_LIST},${newly_stopped}"
|
||||||
|
else
|
||||||
|
STOPPED_LIST="$newly_stopped"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Signal docker_watchdog to defer container restarts
|
||||||
|
rm_state_set_eq "mem_shutdown_active" "true"
|
||||||
|
warn "mem_shutdown_active=true — docker_watchdog will defer restarts"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Restore Level Actions ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
restore_level_3() {
|
||||||
|
log "Restoring from level 3 — starting stopped containers"
|
||||||
|
if [[ -n "$STOPPED_LIST" ]]; then
|
||||||
|
start_containers "$STOPPED_LIST"
|
||||||
|
STOPPED_LIST=""
|
||||||
|
fi
|
||||||
|
rm_state_set_eq "mem_shutdown_active" "false"
|
||||||
|
warn "mem_shutdown_active=false — docker_watchdog restoring normal operation"
|
||||||
|
}
|
||||||
|
|
||||||
|
restore_level_2() {
|
||||||
|
log "Restoring from level 2 — unpausing containers"
|
||||||
|
if [[ -n "$PAUSED_LIST" ]]; then
|
||||||
|
unpause_containers "$PAUSED_LIST"
|
||||||
|
PAUSED_LIST=""
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
restore_level_1() {
|
||||||
|
log "Restoring from level 1 — removing downloader throttle"
|
||||||
|
sabnzbd_set_speed "0"
|
||||||
|
qbit_set_dl_limit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Pressure Decision ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Resource Manager — $MY_ID — $(date '+%H:%M:%S') ━━━"
|
||||||
|
echo " RAM: ${MEM_GB}GB free Load: ${LOAD} Action: ${CURRENT_LEVEL} → ${TARGET_LEVEL} (${LEVEL_NAMES[$TARGET_LEVEL]:-unknown})"
|
||||||
|
|
||||||
|
if [[ "$TARGET_LEVEL" -gt "$CURRENT_LEVEL" ]]; then
|
||||||
|
# ── Escalate ────────────────────────────────────────────────────────────────────────────
|
||||||
|
warn "Pressure escalating to level $TARGET_LEVEL — $TARGET_REASON"
|
||||||
|
notify "Resource Manager: pressure level $TARGET_LEVEL on $(hostname) ($MY_ID) — $TARGET_REASON" \
|
||||||
|
"Resource Manager" "warning"
|
||||||
|
|
||||||
|
for (( lvl = CURRENT_LEVEL + 1; lvl <= TARGET_LEVEL; lvl++ )); do
|
||||||
|
case "$lvl" in
|
||||||
|
1) apply_level_1 ;;
|
||||||
|
2) apply_level_2 ;;
|
||||||
|
3) apply_level_3 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
rm_state_set "rm_action_level" "$TARGET_LEVEL"
|
||||||
|
rm_state_set "rm_recover_cycles" 0
|
||||||
|
|
||||||
|
elif [[ "$TARGET_LEVEL" -lt "$CURRENT_LEVEL" ]]; then
|
||||||
|
# ── Tracking recovery ───────────────────────────────────────────────────────────────────
|
||||||
|
RECOVER_CYCLES=$(( RECOVER_CYCLES + 1 ))
|
||||||
|
rm_state_set "rm_recover_cycles" "$RECOVER_CYCLES"
|
||||||
|
log "Pressure at level $TARGET_LEVEL — recovery cycle $RECOVER_CYCLES/${RM_RECOVER_CYCLES:-3} before restoring level $CURRENT_LEVEL actions"
|
||||||
|
|
||||||
|
if [[ "$RECOVER_CYCLES" -ge "${RM_RECOVER_CYCLES:-3}" ]]; then
|
||||||
|
# Level 3 de-escalation requires RAM above recover threshold
|
||||||
|
if [[ "$CURRENT_LEVEL" -ge 3 && "$MEM_GB" -lt "${RM_RAM_RECOVER_GB:-20}" ]]; then
|
||||||
|
warn "Level 3 restore blocked — RAM ${MEM_GB}GB still below recover threshold ${RM_RAM_RECOVER_GB}GB"
|
||||||
|
else
|
||||||
|
warn "Pressure sustained below level $CURRENT_LEVEL — restoring"
|
||||||
|
case "$CURRENT_LEVEL" in
|
||||||
|
3) restore_level_3 ;;
|
||||||
|
2) restore_level_2 ;;
|
||||||
|
1) restore_level_1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
NEW_LEVEL=$(( CURRENT_LEVEL - 1 ))
|
||||||
|
rm_state_set "rm_action_level" "$NEW_LEVEL"
|
||||||
|
rm_state_set "rm_recover_cycles" 0
|
||||||
|
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||||
|
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||||
|
|
||||||
|
if [[ "$NEW_LEVEL" -gt 0 ]]; then
|
||||||
|
warn "De-escalated to level $NEW_LEVEL (${LEVEL_NAMES[$NEW_LEVEL]}) — ${RM_RECOVER_CYCLES:-3} more cycles to fully clear"
|
||||||
|
else
|
||||||
|
log "All pressure cleared — system at normal operation ✅"
|
||||||
|
notify "Resource Manager: pressure resolved on $(hostname) ($MY_ID) — system back to normal" \
|
||||||
|
"Resource Manager" "normal"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
# ── Steady state ────────────────────────────────────────────────────────────────────────
|
||||||
|
if [[ "$CURRENT_LEVEL" -gt 0 ]]; then
|
||||||
|
log "Pressure holding at level $CURRENT_LEVEL — waiting for sustained recovery"
|
||||||
|
else
|
||||||
|
log "System at normal pressure ✅"
|
||||||
|
fi
|
||||||
|
rm_state_set "rm_recover_cycles" 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Persist State ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
rm_state_set "rm_paused_containers" "$PAUSED_LIST"
|
||||||
|
rm_state_set "rm_stopped_containers" "$STOPPED_LIST"
|
||||||
|
# Touch state file each run so docker_watchdog stale guard sees fresh mtime
|
||||||
|
touch "$RM_STATE_FILE" 2>/dev/null
|
||||||
@@ -88,12 +88,13 @@ validate_unraid_cmd \
|
|||||||
"" "" \
|
"" "" \
|
||||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||||
|
|
||||||
acquire_lock "continuous"
|
acquire_lock
|
||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
TOTAL_CORES=$(nproc)
|
TOTAL_CORES=$(nproc)
|
||||||
DOCKER_TIMEOUT=10
|
DOCKER_TIMEOUT=10
|
||||||
|
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||||
|
|
||||||
# Ensure state files exist
|
# Ensure state files exist
|
||||||
for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \
|
for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \
|
||||||
@@ -129,10 +130,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
||||||
echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%"
|
echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%"
|
||||||
echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%"
|
echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%"
|
||||||
echo "$ICON_MEM RAM warn: < ${SYS_WATCHDOG_MEM_WARN_GB}GB"
|
|
||||||
echo "$ICON_MEM RAM shutdown: < ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB"
|
|
||||||
echo "$ICON_MEM RAM recover: > ${SYS_WATCHDOG_MEM_RECOVER_GB}GB"
|
|
||||||
echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)"
|
echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)"
|
||||||
|
echo " (RAM warn/shutdown/recover managed by resource_manager.sh)"
|
||||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%"
|
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%"
|
||||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)"
|
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)"
|
||||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||||
@@ -141,10 +140,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||||
echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr"
|
echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr"
|
||||||
echo ""
|
echo ""
|
||||||
echo "── Container Shutdown Excluded ──"
|
|
||||||
for c in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do
|
|
||||||
echo " $ICON_RUNNING $c"
|
|
||||||
done
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "── Check Toggles ──"
|
echo "── Check Toggles ──"
|
||||||
echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM"
|
echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM"
|
||||||
@@ -319,56 +314,6 @@ run_strike_check() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
|
||||||
# ── CONTAINER SHUTDOWN (RAM EMERGENCY) ────────────────────────────────────────────────────────
|
|
||||||
# ==============================================================================================
|
|
||||||
|
|
||||||
shutdown_non_essential_containers() {
|
|
||||||
warn "RAM emergency — stopping non-essential containers"
|
|
||||||
local stopped=()
|
|
||||||
|
|
||||||
# Build exclusion map
|
|
||||||
declare -A EXCLUDED_MAP
|
|
||||||
for exc in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do
|
|
||||||
[[ -n "$exc" ]] && EXCLUDED_MAP["$exc"]=1
|
|
||||||
done
|
|
||||||
|
|
||||||
# Stop all running containers not in exclusion list
|
|
||||||
while IFS= read -r container; do
|
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
if [[ -n "${EXCLUDED_MAP[$container]:-}" ]]; then
|
|
||||||
log "$container — excluded from RAM shutdown, leaving running"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
|
||||||
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
|
|
||||||
warn "Stopped $container (RAM emergency)" && \
|
|
||||||
stopped+=("$container") || \
|
|
||||||
error "Failed to stop $container"
|
|
||||||
else
|
|
||||||
warn "DRY RUN — would stop $container (RAM emergency)"
|
|
||||||
stopped+=("$container")
|
|
||||||
fi
|
|
||||||
done < <(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
|
|
||||||
|
|
||||||
if [[ ${#stopped[@]} -gt 0 ]]; then
|
|
||||||
set_state_val "mem_shutdown_active" "true"
|
|
||||||
notify "RAM emergency on $(hostname) ($MY_ID) — stopped ${#stopped[@]} containers. Excluded: ${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[*]}" \
|
|
||||||
"System Watchdog" "warning"
|
|
||||||
warn "Stopped ${#stopped[@]} containers — waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
restart_non_essential_containers() {
|
|
||||||
warn "RAM recovered — restarting containers that were stopped in emergency"
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
|
||||||
set_state_val "mem_shutdown_active" "false"
|
|
||||||
fi
|
|
||||||
# docker_watchdog.sh will detect stopped required containers and restart them
|
|
||||||
# We just clear the state flag here
|
|
||||||
warn "Cleared RAM emergency state — docker_watchdog.sh will restart required containers"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
|
# ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -457,36 +402,11 @@ do_reboot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Clean Shutdown ━━━
|
# ━━━ Single-Pass Health Check ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
WATCHDOG_RUNNING=true
|
warn "System watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
echo ""
|
|
||||||
warn "System watchdog received shutdown signal — stopping cleanly"
|
|
||||||
WATCHDOG_RUNNING=false
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
trap cleanup SIGTERM SIGINT
|
|
||||||
|
|
||||||
# ==============================================================================================
|
|
||||||
# ━━━ Continuous Monitoring Loop ━━━
|
|
||||||
# ==============================================================================================
|
|
||||||
warn "System watchdog started — $MY_ID — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
CYCLE=0
|
|
||||||
|
|
||||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|
||||||
(( CYCLE++ ))
|
|
||||||
|
|
||||||
# Re-source config each cycle — picks up config changes without restart
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
|
||||||
detect_hosts
|
|
||||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
|
||||||
TOTAL_CORES=$(nproc)
|
|
||||||
|
|
||||||
TRIGGERS=()
|
TRIGGERS=()
|
||||||
CRITICAL_TRIGGERS=()
|
CRITICAL_TRIGGERS=()
|
||||||
URGENT_OOM_CONFIRMED=false
|
URGENT_OOM_CONFIRMED=false
|
||||||
@@ -584,12 +504,12 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|||||||
# ── Act on CRITICAL triggers immediately ─────────────────────────────────────────────────
|
# ── Act on CRITICAL triggers immediately ─────────────────────────────────────────────────
|
||||||
if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then
|
if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT — Cycle $CYCLE ━━━"
|
echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT ━━━"
|
||||||
for t in "${CRITICAL_TRIGGERS[@]}"; do
|
for t in "${CRITICAL_TRIGGERS[@]}"; do
|
||||||
error " CRITICAL: $t"
|
error " CRITICAL: $t"
|
||||||
done
|
done
|
||||||
do_reboot "critical" "${CRITICAL_TRIGGERS[@]}"
|
do_reboot "critical" "${CRITICAL_TRIGGERS[@]}"
|
||||||
continue
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ==========================================================================================
|
# ==========================================================================================
|
||||||
@@ -638,60 +558,27 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|||||||
TRIGGERS+=("tmp=${TMP_USED}%")
|
TRIGGERS+=("tmp=${TMP_USED}%")
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── RAM tiers ─────────────────────────────────────────────────────────────────────────────
|
# ── RAM — reboot tier only (warn/shutdown/recover handled by resource_manager.sh) ──────────
|
||||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||||
MEM_SHUTDOWN_ACTIVE=$(get_state_val "mem_shutdown_active")
|
|
||||||
|
|
||||||
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
||||||
# Tier 2 check — bypass if OOM confirms crisis
|
# Tier 2 — bypass strike system if OOM confirms active crisis
|
||||||
if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \
|
if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \
|
||||||
[[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then
|
[[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then
|
||||||
error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this cycle — URGENT bypass"
|
error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this run — URGENT bypass"
|
||||||
OOM_VICTIMS=$(get_oom_victims)
|
OOM_VICTIMS=$(get_oom_victims)
|
||||||
URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}")
|
URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}")
|
||||||
[[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS")
|
[[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS")
|
||||||
do_reboot "urgent" "${URGENT_TRIGGERS[@]}"
|
do_reboot "urgent" "${URGENT_TRIGGERS[@]}"
|
||||||
continue
|
exit 0
|
||||||
fi
|
fi
|
||||||
# Standard strike path
|
# Standard strike path
|
||||||
run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \
|
run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \
|
||||||
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||||
|
|
||||||
elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_SHUTDOWN_GB" ]]; then
|
|
||||||
reset_strikes "ram"
|
|
||||||
# Container shutdown tier — but only once per event
|
|
||||||
if [[ "$MEM_SHUTDOWN_ACTIVE" != "true" ]]; then
|
|
||||||
warn "RAM ${MEM_GB}GB — below shutdown threshold ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB"
|
|
||||||
run_strike_check "ram_shutdown" true "RAM shutdown tier ${MEM_GB}GB" && \
|
|
||||||
shutdown_non_essential_containers
|
|
||||||
else
|
|
||||||
# Already shutdown — check if recovered
|
|
||||||
if [[ "$MEM_GB" -ge "$SYS_WATCHDOG_MEM_RECOVER_GB" ]]; then
|
|
||||||
warn "RAM recovered to ${MEM_GB}GB — clearing emergency state"
|
|
||||||
restart_non_essential_containers
|
|
||||||
reset_strikes "ram_shutdown"
|
|
||||||
else
|
|
||||||
warn "RAM ${MEM_GB}GB — still in emergency shutdown (recover threshold: ${SYS_WATCHDOG_MEM_RECOVER_GB}GB)"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_WARN_GB" ]]; then
|
|
||||||
reset_strikes "ram"
|
|
||||||
reset_strikes "ram_shutdown"
|
|
||||||
warn "RAM ${MEM_GB}GB — below warning threshold ${SYS_WATCHDOG_MEM_WARN_GB}GB"
|
|
||||||
local prev_ram_warn
|
|
||||||
prev_ram_warn=$(get_strikes "ram_warn_notified")
|
|
||||||
if [[ "${prev_ram_warn:-0}" -eq 0 ]]; then
|
|
||||||
notify "RAM warning on $(hostname) ($MY_ID) — ${MEM_GB}GB free (threshold: ${SYS_WATCHDOG_MEM_WARN_GB}GB)" \
|
|
||||||
"System Watchdog" "warning"
|
|
||||||
set_strikes "ram_warn_notified" 1
|
|
||||||
fi
|
|
||||||
else
|
else
|
||||||
reset_strikes "ram"
|
reset_strikes "ram"
|
||||||
reset_strikes "ram_shutdown"
|
|
||||||
set_strikes "ram_warn_notified" 0
|
|
||||||
log "RAM ${MEM_GB}GB free ✅"
|
log "RAM ${MEM_GB}GB free ✅"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -852,38 +739,17 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
|||||||
# ==========================================================================================
|
# ==========================================================================================
|
||||||
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_REBOOT_SMART System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
for t in "${TRIGGERS[@]}"; do
|
for t in "${TRIGGERS[@]}"; do
|
||||||
echo " $ICON_REBOOT_SMART $t"
|
echo " $ICON_REBOOT_SMART $t"
|
||||||
done
|
done
|
||||||
[[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this cycle: $OOM_DELTA"
|
[[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this run: $OOM_DELTA"
|
||||||
echo ""
|
echo ""
|
||||||
do_reboot "standard" "${TRIGGERS[@]}"
|
do_reboot "standard" "${TRIGGERS[@]}"
|
||||||
|
exit 0
|
||||||
else
|
else
|
||||||
log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
|
log "System healthy ✅ ($(date '+%H:%M:%S'))"
|
||||||
|
|
||||||
# Heartbeat — periodic proof of life
|
|
||||||
if [[ "${SYSTEM_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
|
||||||
HB_SECONDS=$(( ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
|
||||||
UPTIME_APPROX=$(( CYCLE * SYSTEM_WATCHDOG_INTERVAL ))
|
|
||||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
|
||||||
(( UPTIME_APPROX % HB_SECONDS < SYSTEM_WATCHDOG_INTERVAL )) && \
|
|
||||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
|
||||||
HB_HR=$(( UPTIME_APPROX / 3600 ))
|
|
||||||
warn "♥ system_watchdog alive — $MY_ID — ~${HB_HR}hr uptime ($(date '+%H:%M:%S'))"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── State file heartbeat — keep mtime fresh every cycle ───────────────────────────────────
|
# Keep state file mtime fresh — docker_watchdog stale guard checks this
|
||||||
# docker_watchdog.sh uses state file mtime to detect stale RAM emergency flags.
|
set_state_val "watchdog_cycle" "$(date +%s)"
|
||||||
# If all checks pass with no set_state_val calls (e.g. KERNEL_OOPS + MDSTAT both disabled),
|
|
||||||
# mtime would not update and stale guard would incorrectly resume docker_watchdog.sh.
|
|
||||||
# Writing watchdog_cycle each tick guarantees mtime stays current while watchdog runs.
|
|
||||||
set_state_val "watchdog_cycle" "$CYCLE"
|
|
||||||
|
|
||||||
# Sleep until next cycle — interruptible by SIGTERM
|
|
||||||
sleep "$SYSTEM_WATCHDOG_INTERVAL" &
|
|
||||||
wait $!
|
|
||||||
|
|
||||||
done
|
|
||||||
Reference in New Issue
Block a user