feat: watchdog architecture v2 — resource manager + single-pass orchestrator
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
f16c962ac0
commit
309546e615
@@ -88,12 +88,13 @@ validate_unraid_cmd \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "continuous"
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
TOTAL_CORES=$(nproc)
|
||||
DOCKER_TIMEOUT=10
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
|
||||
# Ensure state files exist
|
||||
for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \
|
||||
@@ -129,10 +130,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_DISK rootfs warn: ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
||||
echo "$ICON_GEAR /var/log warn: ${SYS_WATCHDOG_LOG_PCT}%"
|
||||
echo "$ICON_GEAR /tmp warn: ${SYS_WATCHDOG_TMP_PCT}%"
|
||||
echo "$ICON_MEM RAM warn: < ${SYS_WATCHDOG_MEM_WARN_GB}GB"
|
||||
echo "$ICON_MEM RAM shutdown: < ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB"
|
||||
echo "$ICON_MEM RAM recover: > ${SYS_WATCHDOG_MEM_RECOVER_GB}GB"
|
||||
echo "$ICON_MEM RAM reboot: < ${SYS_WATCHDOG_MEM_GB}GB (+ strikes)"
|
||||
echo " (RAM warn/shutdown/recover managed by resource_manager.sh)"
|
||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}%"
|
||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x (= $(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) on $TOTAL_CORES cores)"
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT}"
|
||||
@@ -141,10 +140,6 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_REBOOT_SMART Reboot limit: ${SYS_WATCHDOG_REBOOT_LIMIT} in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr"
|
||||
echo ""
|
||||
echo "── Container Shutdown Excluded ──"
|
||||
for c in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do
|
||||
echo " $ICON_RUNNING $c"
|
||||
done
|
||||
echo ""
|
||||
echo "── Check Toggles ──"
|
||||
echo " rootfs=$SYS_WATCHDOG_CHECK_ROOTFS log=$SYS_WATCHDOG_CHECK_LOG ram=$SYS_WATCHDOG_CHECK_RAM"
|
||||
@@ -319,56 +314,6 @@ run_strike_check() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CONTAINER SHUTDOWN (RAM EMERGENCY) ────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
shutdown_non_essential_containers() {
|
||||
warn "RAM emergency — stopping non-essential containers"
|
||||
local stopped=()
|
||||
|
||||
# Build exclusion map
|
||||
declare -A EXCLUDED_MAP
|
||||
for exc in "${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[@]:-}"; do
|
||||
[[ -n "$exc" ]] && EXCLUDED_MAP["$exc"]=1
|
||||
done
|
||||
|
||||
# Stop all running containers not in exclusion list
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ -n "${EXCLUDED_MAP[$container]:-}" ]]; then
|
||||
log "$container — excluded from RAM shutdown, leaving running"
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
|
||||
warn "Stopped $container (RAM emergency)" && \
|
||||
stopped+=("$container") || \
|
||||
error "Failed to stop $container"
|
||||
else
|
||||
warn "DRY RUN — would stop $container (RAM emergency)"
|
||||
stopped+=("$container")
|
||||
fi
|
||||
done < <(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
if [[ ${#stopped[@]} -gt 0 ]]; then
|
||||
set_state_val "mem_shutdown_active" "true"
|
||||
notify "RAM emergency on $(hostname) ($MY_ID) — stopped ${#stopped[@]} containers. Excluded: ${SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED[*]}" \
|
||||
"System Watchdog" "warning"
|
||||
warn "Stopped ${#stopped[@]} containers — waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB"
|
||||
fi
|
||||
}
|
||||
|
||||
restart_non_essential_containers() {
|
||||
warn "RAM recovered — restarting containers that were stopped in emergency"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
set_state_val "mem_shutdown_active" "false"
|
||||
fi
|
||||
# docker_watchdog.sh will detect stopped required containers and restart them
|
||||
# We just clear the state flag here
|
||||
warn "Cleared RAM emergency state — docker_watchdog.sh will restart required containers"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DO REBOOT ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -457,36 +402,11 @@ do_reboot() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Clean Shutdown ━━━
|
||||
# ━━━ Single-Pass Health Check ━━━
|
||||
# ==============================================================================================
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
warn "System watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Continuous Monitoring Loop ━━━
|
||||
# ==============================================================================================
|
||||
warn "System watchdog started — $MY_ID — checking every ${SYSTEM_WATCHDOG_INTERVAL}s"
|
||||
warn "System watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
(( CYCLE++ ))
|
||||
|
||||
# Re-source config each cycle — picks up config changes without restart
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
detect_hosts
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
TRIGGERS=()
|
||||
CRITICAL_TRIGGERS=()
|
||||
URGENT_OOM_CONFIRMED=false
|
||||
@@ -584,12 +504,12 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
# ── Act on CRITICAL triggers immediately ─────────────────────────────────────────────────
|
||||
if [[ ${#CRITICAL_TRIGGERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT — Cycle $CYCLE ━━━"
|
||||
echo "━━━ $ICON_ERROR CRITICAL — IMMEDIATE REBOOT ━━━"
|
||||
for t in "${CRITICAL_TRIGGERS[@]}"; do
|
||||
error " CRITICAL: $t"
|
||||
done
|
||||
do_reboot "critical" "${CRITICAL_TRIGGERS[@]}"
|
||||
continue
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==========================================================================================
|
||||
@@ -638,60 +558,27 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
TRIGGERS+=("tmp=${TMP_USED}%")
|
||||
fi
|
||||
|
||||
# ── RAM tiers ─────────────────────────────────────────────────────────────────────────────
|
||||
# ── RAM — reboot tier only (warn/shutdown/recover handled by resource_manager.sh) ──────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
MEM_SHUTDOWN_ACTIVE=$(get_state_val "mem_shutdown_active")
|
||||
|
||||
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
||||
# Tier 2 check — bypass if OOM confirms crisis
|
||||
# Tier 2 — bypass strike system if OOM confirms active crisis
|
||||
if [[ "$SYS_WATCHDOG_CHECK_OOM" == true ]] && \
|
||||
[[ "$OOM_DELTA" -ge "$SYS_WATCHDOG_OOM_LIMIT" ]]; then
|
||||
error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this cycle — URGENT bypass"
|
||||
error "RAM ${MEM_GB}GB + ${OOM_DELTA} OOM kills this run — URGENT bypass"
|
||||
OOM_VICTIMS=$(get_oom_victims)
|
||||
URGENT_TRIGGERS=("urgent_low_ram=${MEM_GB}GB" "oom_kills=${OOM_DELTA}")
|
||||
[[ -n "$OOM_VICTIMS" ]] && URGENT_TRIGGERS+=("oom_victims: $OOM_VICTIMS")
|
||||
do_reboot "urgent" "${URGENT_TRIGGERS[@]}"
|
||||
continue
|
||||
exit 0
|
||||
fi
|
||||
# Standard strike path
|
||||
run_strike_check "ram" true "RAM ${MEM_GB}GB free" && \
|
||||
TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||
|
||||
elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_SHUTDOWN_GB" ]]; then
|
||||
reset_strikes "ram"
|
||||
# Container shutdown tier — but only once per event
|
||||
if [[ "$MEM_SHUTDOWN_ACTIVE" != "true" ]]; then
|
||||
warn "RAM ${MEM_GB}GB — below shutdown threshold ${SYS_WATCHDOG_MEM_SHUTDOWN_GB}GB"
|
||||
run_strike_check "ram_shutdown" true "RAM shutdown tier ${MEM_GB}GB" && \
|
||||
shutdown_non_essential_containers
|
||||
else
|
||||
# Already shutdown — check if recovered
|
||||
if [[ "$MEM_GB" -ge "$SYS_WATCHDOG_MEM_RECOVER_GB" ]]; then
|
||||
warn "RAM recovered to ${MEM_GB}GB — clearing emergency state"
|
||||
restart_non_essential_containers
|
||||
reset_strikes "ram_shutdown"
|
||||
else
|
||||
warn "RAM ${MEM_GB}GB — still in emergency shutdown (recover threshold: ${SYS_WATCHDOG_MEM_RECOVER_GB}GB)"
|
||||
fi
|
||||
fi
|
||||
|
||||
elif [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_WARN_GB" ]]; then
|
||||
reset_strikes "ram"
|
||||
reset_strikes "ram_shutdown"
|
||||
warn "RAM ${MEM_GB}GB — below warning threshold ${SYS_WATCHDOG_MEM_WARN_GB}GB"
|
||||
local prev_ram_warn
|
||||
prev_ram_warn=$(get_strikes "ram_warn_notified")
|
||||
if [[ "${prev_ram_warn:-0}" -eq 0 ]]; then
|
||||
notify "RAM warning on $(hostname) ($MY_ID) — ${MEM_GB}GB free (threshold: ${SYS_WATCHDOG_MEM_WARN_GB}GB)" \
|
||||
"System Watchdog" "warning"
|
||||
set_strikes "ram_warn_notified" 1
|
||||
fi
|
||||
else
|
||||
reset_strikes "ram"
|
||||
reset_strikes "ram_shutdown"
|
||||
set_strikes "ram_warn_notified" 0
|
||||
log "RAM ${MEM_GB}GB free ✅"
|
||||
fi
|
||||
fi
|
||||
@@ -852,38 +739,17 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
# ==========================================================================================
|
||||
if [[ ${#TRIGGERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "━━━ $ICON_REBOOT_SMART System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
for t in "${TRIGGERS[@]}"; do
|
||||
echo " $ICON_REBOOT_SMART $t"
|
||||
done
|
||||
[[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this cycle: $OOM_DELTA"
|
||||
[[ "$OOM_DELTA" -gt 0 ]] && echo " OOM kills this run: $OOM_DELTA"
|
||||
echo ""
|
||||
do_reboot "standard" "${TRIGGERS[@]}"
|
||||
exit 0
|
||||
else
|
||||
log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))"
|
||||
|
||||
# Heartbeat — periodic proof of life
|
||||
if [[ "${SYSTEM_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
UPTIME_APPROX=$(( CYCLE * SYSTEM_WATCHDOG_INTERVAL ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
||||
(( UPTIME_APPROX % HB_SECONDS < SYSTEM_WATCHDOG_INTERVAL )) && \
|
||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
||||
HB_HR=$(( UPTIME_APPROX / 3600 ))
|
||||
warn "♥ system_watchdog alive — $MY_ID — ~${HB_HR}hr uptime ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
log "System healthy ✅ ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
|
||||
# ── State file heartbeat — keep mtime fresh every cycle ───────────────────────────────────
|
||||
# docker_watchdog.sh uses state file mtime to detect stale RAM emergency flags.
|
||||
# If all checks pass with no set_state_val calls (e.g. KERNEL_OOPS + MDSTAT both disabled),
|
||||
# mtime would not update and stale guard would incorrectly resume docker_watchdog.sh.
|
||||
# Writing watchdog_cycle each tick guarantees mtime stays current while watchdog runs.
|
||||
set_state_val "watchdog_cycle" "$CYCLE"
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$SYSTEM_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
# Keep state file mtime fresh — docker_watchdog stale guard checks this
|
||||
set_state_val "watchdog_cycle" "$(date +%s)"
|
||||
Reference in New Issue
Block a user