feat: watchdog architecture v2 — resource manager + single-pass orchestrator
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
f16c962ac0
commit
309546e615
@@ -97,8 +97,8 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
fi
|
||||
success "Running as root"
|
||||
|
||||
# Continuous mode — skip gracefully if healthy instance already running
|
||||
acquire_lock "continuous"
|
||||
# Skip if another instance is running — no pile-up during long operations
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||||
detect_hosts
|
||||
@@ -332,8 +332,11 @@ is_parity_running() {
|
||||
|
||||
WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt
|
||||
WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying
|
||||
WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers
|
||||
WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session
|
||||
# Loaded from state file — persists across single-pass runs
|
||||
WATCHDOG_DAEMON_STRIKES=$(get_strikes "daemon_strikes" "$WATCHDOG_STATE_FILE")
|
||||
WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES//[^0-9]/}"; WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES:-0}"
|
||||
_dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE")
|
||||
[[ "$_dr_raw" == "true" ]] && WATCHDOG_DAEMON_RESTARTED=true || WATCHDOG_DAEMON_RESTARTED=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
|
||||
@@ -348,10 +351,10 @@ WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart thi
|
||||
# 1 = RAM emergency active — defer container management this cycle
|
||||
|
||||
check_system_watchdog_state() {
|
||||
# Returns 0 = normal operation | 1 = defer, RAM emergency active
|
||||
local state_file="$SYS_WATCHDOG_STATE_FILE"
|
||||
# Returns 0 = normal operation | 1 = defer, resource_manager RAM emergency active
|
||||
local state_file="$RM_STATE_FILE"
|
||||
|
||||
# No state file = system_watchdog not running or not yet written — assume normal
|
||||
# No state file = resource_manager not yet run — assume normal
|
||||
[[ ! -f "$state_file" ]] && return 0
|
||||
|
||||
local mem_shutdown
|
||||
@@ -361,7 +364,7 @@ check_system_watchdog_state() {
|
||||
|
||||
# ── Stale state guard ─────────────────────────────────────────────────────────────────────
|
||||
# If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
|
||||
# system_watchdog.sh may have died — don't be silenced forever by a stale flag.
|
||||
# resource_manager.sh may not be running — don't defer indefinitely on stale state.
|
||||
local state_mtime now age_seconds stale_limit=7200 # 2 hours
|
||||
state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
@@ -369,8 +372,7 @@ check_system_watchdog_state() {
|
||||
|
||||
if [[ "$age_seconds" -gt "$stale_limit" ]]; then
|
||||
warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
|
||||
warn "system_watchdog.sh may not be running — resuming normal container management"
|
||||
warn "If RAM is still low this will be caught on next system_watchdog.sh cycle"
|
||||
warn "resource_manager.sh may not be running — resuming normal container management"
|
||||
return 0 # Resume normal — don't defer indefinitely on stale state
|
||||
fi
|
||||
|
||||
@@ -385,11 +387,14 @@ check_docker_daemon() {
|
||||
queue_notify "Docker daemon recovered on $(hostname)" "normal"
|
||||
WATCHDOG_DAEMON_STRIKES=0
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
|
||||
set_strikes "daemon_strikes" "$WATCHDOG_DAEMON_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"
|
||||
|
||||
if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
|
||||
@@ -415,6 +420,7 @@ check_docker_daemon() {
|
||||
|
||||
# Restart daemon — unRAID uses rc.d scripts, not systemd
|
||||
WATCHDOG_DAEMON_RESTARTED=true
|
||||
set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE"
|
||||
if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then
|
||||
info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
|
||||
sleep "$WATCHDOG_DAEMON_RESTART_WAIT"
|
||||
@@ -424,6 +430,8 @@ check_docker_daemon() {
|
||||
notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
|
||||
WATCHDOG_DAEMON_STRIKES=0
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||
return 0
|
||||
else
|
||||
error "Docker daemon did not recover after restart"
|
||||
@@ -440,38 +448,14 @@ check_docker_daemon() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CLEAN SHUTDOWN ────────────────────────────────────────────────────────────────────────────
|
||||
# ━━━ Single-Pass Monitoring Run ━━━
|
||||
# ==============================================================================================
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
warn "Docker watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Continuous Monitoring Loop ━━━
|
||||
# ==============================================================================================
|
||||
info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
info "$ICON_WATCHDOG Docker watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
((CYCLE++))
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
# ── Re-source config each cycle ──────────────────────────────────────────────────────────
|
||||
# Picks up config changes (new containers, threshold adjustments) without restart.
|
||||
# detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source.
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
detect_hosts
|
||||
|
||||
# ── Per-cycle state — cleared each iteration ──────────────────────────────────────────────
|
||||
# ── Per-run state ─────────────────────────────────────────────────────────────────────────
|
||||
NOTIFY_EVENTS=()
|
||||
T1_RESTARTS=0
|
||||
T1_WARNINGS=0
|
||||
@@ -488,33 +472,28 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# ── Docker daemon health check — first check every cycle ────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip cycle if down
|
||||
# ── Docker daemon health check — first check every run ──────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip run if down
|
||||
if ! check_docker_daemon; then
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Parity check — skip restarts during parity ───────────────────────────────────────────
|
||||
if is_parity_running; then
|
||||
log "Parity check in progress — skipping restart actions this cycle"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
log "Parity check in progress — skipping restart actions this run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── RAM emergency check — system_watchdog.sh managing containers ───────────────────────────
|
||||
# If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container
|
||||
# management this cycle. Docker daemon health checks continue — system still needs
|
||||
# monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown.
|
||||
# ── RAM emergency check — resource_manager.sh managing containers ────────────────────────
|
||||
# If resource_manager.sh has triggered a hard RAM shutdown, defer all container
|
||||
# management this run to prevent undoing the emergency stop and re-pressuring RAM.
|
||||
if ! check_system_watchdog_state; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers"
|
||||
warn "Deferring all container restart logic this cycle"
|
||||
log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
continue
|
||||
warn "RAM emergency active (${MEM_GB}GB free) — resource_manager.sh managing containers"
|
||||
warn "Deferring all container restart logic this run"
|
||||
log "Waiting for RAM to recover above ${RM_RAM_RECOVER_GB:-20}GB before resuming"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Startup grace period ──────────────────────────────────────────────────────────────────
|
||||
@@ -815,29 +794,11 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
|
||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "━━━ $ICON_WATCHDOG Docker Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||
echo "$ICON_WATCHDOG T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
|
||||
|
||||
# Heartbeat — periodic proof of life even when everything is healthy
|
||||
if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
||||
(( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \
|
||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
||||
HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 ))
|
||||
info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Sleep until next cycle — interruptible by SIGTERM
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
log "All healthy ✅ ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
Reference in New Issue
Block a user