refactor: delegate Docker daemon escalation from stability to docker_watchdog
docker_watchdog now writes daemon_confirmed_down=true to its state file when restart is attempted and daemon is still unresponsive. Clears the flag on recovery. stability_watchdog removes the duplicate CRITICAL daemon check (which bypassed all strikes, all abort conditions, and had its own rc.docker restart). Replaced with a standard strike check reading the daemon_confirmed_down flag from docker_watchdog's state file. Result: docker_watchdog owns all daemon restart logic. stability_watchdog escalates to reboot only after N consecutive strike cycles confirming the daemon is truly unrecoverable — same pattern as all other standard checks. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
9c68429e25
commit
f6fb3add77
@@ -528,6 +528,7 @@ check_docker_daemon() {
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
@@ -542,9 +543,11 @@ check_docker_daemon() {
|
||||
fi
|
||||
|
||||
if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then
|
||||
# Restart was already attempted last cycle and daemon is still down.
|
||||
# Write confirmed-down flag — stability_watchdog reads this and strikes toward reboot.
|
||||
error "Docker daemon still unresponsive after restart attempt"
|
||||
error "stability_watchdog.sh will handle further escalation"
|
||||
queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical"
|
||||
set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
|
||||
queue_notify "Docker daemon hung on $(hostname) — restart failed — stability_watchdog escalating" "critical"
|
||||
flush_notify
|
||||
return 1
|
||||
fi
|
||||
@@ -571,16 +574,21 @@ check_docker_daemon() {
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
|
||||
set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
|
||||
return 0
|
||||
else
|
||||
# Restart issued but daemon still down — flag for stability_watchdog on next cycle
|
||||
error "Docker daemon did not recover after restart"
|
||||
queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical"
|
||||
set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
|
||||
queue_notify "Docker daemon restart failed on $(hostname) — stability_watchdog escalating" "critical"
|
||||
flush_notify
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
# rc.docker not found or failed — flag immediately, stability_watchdog escalates
|
||||
error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed"
|
||||
queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical"
|
||||
set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
|
||||
queue_notify "Docker daemon restart command failed on $(hostname) — stability_watchdog escalating" "critical"
|
||||
flush_notify
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -18,11 +18,11 @@
|
||||
# Three-Tier Response System
|
||||
#
|
||||
# Tier 1 — CRITICAL (bypass all strikes, reboot immediately)
|
||||
# Docker daemon unresponsive — nothing can be healed; running it longer makes it worse
|
||||
# rootfs at 99%+ — writes failing; SSH may stop; no recovery options
|
||||
# Kernel oops/BUG in dmesg — kernel running with corrupted state
|
||||
# File descriptor exhaustion — new connections and processes failing silently
|
||||
# /boot read-only unexpectedly — state files and config writes silently failing
|
||||
# (Docker daemon: owned by docker_watchdog — writes daemon_confirmed_down flag → standard strikes)
|
||||
#
|
||||
# Tier 2 — URGENT (bypass strikes when OOM confirms active crisis)
|
||||
# RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle.
|
||||
@@ -472,29 +472,18 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
# ━━━ TIER 1 — CRITICAL CHECKS (bypass all strikes, reboot immediately) ━━━
|
||||
# ==========================================================================================
|
||||
|
||||
# ── Docker daemon — critical: nothing can heal without it ─────────────────────────────────
|
||||
# ── Docker daemon — delegated to docker_watchdog ─────────────────────────────────────────
|
||||
# docker_watchdog owns daemon restart attempts (strike system + rc.docker restart).
|
||||
# When restart fails and daemon is confirmed down, it writes daemon_confirmed_down=true
|
||||
# to WATCHDOG_STATE_FILE. We read that flag and run through the standard strike system.
|
||||
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon unresponsive — CRITICAL"
|
||||
|
||||
# Attempt daemon restart before rebooting
|
||||
warn "Attempting Docker daemon restart..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
/etc/rc.d/rc.docker restart >/dev/null 2>&1
|
||||
sleep 15
|
||||
if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
warn "Docker daemon restarted successfully — continuing monitoring"
|
||||
else
|
||||
error "Docker daemon restart failed — adding to CRITICAL triggers"
|
||||
CRITICAL_TRIGGERS+=("docker_daemon_unresponsive")
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would attempt Docker daemon restart"
|
||||
CRITICAL_TRIGGERS+=("docker_daemon_unresponsive")
|
||||
fi
|
||||
else
|
||||
log "Docker daemon healthy ✅"
|
||||
fi
|
||||
_daemon_down=$(grep -oP "(?<=^daemon_confirmed_down:)[^:]*" "$WATCHDOG_STATE_FILE" 2>/dev/null || echo "false")
|
||||
TRIGGERED=false
|
||||
[[ "$_daemon_down" == "true" ]] && TRIGGERED=true
|
||||
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon confirmed down by docker_watchdog" && \
|
||||
TRIGGERS+=("docker_daemon_unresponsive")
|
||||
[[ "$TRIGGERED" == true ]] && log "Docker daemon confirmed down — strike toward reboot" || \
|
||||
log "Docker daemon flag clear ✅"
|
||||
fi
|
||||
|
||||
# ── rootfs critical — at 99%+ writes are failing ─────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user