From f6fb3add772c7ecc7373036678bcc64f5aab63c3 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 31 May 2026 14:38:03 -0400 Subject: [PATCH] refactor: delegate Docker daemon escalation from stability to docker_watchdog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker_watchdog now writes daemon_confirmed_down=true to its state file when restart is attempted and daemon is still unresponsive. Clears the flag on recovery. stability_watchdog removes the duplicate CRITICAL daemon check (which bypassed all strikes, all abort conditions, and had its own rc.docker restart). Replaced with a standard strike check reading the daemon_confirmed_down flag from docker_watchdog's state file. Result: docker_watchdog owns all daemon restart logic. stability_watchdog escalates to reboot only after N consecutive strike cycles confirming the daemon is truly unrecoverable — same pattern as all other standard checks. Co-Authored-By: Claude Sonnet 4.6 --- Watchdogs/docker_watchdog.sh | 16 +++++++++++---- Watchdogs/stability_watchdog.sh | 35 +++++++++++---------------------- 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 0bf5199..37b17d5 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -528,6 +528,7 @@ check_docker_daemon() { WATCHDOG_DAEMON_RESTARTED=false set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE" set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE" + set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE" fi return 0 fi @@ -542,9 +543,11 @@ check_docker_daemon() { fi if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then + # Restart was already attempted last cycle and daemon is still down. + # Write confirmed-down flag — stability_watchdog reads this and strikes toward reboot. error "Docker daemon still unresponsive after restart attempt" - error "stability_watchdog.sh will handle further escalation" - queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical" + set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE" + queue_notify "Docker daemon hung on $(hostname) — restart failed — stability_watchdog escalating" "critical" flush_notify return 1 fi @@ -571,16 +574,21 @@ check_docker_daemon() { WATCHDOG_DAEMON_RESTARTED=false set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE" set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE" + set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE" return 0 else + # Restart issued but daemon still down — flag for stability_watchdog on next cycle error "Docker daemon did not recover after restart" - queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical" + set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE" + queue_notify "Docker daemon restart failed on $(hostname) — stability_watchdog escalating" "critical" flush_notify return 1 fi else + # rc.docker not found or failed — flag immediately, stability_watchdog escalates error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed" - queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical" + set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE" + queue_notify "Docker daemon restart command failed on $(hostname) — stability_watchdog escalating" "critical" flush_notify return 1 fi diff --git a/Watchdogs/stability_watchdog.sh b/Watchdogs/stability_watchdog.sh index 3e110b4..d0c3d50 100755 --- a/Watchdogs/stability_watchdog.sh +++ b/Watchdogs/stability_watchdog.sh @@ -18,11 +18,11 @@ # Three-Tier Response System # # Tier 1 — CRITICAL (bypass all strikes, reboot immediately) -# Docker daemon unresponsive — nothing can be healed; running it longer makes it worse # rootfs at 99%+ — writes failing; SSH may stop; no recovery options # Kernel oops/BUG in dmesg — kernel running with corrupted state # File descriptor exhaustion — new connections and processes failing silently # /boot read-only unexpectedly — state files and config writes silently failing +# (Docker daemon: owned by docker_watchdog — writes daemon_confirmed_down flag → standard strikes) # # Tier 2 — URGENT (bypass strikes when OOM confirms active crisis) # RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle. @@ -472,29 +472,18 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S') # ━━━ TIER 1 — CRITICAL CHECKS (bypass all strikes, reboot immediately) ━━━ # ========================================================================================== - # ── Docker daemon — critical: nothing can heal without it ───────────────────────────────── + # ── Docker daemon — delegated to docker_watchdog ───────────────────────────────────────── + # docker_watchdog owns daemon restart attempts (strike system + rc.docker restart). + # When restart fails and daemon is confirmed down, it writes daemon_confirmed_down=true + # to WATCHDOG_STATE_FILE. We read that flag and run through the standard strike system. if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then - if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then - error "Docker daemon unresponsive — CRITICAL" - - # Attempt daemon restart before rebooting - warn "Attempting Docker daemon restart..." - if [[ "$DRY_RUN" == false ]]; then - /etc/rc.d/rc.docker restart >/dev/null 2>&1 - sleep 15 - if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then - warn "Docker daemon restarted successfully — continuing monitoring" - else - error "Docker daemon restart failed — adding to CRITICAL triggers" - CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") - fi - else - warn "DRY RUN — would attempt Docker daemon restart" - CRITICAL_TRIGGERS+=("docker_daemon_unresponsive") - fi - else - log "Docker daemon healthy ✅" - fi + _daemon_down=$(grep -oP "(?<=^daemon_confirmed_down:)[^:]*" "$WATCHDOG_STATE_FILE" 2>/dev/null || echo "false") + TRIGGERED=false + [[ "$_daemon_down" == "true" ]] && TRIGGERED=true + run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon confirmed down by docker_watchdog" && \ + TRIGGERS+=("docker_daemon_unresponsive") + [[ "$TRIGGERED" == true ]] && log "Docker daemon confirmed down — strike toward reboot" || \ + log "Docker daemon flag clear ✅" fi # ── rootfs critical — at 99%+ writes are failing ─────────────────────────────────────────