refactor: delegate Docker daemon escalation from stability to docker_watchdog

docker_watchdog now writes daemon_confirmed_down=true to its state file
when restart is attempted and daemon is still unresponsive. Clears the
flag on recovery.

stability_watchdog removes the duplicate CRITICAL daemon check (which
bypassed all strikes, all abort conditions, and had its own rc.docker
restart). Replaced with a standard strike check reading the
daemon_confirmed_down flag from docker_watchdog's state file.

Result: docker_watchdog owns all daemon restart logic. stability_watchdog
escalates to reboot only after N consecutive strike cycles confirming the
daemon is truly unrecoverable — same pattern as all other standard checks.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Watchdogs/docker_watchdog.sh

@@ -528,6 +528,7 @@ check_docker_daemon() {
             WATCHDOG_DAEMON_RESTARTED=false
             set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
             set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
+            set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
         fi
         return 0
     fi
@@ -542,9 +543,11 @@ check_docker_daemon() {
     fi
 
     if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then
+        # Restart was already attempted last cycle and daemon is still down.
+        # Write confirmed-down flag — stability_watchdog reads this and strikes toward reboot.
         error "Docker daemon still unresponsive after restart attempt"
-        error "stability_watchdog.sh will handle further escalation"
+        set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
-        queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical"
+        queue_notify "Docker daemon hung on $(hostname) — restart failed — stability_watchdog escalating" "critical"
         flush_notify
         return 1
     fi
@@ -571,16 +574,21 @@ check_docker_daemon() {
             WATCHDOG_DAEMON_RESTARTED=false
             set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
             set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
+            set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
             return 0
         else
+            # Restart issued but daemon still down — flag for stability_watchdog on next cycle
             error "Docker daemon did not recover after restart"
-            queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical"
+            set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
+            queue_notify "Docker daemon restart failed on $(hostname) — stability_watchdog escalating" "critical"
             flush_notify
             return 1
         fi
     else
+        # rc.docker not found or failed — flag immediately, stability_watchdog escalates
         error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed"
-        queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical"
+        set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
+        queue_notify "Docker daemon restart command failed on $(hostname) — stability_watchdog escalating" "critical"
         flush_notify
         return 1
     fi

Watchdogs/stability_watchdog.sh

@@ -18,11 +18,11 @@
 # Three-Tier Response System
 #
 #   Tier 1 — CRITICAL (bypass all strikes, reboot immediately)
-#     Docker daemon unresponsive    — nothing can be healed; running it longer makes it worse
 #     rootfs at 99%+               — writes failing; SSH may stop; no recovery options
 #     Kernel oops/BUG in dmesg     — kernel running with corrupted state
 #     File descriptor exhaustion   — new connections and processes failing silently
 #     /boot read-only unexpectedly — state files and config writes silently failing
+#     (Docker daemon: owned by docker_watchdog — writes daemon_confirmed_down flag → standard strikes)
 #
 #   Tier 2 — URGENT (bypass strikes when OOM confirms active crisis)
 #     RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle.
@@ -472,29 +472,18 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
     # ━━━ TIER 1 — CRITICAL CHECKS (bypass all strikes, reboot immediately) ━━━
     # ==========================================================================================
 
-    # ── Docker daemon — critical: nothing can heal without it ─────────────────────────────────
+    # ── Docker daemon — delegated to docker_watchdog ─────────────────────────────────────────
+    # docker_watchdog owns daemon restart attempts (strike system + rc.docker restart).
+    # When restart fails and daemon is confirmed down, it writes daemon_confirmed_down=true
+    # to WATCHDOG_STATE_FILE. We read that flag and run through the standard strike system.
     if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
-        if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
+        _daemon_down=$(grep -oP "(?<=^daemon_confirmed_down:)[^:]*" "$WATCHDOG_STATE_FILE" 2>/dev/null || echo "false")
-            error "Docker daemon unresponsive — CRITICAL"
+        TRIGGERED=false
-
+        [[ "$_daemon_down" == "true" ]] && TRIGGERED=true
-            # Attempt daemon restart before rebooting
+        run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon confirmed down by docker_watchdog" && \
-            warn "Attempting Docker daemon restart..."
+            TRIGGERS+=("docker_daemon_unresponsive")
-            if [[ "$DRY_RUN" == false ]]; then
+        [[ "$TRIGGERED" == true ]] && log "Docker daemon confirmed down — strike toward reboot" || \
-                /etc/rc.d/rc.docker restart >/dev/null 2>&1
+            log "Docker daemon flag clear ✅"
-                sleep 15
-                if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
-                    warn "Docker daemon restarted successfully — continuing monitoring"
-                else
-                    error "Docker daemon restart failed — adding to CRITICAL triggers"
-                    CRITICAL_TRIGGERS+=("docker_daemon_unresponsive")
-                fi
-            else
-                warn "DRY RUN — would attempt Docker daemon restart"
-                CRITICAL_TRIGGERS+=("docker_daemon_unresponsive")
-            fi
-        else
-            log "Docker daemon healthy ✅"
-        fi
     fi
 
     # ── rootfs critical — at 99%+ writes are failing ─────────────────────────────────────────