FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: delegate Docker daemon escalation from stability to docker_watchdog

Browse Source 
docker_watchdog now writes daemon_confirmed_down=true to its state file
when restart is attempted and daemon is still unresponsive. Clears the
flag on recovery.

stability_watchdog removes the duplicate CRITICAL daemon check (which
bypassed all strikes, all abort conditions, and had its own rc.docker
restart). Replaced with a standard strike check reading the
daemon_confirmed_down flag from docker_watchdog's state file.

Result: docker_watchdog owns all daemon restart logic. stability_watchdog
escalates to reboot only after N consecutive strike cycles confirming the
daemon is truly unrecoverable — same pattern as all other standard checks.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
2026-05-31 14:38:03 -04:00
co-authored by Claude Sonnet 4.6
parent 9c68429e25
commit f6fb3add77
2 changed files with 24 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Watchdogs/docker_watchdog.sh
+12 -4
View File
@@ -528,6 +528,7 @@ check_docker_daemon() {
WATCHDOG_DAEMON_RESTARTED=false
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
fi
return 0
fi
@@ -542,9 +543,11 @@ check_docker_daemon() {
fi
if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then
# Restart was already attempted last cycle and daemon is still down.
# Write confirmed-down flag — stability_watchdog reads this and strikes toward reboot.
error "Docker daemon still unresponsive after restart attempt"
error "stability_watchdog.sh will handle further escalation"
queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical"
set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
queue_notify "Docker daemon hung on $(hostname) — restart failed — stability_watchdog escalating" "critical"
flush_notify
return 1
fi
@@ -571,16 +574,21 @@ check_docker_daemon() {
WATCHDOG_DAEMON_RESTARTED=false
set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE"
set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE"
set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE"
return 0
else
# Restart issued but daemon still down — flag for stability_watchdog on next cycle
error "Docker daemon did not recover after restart"
queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical"
set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
queue_notify "Docker daemon restart failed on $(hostname) — stability_watchdog escalating" "critical"
flush_notify
return 1
fi
else
# rc.docker not found or failed — flag immediately, stability_watchdog escalates
error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed"
queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical"
set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE"
queue_notify "Docker daemon restart command failed on $(hostname) — stability_watchdog escalating" "critical"
flush_notify
return 1
fi