fix: correct state file ownership and function naming across watchdogs
stability_watchdog: remove SYS_WATCHDOG_FAILED_FILE from state file touch list — skip list is owned by docker_watchdog, not stability. docker_watchdog: rename check_system_watchdog_state → check_resource_watchdog_state (reads RW_STATE_FILE from resource_watchdog, not SYS_WATCHDOG_STATE_FILE from stability_watchdog). Update all comments and state file docs to reflect correct ownership. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
cce6fc5b65
commit
9d6772fbd6
@@ -53,8 +53,8 @@
|
|||||||
# Timeout protection — all docker commands wrapped in timeout
|
# Timeout protection — all docker commands wrapped in timeout
|
||||||
# Docker daemon check — each cycle begins with daemon health check; hung daemon →
|
# Docker daemon check — each cycle begins with daemon health check; hung daemon →
|
||||||
# restart via rc.docker → stability_watchdog.sh escalates if needed
|
# restart via rc.docker → stability_watchdog.sh escalates if needed
|
||||||
# RAM emergency defer — reads SYS_WATCHDOG_STATE_FILE; stands down while
|
# RAM emergency defer — reads RW_STATE_FILE; stands down while
|
||||||
# stability_watchdog.sh is managing a RAM emergency
|
# resource_watchdog.sh is managing a RAM emergency
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
@@ -116,10 +116,10 @@
|
|||||||
# critical, skip cycle. stability_watchdog.sh handles further escalation.
|
# critical, skip cycle. stability_watchdog.sh handles further escalation.
|
||||||
#
|
#
|
||||||
# RAM Emergency Deferral
|
# RAM Emergency Deferral
|
||||||
# Reads SYS_WATCHDOG_STATE_FILE each cycle. If stability_watchdog.sh has set
|
# Reads RW_STATE_FILE each cycle. If resource_watchdog.sh has set
|
||||||
# mem_shutdown_active=true, all restart logic defers until the flag clears.
|
# mem_shutdown_active=true, all restart logic defers until the flag clears.
|
||||||
# Stale state guard: if file is >2 hours old with flag still set,
|
# Stale state guard: if file is >2 hours old with flag still set,
|
||||||
# stability_watchdog.sh has likely stopped — watchdog resumes normal operation.
|
# resource_watchdog.sh has likely stopped — watchdog resumes normal operation.
|
||||||
#
|
#
|
||||||
# Timeout Protection
|
# Timeout Protection
|
||||||
# All docker commands wrapped in timeout. Daemon hangs cannot stall the
|
# All docker commands wrapped in timeout. Daemon hangs cannot stall the
|
||||||
@@ -133,13 +133,10 @@
|
|||||||
# STATE FILES
|
# STATE FILES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
# WATCHDOG_STATE_FILE — strike counts (default: /tmp — resets on reboot)
|
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
|
||||||
# SYS_WATCHDOG_FAILED_FILE — skip list (default: /boot/config — survives reboots)
|
# SYS_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
|
||||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
|
||||||
# SYS_WATCHDOG_STATE_FILE — shared state with system_watchdog.sh (RAM emergency flag)
|
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
|
||||||
#
|
|
||||||
# /tmp files reset on reboot — correct, pre-reboot strike counts are meaningless after it.
|
|
||||||
# /boot/config files survive reboots — correct, a skip-listed container is still broken after one.
|
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
@@ -489,7 +486,7 @@ _dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE")
|
|||||||
# 0 = normal — run all checks
|
# 0 = normal — run all checks
|
||||||
# 1 = RAM emergency active — defer container management this cycle
|
# 1 = RAM emergency active — defer container management this cycle
|
||||||
|
|
||||||
check_system_watchdog_state() {
|
check_resource_watchdog_state() {
|
||||||
# Returns 0 = normal operation | 1 = defer, resource_watchdog RAM emergency active
|
# Returns 0 = normal operation | 1 = defer, resource_watchdog RAM emergency active
|
||||||
local state_file="$RW_STATE_FILE"
|
local state_file="$RW_STATE_FILE"
|
||||||
|
|
||||||
@@ -635,7 +632,7 @@ CYCLE_START=$(date +%s)
|
|||||||
# ── RAM emergency check — resource_watchdog.sh managing containers ────────────────────────
|
# ── RAM emergency check — resource_watchdog.sh managing containers ────────────────────────
|
||||||
# If resource_watchdog.sh has triggered a hard RAM shutdown, defer all container
|
# If resource_watchdog.sh has triggered a hard RAM shutdown, defer all container
|
||||||
# management this run to prevent undoing the emergency stop and re-pressuring RAM.
|
# management this run to prevent undoing the emergency stop and re-pressuring RAM.
|
||||||
if ! check_system_watchdog_state; then
|
if ! check_resource_watchdog_state; then
|
||||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||||
warn "RAM emergency active (${MEM_GB}GB free) — resource_watchdog.sh managing containers"
|
warn "RAM emergency active (${MEM_GB}GB free) — resource_watchdog.sh managing containers"
|
||||||
|
|||||||
@@ -134,7 +134,7 @@ SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
|||||||
|
|
||||||
# Ensure state files exist
|
# Ensure state files exist
|
||||||
for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \
|
for state_file in "$SYS_WATCHDOG_STATE_FILE" "$SYS_WATCHDOG_REBOOT_LOG" \
|
||||||
"$SYS_WATCHDOG_FAILED_FILE" "$SYS_WATCHDOG_OOM_FILE"; do
|
"$SYS_WATCHDOG_OOM_FILE"; do
|
||||||
touch "$state_file" 2>/dev/null || {
|
touch "$state_file" 2>/dev/null || {
|
||||||
error "Cannot create state file: $state_file"
|
error "Cannot create state file: $state_file"
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
Reference in New Issue
Block a user