From 0b4ff27be4a768e3a0bc426ddd6e7cfe7c56f14d Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 2 Aug 2026 00:45:29 -0400 Subject: [PATCH] Prune stale container strike counts, not just skip-list entries The 2026-07-19 prune covered the skip list and intentional stops but not the per-container strike counts, which have the same failure: set_strikes 0 only fires when a container is seen running again, so an uninstalled one keeps its count forever and every consumer reads the host as unhealthy. Reserved daemon_* keys are excluded. --- Watchdogs/docker_watchdog.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 3266a10..6d67184 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -137,6 +137,17 @@ # All docker commands wrapped in timeout. Daemon hangs cannot stall the # watchdog and leave containers unmonitored between cycles. # +# Stale Entry Pruning +# Every cycle, entries naming a container that no longer exists are removed from +# DOCKER_WATCHDOG_FAILED_FILE, DOCKER_WATCHDOG_INTENTIONAL_FILE, and the per-container +# strike counts in WATCHDOG_STATE_FILE. None of these can clear themselves: every +# clearing path requires seeing the container running again, which never happens once +# it is uninstalled. Without pruning, one removed container makes this host report +# unhealthy permanently. Reserved daemon_* keys in WATCHDOG_STATE_FILE are never pruned. +# Pruning runs under --dry-run as well: the documented contract for that flag is that +# nothing gets restarted, and dropping a record of a container that no longer exists is +# housekeeping, not an action. It is idempotent — repeated runs converge. +# # Notification Batching # Events collected across a full cycle and sent as a single summary. # Prevents notification floods when a shared dependency failure cascades. @@ -710,6 +721,28 @@ CYCLE_START=$(date +%s) done unset _prune_file _prune_snapshot _prune_container + # Same failure, different file. WATCHDOG_STATE_FILE cannot join the loop above: it stores + # key:value rather than one name per line, and it holds reserved daemon_* keys alongside the + # per-container strike counts. A strike entry for an uninstalled container never clears on its + # own — set_strikes "$container" 0 only fires when the container is seen running again, which + # by definition never happens — so the count sits there and every consumer that treats a + # non-zero strike as unhealthy reports this host unhealthy forever. Confirmed live 2026-08-02: + # claudeclaw_docker at 2119 strikes long after removal, which is what kept the monitor page's + # watchdog summary from ever going green. + if [[ -s "$WATCHDOG_STATE_FILE" ]]; then + mapfile -t _prune_snapshot < "$WATCHDOG_STATE_FILE" + for _prune_line in "${_prune_snapshot[@]}"; do + _prune_container="${_prune_line%%:*}" + [[ -z "$_prune_container" ]] && continue + [[ "$_prune_container" == daemon_* ]] && continue # reserved keys, not containers + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$_prune_container" &>/dev/null; then + sed -i "/^${_prune_container}:/d" "$WATCHDOG_STATE_FILE" 2>/dev/null + warn "$_prune_container no longer exists — removed from $(basename "$WATCHDOG_STATE_FILE")" + fi + done + unset _prune_snapshot _prune_line _prune_container + fi + _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs) [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed" _intentional_contents=$(cat "$DOCKER_WATCHDOG_INTENTIONAL_FILE" 2>/dev/null | tr '\n' ' ' | xargs)