From 0785a46fe32a3e401719fa4e931c0a7f13078a86 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 9 Aug 2026 18:33:08 -0400 Subject: [PATCH] Judge watchdog strike keys by their container, not the key itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The state file is keyed both ways — a bare container name for the container check, container plus suffix for the HTTP, API, CPU and docker checks — so inspecting the key asked docker about Emby_http and deleted a live counter every cycle, putting the two-strike limits out of reach. --- Watchdogs/docker_watchdog.sh | 43 +++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 6d67184..6f25d53 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -144,6 +144,14 @@ # clearing path requires seeing the container running again, which never happens once # it is uninstalled. Without pruning, one removed container makes this host report # unhealthy permanently. Reserved daemon_* keys in WATCHDOG_STATE_FILE are never pruned. +# +# WATCHDOG_STATE_FILE is keyed two ways and the prune has to know it. The container check +# stores a bare name; the HTTP, API, CPU and docker checks store container + suffix. The +# existence test is always against the base container — inspecting the composite key means +# asking docker about "Emby_http", which fails while Emby is running perfectly, and deleting +# the live counter every cycle. That made RESP_FAIL_LIMIT and CPU_FAIL_LIMIT unreachable, +# since a counter reset each cycle never reaches 2. Only the four known suffixes are +# stripped: a general split on the last underscore would break PostgreSQL_Immich. # Pruning runs under --dry-run as well: the documented contract for that flag is that # nothing gets restarted, and dropping a record of a container that no longer exists is # housekeeping, not an action. It is idempotent — repeated runs converge. @@ -732,15 +740,38 @@ CYCLE_START=$(date +%s) if [[ -s "$WATCHDOG_STATE_FILE" ]]; then mapfile -t _prune_snapshot < "$WATCHDOG_STATE_FILE" for _prune_line in "${_prune_snapshot[@]}"; do - _prune_container="${_prune_line%%:*}" - [[ -z "$_prune_container" ]] && continue - [[ "$_prune_container" == daemon_* ]] && continue # reserved keys, not containers + _prune_key="${_prune_line%%:*}" + [[ -z "$_prune_key" ]] && continue + [[ "$_prune_key" == daemon_* ]] && continue # reserved keys, not containers + + # Keys come in two shapes: the bare container name for the container check, and + # container + check suffix for the others. `docker inspect Emby_http` fails for a + # perfectly healthy Emby, so inspecting the key itself deleted every live per-check + # counter on every cycle — before the checks below read them. RESP_FAIL_LIMIT and + # CPU_FAIL_LIMIT of 2 were therefore unreachable: a counter wiped each cycle can + # only ever reach 1. Latent rather than harmful only because no container here had + # yet failed one of those checks. + # + # Strip only the four known suffixes, never on the last underscore: PostgreSQL_Immich + # is a real container, and splitting it would prune a name that does exist. + # + # Known limit: a container literally named something_http or something_docker would + # have its own counter judged by whether "something" exists. No container here ends + # in one of the four, and the alternative — inspecting the key, then the base — pays + # a second docker call for every composite key on every cycle to cover a name nobody + # uses. Worth revisiting only alongside replacing these per-key inspects with one + # `docker ps -a` membership test. + _prune_container="$_prune_key" + case "$_prune_key" in + *_http|*_api|*_cpu|*_docker) _prune_container="${_prune_key%_*}" ;; + esac + if ! timeout "$DOCKER_TIMEOUT" docker inspect "$_prune_container" &>/dev/null; then - sed -i "/^${_prune_container}:/d" "$WATCHDOG_STATE_FILE" 2>/dev/null - warn "$_prune_container no longer exists — removed from $(basename "$WATCHDOG_STATE_FILE")" + sed -i "/^${_prune_key}:/d" "$WATCHDOG_STATE_FILE" 2>/dev/null + warn "$_prune_container no longer exists — removed $_prune_key from $(basename "$WATCHDOG_STATE_FILE")" fi done - unset _prune_snapshot _prune_line _prune_container + unset _prune_snapshot _prune_line _prune_key _prune_container fi _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs)