From 049f6336678420bdbf943337ee76584fbb122a67 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 24 Aug 2026 18:39:59 -0400 Subject: [PATCH] =?UTF-8?q?Bound=20the=20watchdogs'=20docker=20calls=20?= =?UTF-8?q?=E2=80=94=20a=20hung=20daemon=20is=20what=20they=20exist=20to?= =?UTF-8?q?=20catch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Watchdogs/System/network_watchdog.sh | 4 ++-- Watchdogs/docker_watchdog.sh | 2 +- Watchdogs/resource_watchdog.sh | 7 +++++-- Watchdogs/stability_watchdog.sh | 8 ++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Watchdogs/System/network_watchdog.sh b/Watchdogs/System/network_watchdog.sh index da5aa12..e809025 100755 --- a/Watchdogs/System/network_watchdog.sh +++ b/Watchdogs/System/network_watchdog.sh @@ -285,7 +285,7 @@ if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then warn "DRY RUN — would restart $DDNS_CONTAINER" else warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..." - if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then + if timeout "$DOCKER_TIMEOUT" docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then warn "$DDNS_CONTAINER restarted ✅" notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \ "Network Watchdog" "warning" @@ -342,7 +342,7 @@ if [[ -n "$NPM_URL" ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart NginxProxyManager" else - if docker restart NginxProxyManager >/dev/null 2>&1; then + if timeout "$DOCKER_TIMEOUT" docker restart NginxProxyManager >/dev/null 2>&1; then warn "NginxProxyManager restarted ✅" set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}" notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \ diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 1edd401..da8c1ae 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -1231,7 +1231,7 @@ CYCLE_START=$(date +%s) TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) CYCLE_END=$(date +%s) - CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ') + CONTAINER_COUNT=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l | tr -d ' ') if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings" echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START ))) Containers: $CONTAINER_COUNT" diff --git a/Watchdogs/resource_watchdog.sh b/Watchdogs/resource_watchdog.sh index 0282d9f..c7c80d6 100755 --- a/Watchdogs/resource_watchdog.sh +++ b/Watchdogs/resource_watchdog.sh @@ -210,9 +210,12 @@ _rw_trap_restart_stopped() { [[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return for c in "${_RW_TRAP_STOPPED[@]}"; do [[ -z "$c" ]] && continue - if docker inspect "$c" >/dev/null 2>&1; then + # Bounded, because this runs from the EXIT trap. An unbounded docker call here means a + # hung daemon stops the script exiting at all — it keeps its lock, and the containers + # this trap exists to bring back stay down. + if timeout "$DOCKER_TIMEOUT" docker inspect "$c" >/dev/null 2>&1; then warn "Exit trap: restarting $c (stopped but state not persisted)" - docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c" + timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c" fi done } diff --git a/Watchdogs/stability_watchdog.sh b/Watchdogs/stability_watchdog.sh index bf986ec..0c7f4ad 100755 --- a/Watchdogs/stability_watchdog.sh +++ b/Watchdogs/stability_watchdog.sh @@ -449,7 +449,11 @@ _trap_sys_reboot_restart() { warn "Exit trap: restarting containers stopped before aborted reboot" for c in "${_SYS_REBOOT_STOPPED[@]}"; do [[ -z "$c" ]] && continue - docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true + # Bounded — this is the aborted-reboot recovery path, running from an EXIT trap. If the + # daemon is wedged (which is a reason a reboot was being attempted), an unbounded call + # here leaves every container it stopped down and the script never exits. + timeout "$DOCKER_TIMEOUT" docker inspect "$c" >/dev/null 2>&1 \ + && timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1 || true done } @@ -529,7 +533,7 @@ do_reboot() { warn "Stopping Docker containers..." if is_docker_enabled && command -v docker >/dev/null 2>&1; then - mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null) + mapfile -t _SYS_REBOOT_STOPPED < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' 2>/dev/null) trap _trap_sys_reboot_restart EXIT timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1 fi