Bound the watchdogs' docker calls — a hung daemon is what they exist to catch

This commit is contained in:
Gmer4Lfe
2026-08-24 18:39:59 -04:00
parent 84946ed0c6
commit 049f633667
4 changed files with 14 additions and 7 deletions
+2 -2
View File
@@ -285,7 +285,7 @@ if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
warn "DRY RUN — would restart $DDNS_CONTAINER" warn "DRY RUN — would restart $DDNS_CONTAINER"
else else
warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..." warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..."
if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then if timeout "$DOCKER_TIMEOUT" docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then
warn "$DDNS_CONTAINER restarted ✅" warn "$DDNS_CONTAINER restarted ✅"
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP$DDNS_CONTAINER restarted" \ notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP$DDNS_CONTAINER restarted" \
"Network Watchdog" "warning" "Network Watchdog" "warning"
@@ -342,7 +342,7 @@ if [[ -n "$NPM_URL" ]]; then
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would restart NginxProxyManager" warn "DRY RUN — would restart NginxProxyManager"
else else
if docker restart NginxProxyManager >/dev/null 2>&1; then if timeout "$DOCKER_TIMEOUT" docker restart NginxProxyManager >/dev/null 2>&1; then
warn "NginxProxyManager restarted ✅" warn "NginxProxyManager restarted ✅"
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}" set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \ notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \
+1 -1
View File
@@ -1231,7 +1231,7 @@ CYCLE_START=$(date +%s)
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
CYCLE_END=$(date +%s) CYCLE_END=$(date +%s)
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ') CONTAINER_COUNT=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l | tr -d ' ')
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings" echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START ))) Containers: $CONTAINER_COUNT" echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START ))) Containers: $CONTAINER_COUNT"
+5 -2
View File
@@ -210,9 +210,12 @@ _rw_trap_restart_stopped() {
[[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return [[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
for c in "${_RW_TRAP_STOPPED[@]}"; do for c in "${_RW_TRAP_STOPPED[@]}"; do
[[ -z "$c" ]] && continue [[ -z "$c" ]] && continue
if docker inspect "$c" >/dev/null 2>&1; then # Bounded, because this runs from the EXIT trap. An unbounded docker call here means a
# hung daemon stops the script exiting at all — it keeps its lock, and the containers
# this trap exists to bring back stay down.
if timeout "$DOCKER_TIMEOUT" docker inspect "$c" >/dev/null 2>&1; then
warn "Exit trap: restarting $c (stopped but state not persisted)" warn "Exit trap: restarting $c (stopped but state not persisted)"
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c" timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
fi fi
done done
} }
+6 -2
View File
@@ -449,7 +449,11 @@ _trap_sys_reboot_restart() {
warn "Exit trap: restarting containers stopped before aborted reboot" warn "Exit trap: restarting containers stopped before aborted reboot"
for c in "${_SYS_REBOOT_STOPPED[@]}"; do for c in "${_SYS_REBOOT_STOPPED[@]}"; do
[[ -z "$c" ]] && continue [[ -z "$c" ]] && continue
docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true # Bounded — this is the aborted-reboot recovery path, running from an EXIT trap. If the
# daemon is wedged (which is a reason a reboot was being attempted), an unbounded call
# here leaves every container it stopped down and the script never exits.
timeout "$DOCKER_TIMEOUT" docker inspect "$c" >/dev/null 2>&1 \
&& timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1 || true
done done
} }
@@ -529,7 +533,7 @@ do_reboot() {
warn "Stopping Docker containers..." warn "Stopping Docker containers..."
if is_docker_enabled && command -v docker >/dev/null 2>&1; then if is_docker_enabled && command -v docker >/dev/null 2>&1; then
mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null) mapfile -t _SYS_REBOOT_STOPPED < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' 2>/dev/null)
trap _trap_sys_reboot_restart EXIT trap _trap_sys_reboot_restart EXIT
timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1 timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1
fi fi