Bound the watchdogs' docker calls — a hung daemon is what they exist to catch
This commit is contained in:
@@ -285,7 +285,7 @@ if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
|
|||||||
warn "DRY RUN — would restart $DDNS_CONTAINER"
|
warn "DRY RUN — would restart $DDNS_CONTAINER"
|
||||||
else
|
else
|
||||||
warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..."
|
warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..."
|
||||||
if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then
|
if timeout "$DOCKER_TIMEOUT" docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then
|
||||||
warn "$DDNS_CONTAINER restarted ✅"
|
warn "$DDNS_CONTAINER restarted ✅"
|
||||||
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \
|
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \
|
||||||
"Network Watchdog" "warning"
|
"Network Watchdog" "warning"
|
||||||
@@ -342,7 +342,7 @@ if [[ -n "$NPM_URL" ]]; then
|
|||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would restart NginxProxyManager"
|
warn "DRY RUN — would restart NginxProxyManager"
|
||||||
else
|
else
|
||||||
if docker restart NginxProxyManager >/dev/null 2>&1; then
|
if timeout "$DOCKER_TIMEOUT" docker restart NginxProxyManager >/dev/null 2>&1; then
|
||||||
warn "NginxProxyManager restarted ✅"
|
warn "NginxProxyManager restarted ✅"
|
||||||
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||||
notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \
|
notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \
|
||||||
|
|||||||
@@ -1231,7 +1231,7 @@ CYCLE_START=$(date +%s)
|
|||||||
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
|
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
|
||||||
CYCLE_END=$(date +%s)
|
CYCLE_END=$(date +%s)
|
||||||
|
|
||||||
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')
|
CONTAINER_COUNT=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l | tr -d ' ')
|
||||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START ))) Containers: $CONTAINER_COUNT"
|
echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START ))) Containers: $CONTAINER_COUNT"
|
||||||
|
|||||||
@@ -210,9 +210,12 @@ _rw_trap_restart_stopped() {
|
|||||||
[[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
|
[[ ${#_RW_TRAP_STOPPED[@]} -eq 0 ]] && return
|
||||||
for c in "${_RW_TRAP_STOPPED[@]}"; do
|
for c in "${_RW_TRAP_STOPPED[@]}"; do
|
||||||
[[ -z "$c" ]] && continue
|
[[ -z "$c" ]] && continue
|
||||||
if docker inspect "$c" >/dev/null 2>&1; then
|
# Bounded, because this runs from the EXIT trap. An unbounded docker call here means a
|
||||||
|
# hung daemon stops the script exiting at all — it keeps its lock, and the containers
|
||||||
|
# this trap exists to bring back stay down.
|
||||||
|
if timeout "$DOCKER_TIMEOUT" docker inspect "$c" >/dev/null 2>&1; then
|
||||||
warn "Exit trap: restarting $c (stopped but state not persisted)"
|
warn "Exit trap: restarting $c (stopped but state not persisted)"
|
||||||
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -449,7 +449,11 @@ _trap_sys_reboot_restart() {
|
|||||||
warn "Exit trap: restarting containers stopped before aborted reboot"
|
warn "Exit trap: restarting containers stopped before aborted reboot"
|
||||||
for c in "${_SYS_REBOOT_STOPPED[@]}"; do
|
for c in "${_SYS_REBOOT_STOPPED[@]}"; do
|
||||||
[[ -z "$c" ]] && continue
|
[[ -z "$c" ]] && continue
|
||||||
docker inspect "$c" >/dev/null 2>&1 && docker start "$c" >/dev/null 2>&1 || true
|
# Bounded — this is the aborted-reboot recovery path, running from an EXIT trap. If the
|
||||||
|
# daemon is wedged (which is a reason a reboot was being attempted), an unbounded call
|
||||||
|
# here leaves every container it stopped down and the script never exits.
|
||||||
|
timeout "$DOCKER_TIMEOUT" docker inspect "$c" >/dev/null 2>&1 \
|
||||||
|
&& timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1 || true
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -529,7 +533,7 @@ do_reboot() {
|
|||||||
|
|
||||||
warn "Stopping Docker containers..."
|
warn "Stopping Docker containers..."
|
||||||
if is_docker_enabled && command -v docker >/dev/null 2>&1; then
|
if is_docker_enabled && command -v docker >/dev/null 2>&1; then
|
||||||
mapfile -t _SYS_REBOOT_STOPPED < <(docker ps --format '{{.Names}}' 2>/dev/null)
|
mapfile -t _SYS_REBOOT_STOPPED < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' 2>/dev/null)
|
||||||
trap _trap_sys_reboot_restart EXIT
|
trap _trap_sys_reboot_restart EXIT
|
||||||
timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1
|
timeout 60 docker ps -q 2>/dev/null | xargs -r docker stop >/dev/null 2>&1
|
||||||
fi
|
fi
|
||||||
|
|||||||
Reference in New Issue
Block a user