From 4722382d491470bd0b2dc49f9e98f80143ee36a5 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 31 May 2026 20:31:20 -0400 Subject: [PATCH] fix: prevent orchestrator lock from blocking 45min on daemon restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two root causes of the May 31 19:30 hang (orchestrator stuck for 45min): 1. /etc/rc.d/rc.docker restart had no timeout — on a host with many containers, stopping them all before daemon restart can take 30+ minutes. Added timeout 180 so it's bounded. 2. WATCHDOG_DAEMON_RESTART_WAIT=900 (15min sleep after restart) — unnecessary. The daemon itself comes up in seconds; containers are handled by subsequent docker_watchdog cycles. Reduced to 90s in master.conf, cutting worst-case orchestrator block from 45min to ~5min. Co-Authored-By: Claude Sonnet 4.6 --- Watchdogs/docker_watchdog.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 3e7b11c..a364b57 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -570,9 +570,11 @@ check_docker_daemon() { fi # Restart daemon — unRAID uses rc.d scripts, not systemd + # timeout 180: rc.docker stops all containers before restarting — without a limit this + # can block for 30+ min on a busy host, holding the orchestrator lock the entire time. WATCHDOG_DAEMON_RESTARTED=true set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE" - if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then + if timeout 180 /etc/rc.d/rc.docker restart >/dev/null 2>&1; then log "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..." sleep "$WATCHDOG_DAEMON_RESTART_WAIT"