Resolve network watchdog host config after detect_hosts

MY_ID is not exported and is unset until detect_hosts runs, so the DDNS and NPM checks were silently resolving empty config and skipping themselves every cycle.
2026-08-01 20:37:58 -04:00
parent 3f5ad22632
commit 77b24e9823
2 changed files with 74 additions and 12 deletions
@@ -33,6 +33,33 @@
 #   External check — verifies the full stack (DNS → NPM → backend), not just NPM running.
 #
 # ==============================================================================================
+# DESIGN PRINCIPLES
+# ==============================================================================================
+#
+# Gate on Root Cause
+#   The internet check runs first and short-circuits everything after it. DDNS
+#   and NPM checks both depend on outbound connectivity — running them during an
+#   outage produces three alarms for one fault and can trigger container restarts
+#   that fix nothing.
+#
+# Verify the Path, Not the Process
+#   NPM is checked by fetching an external URL rather than asking whether the
+#   container is running. A running container behind broken DNS or a broken
+#   upstream still serves nothing. Checking the whole path is the only result
+#   that means anything to a user.
+#
+# Restart Only What Restarting Fixes
+#   DDNS and NPM are restarted because a restart forces a record update or
+#   reloads proxy config — the restart is the remedy. Tailscale is notify-only:
+#   its failures are auth, key expiry or ACL problems that a restart cannot
+#   resolve and may obscure.
+#
+# Strike Before Restarting NPM
+#   NPM sits in front of every externally reachable service, so restarting it is
+#   itself disruptive. A single failed fetch can be a transient upstream blip;
+#   two consecutive failures justify the interruption.
+#
+# ==============================================================================================
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
@@ -42,6 +69,19 @@
 # Single Instance Lock
 #   acquire_lock prevents concurrent runs.
 #
+# Host Detection Before Config Resolution
+#   detect_hosts() runs before the HOST*_NETWORK_WATCHDOG_* names are built. MY_ID
+#   is not exported, so an orchestrated run starts with it empty — resolving these
+#   any earlier silently produces empty DDNS and NPM config and skips both checks.
+#
+# Unresolved Config Warning
+#   Warns when neither DDNS nor NPM config resolves for this host. "Not configured"
+#   is a legitimate state, but it looks identical to a broken lookup, so it is
+#   stated out loud rather than passed over in silence.
+#
+# NETWORK_WATCHDOG_ENABLED Toggle
+#   Exits cleanly when disabled, without removing it from the orchestrator list.
+#
 # Internet Gates All Checks
 #   If internet is down, DDNS and NPM checks are skipped — no cascade of false positives.
 #
@@ -49,6 +89,17 @@
 #   Single curl failure could be transient DNS hiccup or CDN blip.
 #   Two consecutive failures confirms NPM is the problem.
 #
+# Notify-Only for Tailscale
+#   Tailscale is never restarted. Its failures are auth, key expiry or ACL issues
+#   that a restart cannot fix and would only obscure.
+#
+# Timeout Protection
+#   Every connectivity probe carries an explicit timeout, so a black-holed route
+#   cannot stall the every-minute watchdog chain.
+#
+# Dry Run Support
+#   --dry-run performs all checks and reports restarts without issuing them.
+#
 # ==============================================================================================
 # CONFIGURATION
 # ==============================================================================================
@@ -93,9 +144,24 @@ source "$SCRIPT_DIR/../../load_config.sh"

 parse_args "$@"

+# ==============================================================================================
+# ━━━ Setup ━━━
+# ==============================================================================================
+if [[ "$EUID" -ne 0 ]]; then
+    error "Must be run as root"
+    exit 1
+fi
+
+acquire_lock
+
+detect_hosts
+
 # ==============================================================================================
 # ━━━ Resolve host-specific config ━━━
 # ==============================================================================================
+# MUST run after detect_hosts() — MY_ID is set there and is not exported, so orchestrated
+# runs start with it empty. Building these names any earlier yields "_NETWORK_WATCHDOG_*",
+# which is always unset, and the DDNS and NPM checks silently skip as "not configured".
 _ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN"
 _ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER"
 _npm_url_var="${MY_ID}_NETWORK_WATCHDOG_NPM_URL"
@@ -104,19 +170,13 @@ DDNS_DOMAIN="${!_ddns_domain_var:-}"
 DDNS_CONTAINER="${!_ddns_container_var:-}"
 NPM_URL="${!_npm_url_var:-}"

-# ==============================================================================================
-# ━━━ Setup ━━━
-# ==============================================================================================
-if [[ "$EUID" -ne 0 ]]; then
-    error "Must be run as root"
-    exit 1
+# A check that is configured but resolves empty means the lookup broke, not that the
+# operator opted out. Say so — silent skipping is what hid this for so long.
+if [[ -z "$DDNS_DOMAIN" && -z "$NPM_URL" ]]; then
+    warn "No DDNS or NPM config resolved for ${MY_ID:-unknown host}"
+    warn "Expected ${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN / ${MY_ID}_NETWORK_WATCHDOG_NPM_URL in host*.conf"
 fi

-
-acquire_lock
-
-detect_hosts
-
 [[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0
 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"