diff --git a/Watchdogs/System/network_watchdog.sh b/Watchdogs/System/network_watchdog.sh index a4c9d79..da5aa12 100755 --- a/Watchdogs/System/network_watchdog.sh +++ b/Watchdogs/System/network_watchdog.sh @@ -33,6 +33,33 @@ # External check — verifies the full stack (DNS → NPM → backend), not just NPM running. # # ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Gate on Root Cause +# The internet check runs first and short-circuits everything after it. DDNS +# and NPM checks both depend on outbound connectivity — running them during an +# outage produces three alarms for one fault and can trigger container restarts +# that fix nothing. +# +# Verify the Path, Not the Process +# NPM is checked by fetching an external URL rather than asking whether the +# container is running. A running container behind broken DNS or a broken +# upstream still serves nothing. Checking the whole path is the only result +# that means anything to a user. +# +# Restart Only What Restarting Fixes +# DDNS and NPM are restarted because a restart forces a record update or +# reloads proxy config — the restart is the remedy. Tailscale is notify-only: +# its failures are auth, key expiry or ACL problems that a restart cannot +# resolve and may obscure. +# +# Strike Before Restarting NPM +# NPM sits in front of every externally reachable service, so restarting it is +# itself disruptive. A single failed fetch can be a transient upstream blip; +# two consecutive failures justify the interruption. +# +# ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # @@ -42,6 +69,19 @@ # Single Instance Lock # acquire_lock prevents concurrent runs. # +# Host Detection Before Config Resolution +# detect_hosts() runs before the HOST*_NETWORK_WATCHDOG_* names are built. MY_ID +# is not exported, so an orchestrated run starts with it empty — resolving these +# any earlier silently produces empty DDNS and NPM config and skips both checks. +# +# Unresolved Config Warning +# Warns when neither DDNS nor NPM config resolves for this host. "Not configured" +# is a legitimate state, but it looks identical to a broken lookup, so it is +# stated out loud rather than passed over in silence. +# +# NETWORK_WATCHDOG_ENABLED Toggle +# Exits cleanly when disabled, without removing it from the orchestrator list. +# # Internet Gates All Checks # If internet is down, DDNS and NPM checks are skipped — no cascade of false positives. # @@ -49,6 +89,17 @@ # Single curl failure could be transient DNS hiccup or CDN blip. # Two consecutive failures confirms NPM is the problem. # +# Notify-Only for Tailscale +# Tailscale is never restarted. Its failures are auth, key expiry or ACL issues +# that a restart cannot fix and would only obscure. +# +# Timeout Protection +# Every connectivity probe carries an explicit timeout, so a black-holed route +# cannot stall the every-minute watchdog chain. +# +# Dry Run Support +# --dry-run performs all checks and reports restarts without issuing them. +# # ============================================================================================== # CONFIGURATION # ============================================================================================== @@ -93,9 +144,24 @@ source "$SCRIPT_DIR/../../load_config.sh" parse_args "$@" +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +acquire_lock + +detect_hosts + # ============================================================================================== # ━━━ Resolve host-specific config ━━━ # ============================================================================================== +# MUST run after detect_hosts() — MY_ID is set there and is not exported, so orchestrated +# runs start with it empty. Building these names any earlier yields "_NETWORK_WATCHDOG_*", +# which is always unset, and the DDNS and NPM checks silently skip as "not configured". _ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN" _ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER" _npm_url_var="${MY_ID}_NETWORK_WATCHDOG_NPM_URL" @@ -104,19 +170,13 @@ DDNS_DOMAIN="${!_ddns_domain_var:-}" DDNS_CONTAINER="${!_ddns_container_var:-}" NPM_URL="${!_npm_url_var:-}" -# ============================================================================================== -# ━━━ Setup ━━━ -# ============================================================================================== -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root" - exit 1 +# A check that is configured but resolves empty means the lookup broke, not that the +# operator opted out. Say so — silent skipping is what hid this for so long. +if [[ -z "$DDNS_DOMAIN" && -z "$NPM_URL" ]]; then + warn "No DDNS or NPM config resolved for ${MY_ID:-unknown host}" + warn "Expected ${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN / ${MY_ID}_NETWORK_WATCHDOG_NPM_URL in host*.conf" fi - -acquire_lock - -detect_hosts - [[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" diff --git a/load_config.sh b/load_config.sh index efe4dd6..5dfc6b2 100755 --- a/load_config.sh +++ b/load_config.sh @@ -116,7 +116,9 @@ unset _VV_CONF_CACHE _conf_base _disk_conf_basenames # ━━━ Source shared functions ━━━ -# common.sh sourced last — it calls detect_hosts() which needs HOST* vars to be set. +# common.sh sourced last — it defines detect_hosts(), which needs HOST* vars to be set. +# It does NOT call it. Each script calls detect_hosts() itself, so MY_ID and REMOTE_ID +# are unset until it does — anything building HOST*-prefixed variable names must come after. if [[ ! -f "$LOAD_CONFIG_DIR/common.sh" ]]; then echo "[FATAL] common.sh not found at $LOAD_CONFIG_DIR/common.sh" >&2 exit 1