Varaverk/Watchdogs/System/network_watchdog.sh

#!/bin/bash
# ==============================================================================================
# ================================= Network Watchdog ===========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Services-layer connectivity — checks that the outside world can actually reach
# what it needs to reach. Silent when everything is reachable. Only fires when
# something in the connectivity chain has broken.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Check 1 — Internet Connectivity
#   curl external endpoint → fail = alert + skip all remaining checks.
#   Internet down means DDNS and NPM checks would false-positive — gating prevents noise.
#
# Check 2 — DDNS (Cloudflare)
#   Public IP via ifconfig.me vs DNS record via dig @1.1.1.1.
#   Match → pass (silent). Mismatch → restart DDNS container + notify.
#   Container restart triggers an immediate Cloudflare record update.
#
# Check 3 — Tailscale
#   tailscale status → Running → pass (silent). Not running → notify.
#   Notify only — no restart attempt. Tailscale state issues warrant human review.
#
# Check 4 — NPM Proxy (external check)
#   curl external URL → 2-strike system before restarting NginxProxyManager.
#   Strike 1: warn + notify. Strike 2: restart NPM + notify + clear strikes.
#   Strikes auto-clear when the external URL becomes reachable again.
#   External check — verifies the full stack (DNS → NPM → backend), not just NPM running.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Gate on Root Cause
#   The internet check runs first and short-circuits everything after it. DDNS
#   and NPM checks both depend on outbound connectivity — running them during an
#   outage produces three alarms for one fault and can trigger container restarts
#   that fix nothing.
#
# Verify the Path, Not the Process
#   NPM is checked by fetching an external URL rather than asking whether the
#   container is running. A running container behind broken DNS or a broken
#   upstream still serves nothing. Checking the whole path is the only result
#   that means anything to a user.
#
# Restart Only What Restarting Fixes
#   DDNS and NPM are restarted because a restart forces a record update or
#   reloads proxy config — the restart is the remedy. Tailscale is notify-only:
#   its failures are auth, key expiry or ACL problems that a restart cannot
#   resolve and may obscure.
#
# Strike Before Restarting NPM
#   NPM sits in front of every externally reachable service, so restarting it is
#   itself disruptive. A single failed fetch can be a transient upstream blip;
#   two consecutive failures justify the interruption.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Required
#   docker restart requires root.
#
# Single Instance Lock
#   acquire_lock prevents concurrent runs.
#
# Host Detection Before Config Resolution
#   detect_hosts() runs before the HOST*_NETWORK_WATCHDOG_* names are built. MY_ID
#   is not exported, so an orchestrated run starts with it empty — resolving these
#   any earlier silently produces empty DDNS and NPM config and skips both checks.
#
# Unresolved Config Warning
#   Warns when neither DDNS nor NPM config resolves for this host. "Not configured"
#   is a legitimate state, but it looks identical to a broken lookup, so it is
#   stated out loud rather than passed over in silence.
#
# NETWORK_WATCHDOG_ENABLED Toggle
#   Exits cleanly when disabled, without removing it from the orchestrator list.
#
# Internet Gates All Checks
#   If internet is down, DDNS and NPM checks are skipped — no cascade of false positives.
#
# Strike Before Acting on NPM
#   Single curl failure could be transient DNS hiccup or CDN blip.
#   Two consecutive failures confirms NPM is the problem.
#
# Notify-Only for Tailscale
#   Tailscale is never restarted. Its failures are auth, key expiry or ACL issues
#   that a restart cannot fix and would only obscure.
#
# Timeout Protection
#   Every connectivity probe carries an explicit timeout, so a black-holed route
#   cannot stall the every-minute watchdog chain.
#
# Dry Run Support
#   --dry-run performs all checks and reports restarts without issuing them.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   NETWORK_WATCHDOG_ENABLED          toggle entire watchdog (default: true)
#   NETWORK_WATCHDOG_INTERNET_URL     endpoint for internet reachability check
#   NETWORK_WATCHDOG_INTERNET_TIMEOUT curl timeout in seconds for internet check
#   NETWORK_WATCHDOG_CHECK_TAILSCALE  toggle tailscale check (default: true)
#   NETWORK_WATCHDOG_NPM_TIMEOUT      curl timeout for NPM external check
#   NETWORK_WATCHDOG_NPM_STRIKE_LIMIT consecutive failures before NPM restart
#   NETWORK_WATCHDOG_NPM_STATE_FILE   strike count persistence (/tmp — resets on reboot)
#
# host*.conf (host-specific)
#
#   HOST*_NETWORK_WATCHDOG_DDNS_DOMAIN     domain to resolve and compare to public IP
#   HOST*_NETWORK_WATCHDOG_DDNS_CONTAINER  container to restart on DDNS mismatch
#   HOST*_NETWORK_WATCHDOG_NPM_URL         external URL to test full proxy stack
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# network_watchdog.sh
#     Run all connectivity checks. Silent when healthy.
#
# network_watchdog.sh --dry-run
#     Run all checks without restarting any containers.
#
# network_watchdog.sh --status
#     Show configuration, current public IP, DNS record, NPM strike state.
#
# network_watchdog.sh --log
#     Verbose output — show each check result even when passing.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

detect_hosts

# ==============================================================================================
# ━━━ Resolve host-specific config ━━━
# ==============================================================================================
# MUST run after detect_hosts() — MY_ID is set there and is not exported, so orchestrated
# runs start with it empty. Building these names any earlier yields "_NETWORK_WATCHDOG_*",
# which is always unset, and the DDNS and NPM checks silently skip as "not configured".
_ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN"
_ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER"
_npm_url_var="${MY_ID}_NETWORK_WATCHDOG_NPM_URL"

DDNS_DOMAIN="${!_ddns_domain_var:-}"
DDNS_CONTAINER="${!_ddns_container_var:-}"
NPM_URL="${!_npm_url_var:-}"

# A check that is configured but resolves empty means the lookup broke, not that the
# operator opted out. Say so — silent skipping is what hid this for so long.
if [[ -z "$DDNS_DOMAIN" && -z "$NPM_URL" ]]; then
    warn "No DDNS or NPM config resolved for ${MY_ID:-unknown host}"
    warn "Expected ${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN / ${MY_ID}_NETWORK_WATCHDOG_NPM_URL in host*.conf"
fi

[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && echo "Network watchdog disabled" && exit 0
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"

log "$ICON_GEAR Config: internet=${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1} timeout=${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}s tailscale=${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true} npm-strikes=${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
log "$ICON_NET   DDNS: ${DDNS_DOMAIN:-not configured} → ${DDNS_CONTAINER:-no container}  NPM: ${NPM_URL:-not configured}"

touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null

# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
get_strikes() { wd_state_get "$1" "$2"; }
set_strikes() { wd_state_set "$1" "$2" "$3"; }

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY NETWORK WATCHDOG STATUS ━━━━━"
    echo "$ICON_HOST  Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR  Internet URL:   ${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}"
    echo "$ICON_GEAR  DDNS domain:    ${DDNS_DOMAIN:-not configured}"
    echo "$ICON_GEAR  DDNS container: ${DDNS_CONTAINER:-not configured}"
    echo "$ICON_GEAR  Tailscale:      ${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}"
    echo "$ICON_GEAR  NPM URL:        ${NPM_URL:-not configured}"
    echo "$ICON_GEAR  NPM strikes:    $(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}") / ${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
    echo ""
    echo "── Current State ──"

    if curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \
        "${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then
        echo "  $ICON_SUCCESS Internet:   reachable"
    else
        echo "  $ICON_ERROR  Internet:   NOT reachable"
    fi

    if [[ -n "$DDNS_DOMAIN" ]]; then
        _pub=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
        _dns=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
        echo "  $ICON_GEAR   Public IP:  ${_pub:-unknown}"
        echo "  $ICON_GEAR   DNS record: ${_dns:-unknown}"
        [[ "$_pub" == "$_dns" ]] && \
            echo "  $ICON_SUCCESS DDNS:       in sync" || \
            echo "  $ICON_ERROR  DDNS:       MISMATCH — public=$_pub dns=$_dns"
    else
        echo "  $ICON_GEAR   DDNS:       not configured for $MY_ID"
    fi

    if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]] && command -v tailscale >/dev/null 2>&1; then
        if tailscale status --json 2>/dev/null | grep -qE '"BackendState":\s*"Running"'; then
            echo "  $ICON_SUCCESS Tailscale:  running"
        else
            echo "  $ICON_ERROR  Tailscale:  NOT running"
        fi
    else
        echo "  $ICON_GEAR   Tailscale:  check disabled or not installed"
    fi

    if [[ -n "$NPM_URL" ]]; then
        if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
            echo "  $ICON_SUCCESS NPM proxy:  reachable ($NPM_URL)"
        else
            echo "  $ICON_ERROR  NPM proxy:  NOT reachable ($NPM_URL)"
        fi
    else
        echo "  $ICON_GEAR   NPM proxy:  not configured for $MY_ID"
    fi

    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Check 1 — Internet Connectivity ━━━
# ==============================================================================================
echo "━━━ $ICON_NET Network Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

ISSUES=0

if ! curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \
    "${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then
    warn "$ICON_ERROR Internet not reachable — skipping DDNS, Tailscale, and NPM checks"
    notify "Network watchdog: internet not reachable on $(hostname) ($MY_ID)" \
        "Network Watchdog" "warning"
    exit 1
fi

log "$ICON_SUCCESS Internet reachable"

# ==============================================================================================
# ━━━ Check 2 — DDNS ━━━
# ==============================================================================================
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
    PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
    DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)

    if [[ -z "$PUBLIC_IP" ]]; then
        warn "Could not determine public IP — skipping DDNS check"
    elif [[ -z "$DNS_IP" ]]; then
        warn "Could not resolve $DDNS_DOMAIN — skipping DDNS check"
    elif [[ "$PUBLIC_IP" == "$DNS_IP" ]]; then
        log "$ICON_SUCCESS DDNS in sync — $DDNS_DOMAIN → $DNS_IP"
    else
        warn "$ICON_ERROR DDNS mismatch — public=$PUBLIC_IP dns=$DNS_IP"
        (( ISSUES++ ))
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $DDNS_CONTAINER"
        else
            warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..."
            if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then
                warn "$DDNS_CONTAINER restarted ✅"
                notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \
                    "Network Watchdog" "warning"
            else
                warn "$DDNS_CONTAINER restart failed"
                notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_CONTAINER restart failed — manual intervention needed" \
                    "Network Watchdog" "warning"
            fi
        fi
    fi
else
    log "DDNS check not configured for $MY_ID — skipping"
fi

# ==============================================================================================
# ━━━ Check 3 — Tailscale ━━━
# ==============================================================================================
if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]]; then
    if ! command -v tailscale >/dev/null 2>&1; then
        log "Tailscale not installed — skipping"
    elif tailscale status --json 2>/dev/null | grep -qE '"BackendState":\s*"Running"'; then
        log "$ICON_SUCCESS Tailscale running"
    else
        warn "$ICON_ERROR Tailscale not in Running state"
        notify "Tailscale not running on $(hostname) ($MY_ID) — manual check needed" \
            "Network Watchdog" "warning"
    fi
fi

# ==============================================================================================
# ━━━ Check 4 — NPM Proxy (external) ━━━
# ==============================================================================================
if [[ -n "$NPM_URL" ]]; then
    NPM_STRIKES=$(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}")
    NPM_STRIKE_LIMIT="${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"

    if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
        log "$ICON_SUCCESS NPM proxy reachable — $NPM_URL"
        if [[ "$NPM_STRIKES" -gt 0 ]]; then
            echo "NPM strikes cleared (was $NPM_STRIKES)"
            set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
        fi
    else
        NPM_STRIKES=$(( NPM_STRIKES + 1 ))
        set_strikes "npm" "$NPM_STRIKES" "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
        (( ISSUES++ ))

        if [[ "$NPM_STRIKES" -lt "$NPM_STRIKE_LIMIT" ]]; then
            warn "$ICON_ERROR NPM proxy not reachable — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)"
            notify "NPM proxy not reachable on $(hostname) ($MY_ID) — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)" \
                "Network Watchdog" "warning"
        else
            warn "$ICON_ERROR NPM proxy strike limit reached ($NPM_STRIKES) — restarting NginxProxyManager"
            if [[ "$DRY_RUN" == true ]]; then
                warn "DRY RUN — would restart NginxProxyManager"
            else
                if docker restart NginxProxyManager >/dev/null 2>&1; then
                    warn "NginxProxyManager restarted ✅"
                    set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
                    notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \
                        "Network Watchdog" "warning"
                else
                    warn "NginxProxyManager restart failed — manual intervention needed"
                    notify "NPM proxy restart FAILED on $(hostname) ($MY_ID) — manual intervention needed" \
                        "Network Watchdog" "warning"
                fi
            fi
        fi
    fi
else
    log "NPM check not configured for $MY_ID — skipping"
fi

# ==============================================================================================
# ━━━ Exit ━━━
# ==============================================================================================
if [[ "$ISSUES" -gt 0 ]]; then
    exit 1
else
    echo "Network healthy ✅ ($(date '+%H:%M:%S'))"
    exit 0
fi