diff --git a/Configurations/host1.conf b/Configurations/host1.conf index ab88282..f77e87e 100644 --- a/Configurations/host1.conf +++ b/Configurations/host1.conf @@ -346,6 +346,12 @@ ["7dtd"]="20480" # 20GB — game server world data, expected to be large ) +# ━━━ Network Watchdog ━━━ +# Host-specific connectivity config for Watchdogs/System/network_watchdog.sh. + HOST1_NETWORK_WATCHDOG_DDNS_DOMAIN="gmer4lfe.com" + HOST1_NETWORK_WATCHDOG_DDNS_CONTAINER="Gmer4Lfe.com" + HOST1_NETWORK_WATCHDOG_NPM_URL="https://gmer4lfe.com" + # ━━━ Docker Network Connect ━━━ # Containers connected to custom networks at array start by docker_network_connect.sh. # Networks created if they don't exist — idempotent, safe to re-run. diff --git a/Configurations/master.conf b/Configurations/master.conf index 234e2e0..a74f81c 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -308,6 +308,7 @@ SYSTEM_WATCHDOG_SCRIPTS=( "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection "Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp + "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy ) # ━━━ Critical Sync Maintenance ━━━ @@ -815,6 +816,17 @@ LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small) LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB) +# ━━━ Network Watchdog ━━━ +# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy. +# Host-specific values (domain, container, NPM URL) live in host*.conf. + NETWORK_WATCHDOG_ENABLED=true + NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1" + NETWORK_WATCHDOG_INTERNET_TIMEOUT=5 + NETWORK_WATCHDOG_CHECK_TAILSCALE=true + NETWORK_WATCHDOG_NPM_TIMEOUT=10 + NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2 + NETWORK_WATCHDOG_NPM_STATE_FILE="/tmp/network_watchdog_state.db" + # ━━━ WebGUI Watchdog ━━━ # Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. # Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. diff --git a/Watchdogs/System/network_watchdog.sh b/Watchdogs/System/network_watchdog.sh new file mode 100755 index 0000000..3998127 --- /dev/null +++ b/Watchdogs/System/network_watchdog.sh @@ -0,0 +1,315 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Network Watchdog =========================================== +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Services-layer connectivity — checks that the outside world can actually reach +# what it needs to reach. Silent when everything is reachable. Only fires when +# something in the connectivity chain has broken. +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Check 1 — Internet Connectivity +# curl external endpoint → fail = alert + skip all remaining checks. +# Internet down means DDNS and NPM checks would false-positive — gating prevents noise. +# +# Check 2 — DDNS (Cloudflare) +# Public IP via ifconfig.me vs DNS record via dig @1.1.1.1. +# Match → pass (silent). Mismatch → restart DDNS container + notify. +# Container restart triggers an immediate Cloudflare record update. +# +# Check 3 — Tailscale +# tailscale status → Running → pass (silent). Not running → notify. +# Notify only — no restart attempt. Tailscale state issues warrant human review. +# +# Check 4 — NPM Proxy (external check) +# curl external URL → 2-strike system before restarting NginxProxyManager. +# Strike 1: warn + notify. Strike 2: restart NPM + notify + clear strikes. +# Strikes auto-clear when the external URL becomes reachable again. +# External check — verifies the full stack (DNS → NPM → backend), not just NPM running. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# docker restart requires root. +# +# Single Instance Lock +# acquire_lock prevents concurrent runs. +# +# Internet Gates All Checks +# If internet is down, DDNS and NPM checks are skipped — no cascade of false positives. +# +# Strike Before Acting on NPM +# Single curl failure could be transient DNS hiccup or CDN blip. +# Two consecutive failures confirms NPM is the problem. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# NETWORK_WATCHDOG_ENABLED toggle entire watchdog (default: true) +# NETWORK_WATCHDOG_INTERNET_URL endpoint for internet reachability check +# NETWORK_WATCHDOG_INTERNET_TIMEOUT curl timeout in seconds for internet check +# NETWORK_WATCHDOG_CHECK_TAILSCALE toggle tailscale check (default: true) +# NETWORK_WATCHDOG_NPM_TIMEOUT curl timeout for NPM external check +# NETWORK_WATCHDOG_NPM_STRIKE_LIMIT consecutive failures before NPM restart +# NETWORK_WATCHDOG_NPM_STATE_FILE strike count persistence (/tmp — resets on reboot) +# +# host*.conf (host-specific) +# +# HOST*_NETWORK_WATCHDOG_DDNS_DOMAIN domain to resolve and compare to public IP +# HOST*_NETWORK_WATCHDOG_DDNS_CONTAINER container to restart on DDNS mismatch +# HOST*_NETWORK_WATCHDOG_NPM_URL external URL to test full proxy stack +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# network_watchdog.sh +# Run all connectivity checks. Silent when healthy. +# +# network_watchdog.sh --dry-run +# Run all checks without restarting any containers. +# +# network_watchdog.sh --status +# Show configuration, current public IP, DNS record, NPM strike state. +# +# network_watchdog.sh --log +# Verbose output — show each check result even when passing. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Resolve host-specific config ━━━ +# ============================================================================================== +_ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN" +_ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER" +_npm_url_var="${MY_ID}_NETWORK_WATCHDOG_NPM_URL" + +DDNS_DOMAIN="${!_ddns_domain_var:-}" +DDNS_CONTAINER="${!_ddns_container_var:-}" +NPM_URL="${!_npm_url_var:-}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +validate_unraid_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +detect_hosts + +[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && log "Network watchdog disabled" && exit 0 +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" + +touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null + +# ━━━ Strike helpers ━━━ +get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; } +set_strikes() { + local key="$1" count="$2" file="$3" + if grep -q "^${key}:" "$file" 2>/dev/null; then + sed -i "s|^${key}:.*|${key}:${count}|" "$file" + else + echo "${key}:${count}" >> "$file" + fi +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY NETWORK WATCHDOG STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Internet URL: ${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" + echo "$ICON_GEAR DDNS domain: ${DDNS_DOMAIN:-not configured}" + echo "$ICON_GEAR DDNS container: ${DDNS_CONTAINER:-not configured}" + echo "$ICON_GEAR Tailscale: ${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" + echo "$ICON_GEAR NPM URL: ${NPM_URL:-not configured}" + echo "$ICON_GEAR NPM strikes: $(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}") / ${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}" + echo "" + echo "── Current State ──" + + if curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \ + "${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then + echo " $ICON_SUCCESS Internet: reachable" + else + echo " $ICON_ERROR Internet: NOT reachable" + fi + + if [[ -n "$DDNS_DOMAIN" ]]; then + _pub=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]') + _dns=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1) + echo " $ICON_GEAR Public IP: ${_pub:-unknown}" + echo " $ICON_GEAR DNS record: ${_dns:-unknown}" + [[ "$_pub" == "$_dns" ]] && \ + echo " $ICON_SUCCESS DDNS: in sync" || \ + echo " $ICON_ERROR DDNS: MISMATCH — public=$_pub dns=$_dns" + else + echo " $ICON_GEAR DDNS: not configured for $MY_ID" + fi + + if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]] && command -v tailscale >/dev/null 2>&1; then + if tailscale status --json 2>/dev/null | grep -q '"BackendState":"Running"'; then + echo " $ICON_SUCCESS Tailscale: running" + else + echo " $ICON_ERROR Tailscale: NOT running" + fi + else + echo " $ICON_GEAR Tailscale: check disabled or not installed" + fi + + if [[ -n "$NPM_URL" ]]; then + if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then + echo " $ICON_SUCCESS NPM proxy: reachable ($NPM_URL)" + else + echo " $ICON_ERROR NPM proxy: NOT reachable ($NPM_URL)" + fi + else + echo " $ICON_GEAR NPM proxy: not configured for $MY_ID" + fi + + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Check 1 — Internet Connectivity ━━━ +# ============================================================================================== +log "Network watchdog — $MY_ID — $(date '+%H:%M:%S')" + +ISSUES=0 + +if ! curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \ + "${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then + warn "$ICON_ERROR Internet not reachable — skipping DDNS, Tailscale, and NPM checks" + notify "Network watchdog: internet not reachable on $(hostname) ($MY_ID)" \ + "Network Watchdog" "warning" + exit 1 +fi + +log "$ICON_SUCCESS Internet reachable" + +# ============================================================================================== +# ━━━ Check 2 — DDNS ━━━ +# ============================================================================================== +if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then + PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]') + DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1) + + if [[ -z "$PUBLIC_IP" ]]; then + warn "Could not determine public IP — skipping DDNS check" + elif [[ -z "$DNS_IP" ]]; then + warn "Could not resolve $DDNS_DOMAIN — skipping DDNS check" + elif [[ "$PUBLIC_IP" == "$DNS_IP" ]]; then + log "$ICON_SUCCESS DDNS in sync — $DDNS_DOMAIN → $DNS_IP" + else + warn "$ICON_ERROR DDNS mismatch — public=$PUBLIC_IP dns=$DNS_IP" + (( ISSUES++ )) + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $DDNS_CONTAINER" + else + warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..." + if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then + warn "$DDNS_CONTAINER restarted ✅" + notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \ + "Network Watchdog" "warning" + else + warn "$DDNS_CONTAINER restart failed" + notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_CONTAINER restart failed — manual intervention needed" \ + "Network Watchdog" "warning" + fi + fi + fi +else + log "DDNS check not configured for $MY_ID — skipping" +fi + +# ============================================================================================== +# ━━━ Check 3 — Tailscale ━━━ +# ============================================================================================== +if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]]; then + if ! command -v tailscale >/dev/null 2>&1; then + log "Tailscale not installed — skipping" + elif tailscale status --json 2>/dev/null | grep -q '"BackendState":"Running"'; then + log "$ICON_SUCCESS Tailscale running" + else + warn "$ICON_ERROR Tailscale not in Running state" + (( ISSUES++ )) + notify "Tailscale not running on $(hostname) ($MY_ID) — manual check needed" \ + "Network Watchdog" "warning" + fi +fi + +# ============================================================================================== +# ━━━ Check 4 — NPM Proxy (external) ━━━ +# ============================================================================================== +if [[ -n "$NPM_URL" ]]; then + NPM_STRIKES=$(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}") + NPM_STRIKE_LIMIT="${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}" + + if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then + log "$ICON_SUCCESS NPM proxy reachable — $NPM_URL" + if [[ "$NPM_STRIKES" -gt 0 ]]; then + log "NPM strikes cleared (was $NPM_STRIKES)" + set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}" + fi + else + NPM_STRIKES=$(( NPM_STRIKES + 1 )) + set_strikes "npm" "$NPM_STRIKES" "${NETWORK_WATCHDOG_NPM_STATE_FILE}" + (( ISSUES++ )) + + if [[ "$NPM_STRIKES" -lt "$NPM_STRIKE_LIMIT" ]]; then + warn "$ICON_ERROR NPM proxy not reachable — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)" + notify "NPM proxy not reachable on $(hostname) ($MY_ID) — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)" \ + "Network Watchdog" "warning" + else + warn "$ICON_ERROR NPM proxy strike limit reached ($NPM_STRIKES) — restarting NginxProxyManager" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart NginxProxyManager" + else + if docker restart NginxProxyManager >/dev/null 2>&1; then + warn "NginxProxyManager restarted ✅" + set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}" + notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \ + "Network Watchdog" "warning" + else + warn "NginxProxyManager restart failed — manual intervention needed" + notify "NPM proxy restart FAILED on $(hostname) ($MY_ID) — manual intervention needed" \ + "Network Watchdog" "warning" + fi + fi + fi + fi +else + log "NPM check not configured for $MY_ID — skipping" +fi + +# ============================================================================================== +# ━━━ Exit ━━━ +# ============================================================================================== +[[ "$ISSUES" -gt 0 ]] && exit 1 +exit 0 diff --git a/Watchdogs/System/storage_watchdog.sh b/Watchdogs/System/storage_watchdog.sh index b1e5996..6f7053e 100755 --- a/Watchdogs/System/storage_watchdog.sh +++ b/Watchdogs/System/storage_watchdog.sh @@ -160,6 +160,17 @@ detect_hosts touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null +# ━━━ Strike helpers ━━━ +get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; } +set_strikes() { + local key="$1" count="$2" file="$3" + if grep -q "^${key}:" "$file" 2>/dev/null; then + sed -i "s|^${key}:.*|${key}:${count}|" "$file" + else + echo "${key}:${count}" >> "$file" + fi +} + # ============================================================================================== # ━━━ Status ━━━ # ==============================================================================================