Add System/network_watchdog.sh — services-layer connectivity watchdog
Checks: 1. Internet reachability (gates all remaining checks) 2. DDNS sync — public IP vs Cloudflare DNS record, restarts Gmer4Lfe.com container on mismatch 3. Tailscale status — notify only, no auto-restart 4. NPM proxy — external curl to https://gmer4lfe.com, 2-strike system before NginxProxyManager restart Config: master.conf NETWORK_WATCHDOG_* block, host1.conf HOST1_NETWORK_WATCHDOG_* values Added to SYSTEM_WATCHDOG_SCRIPTS — called by system_watchdog.sh each cycle Fix: storage_watchdog.sh was calling get_strikes/set_strikes without defining them — added local definitions (same pattern as docker_watchdog.sh and stability_watchdog.sh)
This commit is contained in:
Gmer4Lfe
Executable
+315
@@ -0,0 +1,315 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Network Watchdog ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Services-layer connectivity — checks that the outside world can actually reach
|
||||
# what it needs to reach. Silent when everything is reachable. Only fires when
|
||||
# something in the connectivity chain has broken.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Check 1 — Internet Connectivity
|
||||
# curl external endpoint → fail = alert + skip all remaining checks.
|
||||
# Internet down means DDNS and NPM checks would false-positive — gating prevents noise.
|
||||
#
|
||||
# Check 2 — DDNS (Cloudflare)
|
||||
# Public IP via ifconfig.me vs DNS record via dig @1.1.1.1.
|
||||
# Match → pass (silent). Mismatch → restart DDNS container + notify.
|
||||
# Container restart triggers an immediate Cloudflare record update.
|
||||
#
|
||||
# Check 3 — Tailscale
|
||||
# tailscale status → Running → pass (silent). Not running → notify.
|
||||
# Notify only — no restart attempt. Tailscale state issues warrant human review.
|
||||
#
|
||||
# Check 4 — NPM Proxy (external check)
|
||||
# curl external URL → 2-strike system before restarting NginxProxyManager.
|
||||
# Strike 1: warn + notify. Strike 2: restart NPM + notify + clear strikes.
|
||||
# Strikes auto-clear when the external URL becomes reachable again.
|
||||
# External check — verifies the full stack (DNS → NPM → backend), not just NPM running.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# docker restart requires root.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs.
|
||||
#
|
||||
# Internet Gates All Checks
|
||||
# If internet is down, DDNS and NPM checks are skipped — no cascade of false positives.
|
||||
#
|
||||
# Strike Before Acting on NPM
|
||||
# Single curl failure could be transient DNS hiccup or CDN blip.
|
||||
# Two consecutive failures confirms NPM is the problem.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# NETWORK_WATCHDOG_ENABLED toggle entire watchdog (default: true)
|
||||
# NETWORK_WATCHDOG_INTERNET_URL endpoint for internet reachability check
|
||||
# NETWORK_WATCHDOG_INTERNET_TIMEOUT curl timeout in seconds for internet check
|
||||
# NETWORK_WATCHDOG_CHECK_TAILSCALE toggle tailscale check (default: true)
|
||||
# NETWORK_WATCHDOG_NPM_TIMEOUT curl timeout for NPM external check
|
||||
# NETWORK_WATCHDOG_NPM_STRIKE_LIMIT consecutive failures before NPM restart
|
||||
# NETWORK_WATCHDOG_NPM_STATE_FILE strike count persistence (/tmp — resets on reboot)
|
||||
#
|
||||
# host*.conf (host-specific)
|
||||
#
|
||||
# HOST*_NETWORK_WATCHDOG_DDNS_DOMAIN domain to resolve and compare to public IP
|
||||
# HOST*_NETWORK_WATCHDOG_DDNS_CONTAINER container to restart on DDNS mismatch
|
||||
# HOST*_NETWORK_WATCHDOG_NPM_URL external URL to test full proxy stack
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# network_watchdog.sh
|
||||
# Run all connectivity checks. Silent when healthy.
|
||||
#
|
||||
# network_watchdog.sh --dry-run
|
||||
# Run all checks without restarting any containers.
|
||||
#
|
||||
# network_watchdog.sh --status
|
||||
# Show configuration, current public IP, DNS record, NPM strike state.
|
||||
#
|
||||
# network_watchdog.sh --log
|
||||
# Verbose output — show each check result even when passing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Resolve host-specific config ━━━
|
||||
# ==============================================================================================
|
||||
_ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN"
|
||||
_ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER"
|
||||
_npm_url_var="${MY_ID}_NETWORK_WATCHDOG_NPM_URL"
|
||||
|
||||
DDNS_DOMAIN="${!_ddns_domain_var:-}"
|
||||
DDNS_CONTAINER="${!_ddns_container_var:-}"
|
||||
NPM_URL="${!_npm_url_var:-}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "${NETWORK_WATCHDOG_ENABLED:-true}" != "true" ]] && log "Network watchdog disabled" && exit 0
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
touch "${NETWORK_WATCHDOG_NPM_STATE_FILE}" 2>/dev/null
|
||||
|
||||
# ━━━ Strike helpers ━━━
|
||||
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
|
||||
set_strikes() {
|
||||
local key="$1" count="$2" file="$3"
|
||||
if grep -q "^${key}:" "$file" 2>/dev/null; then
|
||||
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
|
||||
else
|
||||
echo "${key}:${count}" >> "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY NETWORK WATCHDOG STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Internet URL: ${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}"
|
||||
echo "$ICON_GEAR DDNS domain: ${DDNS_DOMAIN:-not configured}"
|
||||
echo "$ICON_GEAR DDNS container: ${DDNS_CONTAINER:-not configured}"
|
||||
echo "$ICON_GEAR Tailscale: ${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}"
|
||||
echo "$ICON_GEAR NPM URL: ${NPM_URL:-not configured}"
|
||||
echo "$ICON_GEAR NPM strikes: $(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}") / ${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||
echo ""
|
||||
echo "── Current State ──"
|
||||
|
||||
if curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \
|
||||
"${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then
|
||||
echo " $ICON_SUCCESS Internet: reachable"
|
||||
else
|
||||
echo " $ICON_ERROR Internet: NOT reachable"
|
||||
fi
|
||||
|
||||
if [[ -n "$DDNS_DOMAIN" ]]; then
|
||||
_pub=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
|
||||
_dns=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
echo " $ICON_GEAR Public IP: ${_pub:-unknown}"
|
||||
echo " $ICON_GEAR DNS record: ${_dns:-unknown}"
|
||||
[[ "$_pub" == "$_dns" ]] && \
|
||||
echo " $ICON_SUCCESS DDNS: in sync" || \
|
||||
echo " $ICON_ERROR DDNS: MISMATCH — public=$_pub dns=$_dns"
|
||||
else
|
||||
echo " $ICON_GEAR DDNS: not configured for $MY_ID"
|
||||
fi
|
||||
|
||||
if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]] && command -v tailscale >/dev/null 2>&1; then
|
||||
if tailscale status --json 2>/dev/null | grep -q '"BackendState":"Running"'; then
|
||||
echo " $ICON_SUCCESS Tailscale: running"
|
||||
else
|
||||
echo " $ICON_ERROR Tailscale: NOT running"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_GEAR Tailscale: check disabled or not installed"
|
||||
fi
|
||||
|
||||
if [[ -n "$NPM_URL" ]]; then
|
||||
if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
|
||||
echo " $ICON_SUCCESS NPM proxy: reachable ($NPM_URL)"
|
||||
else
|
||||
echo " $ICON_ERROR NPM proxy: NOT reachable ($NPM_URL)"
|
||||
fi
|
||||
else
|
||||
echo " $ICON_GEAR NPM proxy: not configured for $MY_ID"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 1 — Internet Connectivity ━━━
|
||||
# ==============================================================================================
|
||||
log "Network watchdog — $MY_ID — $(date '+%H:%M:%S')"
|
||||
|
||||
ISSUES=0
|
||||
|
||||
if ! curl -sf --max-time "${NETWORK_WATCHDOG_INTERNET_TIMEOUT:-5}" \
|
||||
"${NETWORK_WATCHDOG_INTERNET_URL:-https://1.1.1.1}" >/dev/null 2>&1; then
|
||||
warn "$ICON_ERROR Internet not reachable — skipping DDNS, Tailscale, and NPM checks"
|
||||
notify "Network watchdog: internet not reachable on $(hostname) ($MY_ID)" \
|
||||
"Network Watchdog" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "$ICON_SUCCESS Internet reachable"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 2 — DDNS ━━━
|
||||
# ==============================================================================================
|
||||
if [[ -n "$DDNS_DOMAIN" ]] && [[ -n "$DDNS_CONTAINER" ]]; then
|
||||
PUBLIC_IP=$(curl -sf --max-time 5 https://ifconfig.me 2>/dev/null | tr -d '[:space:]')
|
||||
DNS_IP=$(dig +short "$DDNS_DOMAIN" @1.1.1.1 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
||||
|
||||
if [[ -z "$PUBLIC_IP" ]]; then
|
||||
warn "Could not determine public IP — skipping DDNS check"
|
||||
elif [[ -z "$DNS_IP" ]]; then
|
||||
warn "Could not resolve $DDNS_DOMAIN — skipping DDNS check"
|
||||
elif [[ "$PUBLIC_IP" == "$DNS_IP" ]]; then
|
||||
log "$ICON_SUCCESS DDNS in sync — $DDNS_DOMAIN → $DNS_IP"
|
||||
else
|
||||
warn "$ICON_ERROR DDNS mismatch — public=$PUBLIC_IP dns=$DNS_IP"
|
||||
(( ISSUES++ ))
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $DDNS_CONTAINER"
|
||||
else
|
||||
warn "Restarting $DDNS_CONTAINER to trigger Cloudflare update..."
|
||||
if docker restart "$DDNS_CONTAINER" >/dev/null 2>&1; then
|
||||
warn "$DDNS_CONTAINER restarted ✅"
|
||||
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_DOMAIN was $DNS_IP, public is $PUBLIC_IP — $DDNS_CONTAINER restarted" \
|
||||
"Network Watchdog" "warning"
|
||||
else
|
||||
warn "$DDNS_CONTAINER restart failed"
|
||||
notify "DDNS mismatch on $(hostname) ($MY_ID) — $DDNS_CONTAINER restart failed — manual intervention needed" \
|
||||
"Network Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "DDNS check not configured for $MY_ID — skipping"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 3 — Tailscale ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${NETWORK_WATCHDOG_CHECK_TAILSCALE:-true}" == "true" ]]; then
|
||||
if ! command -v tailscale >/dev/null 2>&1; then
|
||||
log "Tailscale not installed — skipping"
|
||||
elif tailscale status --json 2>/dev/null | grep -q '"BackendState":"Running"'; then
|
||||
log "$ICON_SUCCESS Tailscale running"
|
||||
else
|
||||
warn "$ICON_ERROR Tailscale not in Running state"
|
||||
(( ISSUES++ ))
|
||||
notify "Tailscale not running on $(hostname) ($MY_ID) — manual check needed" \
|
||||
"Network Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Check 4 — NPM Proxy (external) ━━━
|
||||
# ==============================================================================================
|
||||
if [[ -n "$NPM_URL" ]]; then
|
||||
NPM_STRIKES=$(get_strikes "npm" "${NETWORK_WATCHDOG_NPM_STATE_FILE}")
|
||||
NPM_STRIKE_LIMIT="${NETWORK_WATCHDOG_NPM_STRIKE_LIMIT:-2}"
|
||||
|
||||
if curl -sf --max-time "${NETWORK_WATCHDOG_NPM_TIMEOUT:-10}" "$NPM_URL" >/dev/null 2>&1; then
|
||||
log "$ICON_SUCCESS NPM proxy reachable — $NPM_URL"
|
||||
if [[ "$NPM_STRIKES" -gt 0 ]]; then
|
||||
log "NPM strikes cleared (was $NPM_STRIKES)"
|
||||
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||
fi
|
||||
else
|
||||
NPM_STRIKES=$(( NPM_STRIKES + 1 ))
|
||||
set_strikes "npm" "$NPM_STRIKES" "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||
(( ISSUES++ ))
|
||||
|
||||
if [[ "$NPM_STRIKES" -lt "$NPM_STRIKE_LIMIT" ]]; then
|
||||
warn "$ICON_ERROR NPM proxy not reachable — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)"
|
||||
notify "NPM proxy not reachable on $(hostname) ($MY_ID) — $NPM_URL (strike $NPM_STRIKES/$NPM_STRIKE_LIMIT)" \
|
||||
"Network Watchdog" "warning"
|
||||
else
|
||||
warn "$ICON_ERROR NPM proxy strike limit reached ($NPM_STRIKES) — restarting NginxProxyManager"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart NginxProxyManager"
|
||||
else
|
||||
if docker restart NginxProxyManager >/dev/null 2>&1; then
|
||||
warn "NginxProxyManager restarted ✅"
|
||||
set_strikes "npm" 0 "${NETWORK_WATCHDOG_NPM_STATE_FILE}"
|
||||
notify "NPM proxy restarted on $(hostname) ($MY_ID) — $NPM_URL was unreachable for $NPM_STRIKES cycles" \
|
||||
"Network Watchdog" "warning"
|
||||
else
|
||||
warn "NginxProxyManager restart failed — manual intervention needed"
|
||||
notify "NPM proxy restart FAILED on $(hostname) ($MY_ID) — manual intervention needed" \
|
||||
"Network Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "NPM check not configured for $MY_ID — skipping"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
# ==============================================================================================
|
||||
[[ "$ISSUES" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -160,6 +160,17 @@ detect_hosts
|
||||
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
||||
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
||||
|
||||
# ━━━ Strike helpers ━━━
|
||||
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
|
||||
set_strikes() {
|
||||
local key="$1" count="$2" file="$3"
|
||||
if grep -q "^${key}:" "$file" 2>/dev/null; then
|
||||
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
|
||||
else
|
||||
echo "${key}:${count}" >> "$file"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user