Say fallback, not failover — the synonym was silently matching nothing in three renderers, plus retrying probes and a 2-minute handback window

This commit is contained in:
Gmer4Lfe
2026-08-22 13:43:30 -04:00
parent f1ba418aa8
commit ac17be2cd9
9 changed files with 85 additions and 29 deletions
+33 -6
View File
@@ -1078,16 +1078,43 @@ check_connectivity() {
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
}
# Non-fatal ping — used by fallback.sh which handles its own state machine.
# Returns 0 if reachable, 1 if not — does NOT exit.
# Non-fatal reachability checks — used by fallback.sh, fallback_test.sh, conf_cache_watchdog.sh.
# Return 0 if reachable, 1 if not. Never exit.
#
# Retried, because a single two-packet ping is too thin a basis for the decision it feeds.
# fallback.sh enters FALLBACK on ONE false from ping_remote — no strike count, by design, since
# every second of a real outage is downtime — so one dropped pair of ICMP packets performed a
# full failover: Tier 1 containers up, DDNS moved, operator notified, then a staged handback to
# undo it. Observed 2026-08-22 04:20 on a dry run: HOST2 had 3 weeks uptime and never missed a
# beat in its own loop, while this host declared it down and recovered 35 seconds later.
#
# Success returns immediately, so the healthy path — which is every cycle but the rare one —
# costs exactly what it did before. Only a failure pays for the retries.
#
# worst case = tries × (ping -c2 -W3 ≈ 4s) + (tries-1) × retry delay
# at the defaults: 3 × 4s + 2 × 2s = 16s, comfortably inside FALLBACK_CHECK_INTERVAL=30
#
# Raising retries past that budget would let one cycle overrun the next; the loop sleeps AFTER
# its work, so it would stretch the interval rather than overlap, but the detection latency is
# what the DDNS TTL is racing.
_ping_retry() {
local target="$1"
local tries="${FALLBACK_PROBE_RETRIES:-3}"
local gap="${FALLBACK_PROBE_RETRY_DELAY:-2}"
local i
for (( i = 1; i <= tries; i++ )); do
ping -c2 -W3 "$target" &>/dev/null && return 0
[[ "$i" -lt "$tries" ]] && sleep "$gap"
done
return 1
}
ping_remote() {
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
_ping_retry "$REMOTE_SERVER"
}
# Non-fatal external connectivity check — used by fallback.sh.
# Returns 0 if internet reachable, 1 if not — does NOT exit.
ping_internet() {
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
_ping_retry "${EXTERNAL_IP:-8.8.8.8}"
}
# ==============================================================================================