Say fallback, not failover — the synonym was silently matching nothing in three renderers, plus retrying probes and a 2-minute handback window
This commit is contained in:
@@ -1078,16 +1078,43 @@ check_connectivity() {
|
||||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
||||
}
|
||||
|
||||
# Non-fatal ping — used by fallback.sh which handles its own state machine.
|
||||
# Returns 0 if reachable, 1 if not — does NOT exit.
|
||||
# Non-fatal reachability checks — used by fallback.sh, fallback_test.sh, conf_cache_watchdog.sh.
|
||||
# Return 0 if reachable, 1 if not. Never exit.
|
||||
#
|
||||
# Retried, because a single two-packet ping is too thin a basis for the decision it feeds.
|
||||
# fallback.sh enters FALLBACK on ONE false from ping_remote — no strike count, by design, since
|
||||
# every second of a real outage is downtime — so one dropped pair of ICMP packets performed a
|
||||
# full failover: Tier 1 containers up, DDNS moved, operator notified, then a staged handback to
|
||||
# undo it. Observed 2026-08-22 04:20 on a dry run: HOST2 had 3 weeks uptime and never missed a
|
||||
# beat in its own loop, while this host declared it down and recovered 35 seconds later.
|
||||
#
|
||||
# Success returns immediately, so the healthy path — which is every cycle but the rare one —
|
||||
# costs exactly what it did before. Only a failure pays for the retries.
|
||||
#
|
||||
# worst case = tries × (ping -c2 -W3 ≈ 4s) + (tries-1) × retry delay
|
||||
# at the defaults: 3 × 4s + 2 × 2s = 16s, comfortably inside FALLBACK_CHECK_INTERVAL=30
|
||||
#
|
||||
# Raising retries past that budget would let one cycle overrun the next; the loop sleeps AFTER
|
||||
# its work, so it would stretch the interval rather than overlap, but the detection latency is
|
||||
# what the DDNS TTL is racing.
|
||||
_ping_retry() {
|
||||
local target="$1"
|
||||
local tries="${FALLBACK_PROBE_RETRIES:-3}"
|
||||
local gap="${FALLBACK_PROBE_RETRY_DELAY:-2}"
|
||||
local i
|
||||
for (( i = 1; i <= tries; i++ )); do
|
||||
ping -c2 -W3 "$target" &>/dev/null && return 0
|
||||
[[ "$i" -lt "$tries" ]] && sleep "$gap"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
ping_remote() {
|
||||
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
|
||||
_ping_retry "$REMOTE_SERVER"
|
||||
}
|
||||
|
||||
# Non-fatal external connectivity check — used by fallback.sh.
|
||||
# Returns 0 if internet reachable, 1 if not — does NOT exit.
|
||||
ping_internet() {
|
||||
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
|
||||
_ping_retry "${EXTERNAL_IP:-8.8.8.8}"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user