Say fallback, not failover — the synonym was silently matching nothing in three renderers, plus retrying probes and a 2-minute handback window
This commit is contained in:
Gmer4Lfe
@@ -95,7 +95,7 @@
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover, conf sync.
|
||||
# SSH key used for all server-to-server operations — rsync, fallback, conf sync.
|
||||
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
|
||||
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
|
||||
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
|
||||
@@ -487,6 +487,8 @@
|
||||
HOSTN_RADARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated anime root
|
||||
HOSTN_RADARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Radarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Movies") — blank disables the download orphan cleaner for Radarr
|
||||
HOSTN_RADARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Radarr's container sees it (e.g. "/downloads/Completed/Movies") — needed to trigger import scans on held folders
|
||||
HOSTN_LIDARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Lidarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Music") — blank disables the download orphan cleaner for Lidarr
|
||||
HOSTN_LIDARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Lidarr's container sees it (e.g. "/downloads/Completed/Music") — needed to trigger import scans on held folders
|
||||
|
||||
declare -A HOSTN_RADARR_PATH_MAP=(
|
||||
# ["/movies"]="/mnt/user/Movies"
|
||||
|
||||
@@ -372,7 +372,7 @@
|
||||
# Scripts launched by array_started.sh when the array comes online.
|
||||
# Launched in order — each as a background process.
|
||||
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||||
# Continuous scripts (failover) run until array stops.
|
||||
# Continuous scripts (fallback) run until array stops.
|
||||
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
|
||||
# watchdog_orchestrator.sh — NOT launched here.
|
||||
ARRAY_START_SCRIPTS=(
|
||||
@@ -490,9 +490,9 @@
|
||||
"Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files
|
||||
"Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files
|
||||
# Daily, not weekly — DOWNLOAD_ORPHAN_AGE gates each folder on its own mtime, so
|
||||
# frequency costs nothing in safety, but a weekly run clears a whole week of
|
||||
# newly-eligible orphans at once and can exceed DOWNLOAD_ORPHAN_MAX_DELETE_GB, which
|
||||
# aborts the pass entirely and rolls the backlog into an even larger next run.
|
||||
# frequency costs nothing in safety, while a weekly run makes each pass a whole
|
||||
# week of newly-eligible orphans at once and far more likely to spill past
|
||||
# DOWNLOAD_ORPHAN_MAX_DELETE_GB into a deferred remainder.
|
||||
"Arrs_Stack/arr_download_orphan_cleaner.sh" # sweep orphaned completed downloads out of the SAB Completed folders — deletes junk + already-imported leftovers, triggers import scans for genuinely-missing content
|
||||
# Runs after cleanup, not before — frees disk space from orphans/junk first, so the
|
||||
# searches triggered below (for relocated hasFile=false/episodeFileCount=0 entries)
|
||||
@@ -848,7 +848,23 @@
|
||||
|
||||
EXTERNAL_IP="8.8.8.8"
|
||||
FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
|
||||
FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s)
|
||||
# Cost measured 2026-08-22: ~15ms CPU and 784 bytes per cycle
|
||||
# per host (~2 MB/day, half of it across the mesh). The 2s of
|
||||
# wall time per cycle is ping waiting, not work.
|
||||
FALLBACK_HANDBACK_STRIKES=4 # consecutive healthy checks before initiating handback
|
||||
# 4×30s = 2min. Raising this costs nothing — it requires more
|
||||
# of the checks that already happen, it does not add any.
|
||||
# Any failed check resets the counter, so the window is a
|
||||
# genuine "stayed up this long", not "was up this often".
|
||||
# Entry is deliberately unguarded — one false from ping_remote enters FALLBACK, with no strike
|
||||
# count, because every second of a real outage is downtime and DDNS TTL is racing it. The
|
||||
# confidence comes from the probe itself instead: see FALLBACK_PROBE_* below and _ping_retry()
|
||||
# in common.sh.
|
||||
FALLBACK_PROBE_RETRIES=3 # reachability attempts before declaring a host down
|
||||
FALLBACK_PROBE_RETRY_DELAY=2 # seconds between those attempts
|
||||
# Success returns on the first try, so a healthy cycle costs
|
||||
# nothing extra. Only a failure pays: 3×4s + 2×2s = 16s worst
|
||||
# case, which must stay under FALLBACK_CHECK_INTERVAL.
|
||||
FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
|
||||
FALLBACK_ENABLED=false # set true once both servers are configured and paired
|
||||
# false = suppresses "not running" warnings in status scripts
|
||||
@@ -1238,7 +1254,7 @@
|
||||
LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db"
|
||||
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
|
||||
ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr
|
||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma" "wv" "ape" "aiff" "aif" "alac" "dsf" "dff" "mpc" "tta")
|
||||
LIDARR_PROTECTED_PATTERNS=(
|
||||
# Metadata
|
||||
"*.nfo" "*.tbn"
|
||||
@@ -1362,10 +1378,17 @@
|
||||
# Per-host dirs: HOST*_SONARR_DOWNLOAD_DIR / HOST*_RADARR_DOWNLOAD_DIR (+ _CONTAINER_DIR).
|
||||
DOWNLOAD_ORPHAN_CLEANER_ENABLED=true
|
||||
DOWNLOAD_ORPHAN_AGE=7 # days — entries younger than this may be mid-import, never touched
|
||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives)
|
||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # abort delete pass over this — a partial queue fetch would classify
|
||||
# live downloads as orphans, and a huge total is that failure's symptom;
|
||||
# --i-know-what-im-doing overrides for known backlogs
|
||||
DOWNLOAD_ORPHAN_KEEP_MARKER=".vv-keep" # a file by this name inside a download folder pins it —
|
||||
# never classified, never deleted. For lossless rips the
|
||||
# library holds only as MP3, which REDUNDANT would sweep
|
||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives) — Sonarr/Radarr
|
||||
DOWNLOAD_ORPHAN_MIN_AUDIO_MB=2 # same test for Lidarr — a 50M floor would mark every album folder
|
||||
# as junk, since single tracks rarely reach it
|
||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # per-run delete budget — a backlog above this drains
|
||||
# safest-first (JUNK, REDUNDANT, then UNMATCHED) up to
|
||||
# the budget and defers the rest to the next run.
|
||||
# Live downloads are protected by DOWNLOAD_ORPHAN_AGE,
|
||||
# not by this; --i-know-what-im-doing clears it in one pass
|
||||
|
||||
# ━━━ Arr Content Classification (radarr/sonarr_classification_scan.sh) ━━━
|
||||
#
|
||||
|
||||
@@ -249,7 +249,7 @@ const VV_SCRIPT_CONF_SECTIONS = [
|
||||
'Arrs_Stack/arrs_failed_stalled_recovery.sh' => ['Arr Failed/Stalled Recovery', 'Arr Recovery Toggles'],
|
||||
'Media/play_state_sync.sh' => ['Play State Sync', 'Play State Sync — Handback'],
|
||||
'Orchestrators/sunday_morning_coffee_report.sh' => ['Sunday Morning Coffee Report'],
|
||||
'Fallback/fallback_test.sh' => ['Failover Test'],
|
||||
'Fallback/fallback_test.sh' => ['Fallback Test'],
|
||||
'Rsync/rsync.sh' => ['Rsync Enable/Disable', 'Rsync Defaults',
|
||||
'Rsync Profile System', 'Rsync Merge Auto-Promote'],
|
||||
'System_Essentials/server_reboot.sh' => ['Reboot', 'Emby'],
|
||||
|
||||
@@ -202,7 +202,9 @@ function vv_fallback_state(): array {
|
||||
}
|
||||
return vv_fb_card_extra() + [
|
||||
'state' => $raw['state'] ?? 'UNKNOWN',
|
||||
'failover_start' => $raw['failover_start'] ?? '0',
|
||||
// fallback_start, not failover_start. The state file has only ever written the former, so this
|
||||
// read returned '0' every time and the Monitor card's outage duration never rendered.
|
||||
'fallback_start' => $raw['fallback_start'] ?? '0',
|
||||
'tier2_started' => $raw['tier2_started'] ?? 'false',
|
||||
'tier3_started' => $raw['tier3_started'] ?? 'false',
|
||||
'tier4_started' => $raw['tier4_started'] ?? 'false',
|
||||
|
||||
@@ -223,7 +223,7 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
||||
<div class="vv-fb-ctrl-row">
|
||||
<div>
|
||||
<div class="vv-fb-ctrl-lbl">Fallback</div>
|
||||
<div class="vv-fb-ctrl-sub">Mutual container failover between nodes</div>
|
||||
<div class="vv-fb-ctrl-sub">Mutual container fallback between nodes</div>
|
||||
</div>
|
||||
<div class="vv-fb-tog" id="vv-fb-en-tog" onclick="vvFbToggle(this,'FALLBACK_ENABLED')"></div>
|
||||
</div>
|
||||
@@ -680,7 +680,7 @@ function _verdict(data) {
|
||||
if (!data.partnership_enabled)
|
||||
return void (el.innerHTML = '<span style="color:#ef5350;">PARTNERSHIP_ENABLED=false</span> — all cross-server operations are disabled.');
|
||||
if (!data.fb_enabled)
|
||||
return void (el.innerHTML = '<span style="color:#ffb74d;">FALLBACK_ENABLED=false</span> — nothing will fail over. Arming it takes effect at the next array start.');
|
||||
return void (el.innerHTML = '<span style="color:#ffb74d;">FALLBACK_ENABLED=false</span> — nothing will fall back. Arming it takes effect at the next array start.');
|
||||
|
||||
const inFb = nodes.filter(n => (n.state?.state) === 'FALLBACK');
|
||||
const live = nodes.filter(n => n.proc?.running && n.proc.mode === 'live');
|
||||
@@ -699,7 +699,7 @@ function _verdict(data) {
|
||||
: `<span style="color:#4caf50;">Armed</span> · ${up}/${nodes.length} node${nodes.length!==1?'s':''} running`;
|
||||
|
||||
if (dry.length && !live.length)
|
||||
s += ` — <span style="color:#4a9eff;">all previewing (dry run), nothing would actually fail over</span>`;
|
||||
s += ` — <span style="color:#4a9eff;">all previewing (dry run), nothing would actually fall back</span>`;
|
||||
else if (dry.length)
|
||||
s += ` · <span style="color:#4a9eff;">${dry.map(n=>n.id).join(', ')} previewing (dry run)</span>`;
|
||||
|
||||
@@ -898,7 +898,7 @@ window.vvFbSaveSettings = function() {
|
||||
};
|
||||
|
||||
|
||||
// ── Failover coverage picker ──────────────────────────────────────────────────
|
||||
// ── Fallback coverage picker ──────────────────────────────────────────────────
|
||||
// Holds the whole edit in memory and writes all four tiers in one POST. Per-row saves would
|
||||
// leave the four arrays briefly disagreeing, and fallback.sh reads them as a set.
|
||||
let _vvFbCov = null; // { containers:[], cover:{name:tier}, missing:[] }
|
||||
|
||||
@@ -1204,7 +1204,7 @@ function vvPollMonitor(live) {
|
||||
// ── State label + outage duration ──────────────────────────────────────
|
||||
let stateExtra = '';
|
||||
if (state === 'FALLBACK' || state === 'DARK') {
|
||||
const start = parseInt(fb.failover_start ?? 0);
|
||||
const start = parseInt(fb.fallback_start ?? 0);
|
||||
if (start > 0) {
|
||||
const sec = Math.floor(Date.now() / 1000) - start;
|
||||
stateExtra = ` · ${Math.floor(sec/3600)}h ${Math.floor((sec%3600)/60)}m`;
|
||||
@@ -1267,7 +1267,7 @@ function vvPollMonitor(live) {
|
||||
</div>`;
|
||||
}
|
||||
|
||||
// ── Tier badges (FAILOVER only) ────────────────────────────────────────
|
||||
// ── Tier badges (FALLBACK only) ────────────────────────────────────────
|
||||
if (state === 'FALLBACK') {
|
||||
const t2 = fb.tier2_started === 'true';
|
||||
const t3 = fb.tier3_started === 'true';
|
||||
|
||||
@@ -710,14 +710,16 @@ function _renderConfig(cfg) {
|
||||
|
||||
const fbColors = {
|
||||
NORMAL: '#4caf50',
|
||||
FAILOVER: '#f44336',
|
||||
// FALLBACK — fallback.sh writes NORMAL|FALLBACK|NO_INTERNET|DARK and never 'FAILOVER', so
|
||||
// this entry matched nothing and a covering host rendered as UNKNOWN grey.
|
||||
FALLBACK: '#f44336',
|
||||
NO_INTERNET: '#ff9800',
|
||||
DARK: '#9e9e9e',
|
||||
UNKNOWN: '#444',
|
||||
};
|
||||
const fbLabels = {
|
||||
NORMAL: '✓ Nominal',
|
||||
FAILOVER: '⚠ Failover',
|
||||
FALLBACK: '⚠ Fallback active',
|
||||
NO_INTERNET: '⚡ No internet',
|
||||
DARK: '◌ Dark mode',
|
||||
UNKNOWN: '— Unknown',
|
||||
|
||||
@@ -679,7 +679,7 @@ $runningScripts = array_unique($runningScripts);
|
||||
<div class="vv-sug-body">
|
||||
<pre class="vv-readme-body">Not enterprise HA. No SLAs, no quorum nodes, no guaranteed uptime — don't put your billing system on this.
|
||||
|
||||
What it is: mutual automatic failover between two independent unRAID servers. When one goes down the other starts its containers, cuts over DNS, and keeps users online. When it comes back, everything hands back in the correct sequence — DDNS stops, containers stop, rsync writeback runs, containers start on the primary, primary DDNS starts last — so users hit the returning server only after it's actually ready.
|
||||
What it is: mutual automatic fallback between two independent unRAID servers. When one goes down the other starts its containers, cuts over DNS, and keeps users online. When it comes back, everything hands back in the correct sequence — DDNS stops, containers stop, rsync writeback runs, containers start on the primary, primary DDNS starts last — so users hit the returning server only after it's actually ready.
|
||||
|
||||
Built from scratch. Refined through a year of production testing. The DDNS sequencing and handback order were the hardest parts to get right. Both directions are exercised regularly with fallback_test.sh.</pre>
|
||||
</div>
|
||||
@@ -4967,7 +4967,7 @@ function vvPollSnapshot() {
|
||||
const s = (d.fallback ?? 'UNKNOWN').toUpperCase();
|
||||
fb.textContent = s;
|
||||
fb.style.color = s === 'NORMAL' ? '#4caf50'
|
||||
: s === 'FAILOVER' ? '#f44336'
|
||||
: s === 'FALLBACK' ? '#f44336' // never 'FAILOVER' — see fallback.sh state_set
|
||||
: s === 'NO_INTERNET' ? '#ff9800' : '#666';
|
||||
}
|
||||
const pt = document.getElementById('vv-snap-partner');
|
||||
|
||||
@@ -1078,16 +1078,43 @@ check_connectivity() {
|
||||
info "$ICON_PING $REMOTE_SERVER_NAME is reachable"
|
||||
}
|
||||
|
||||
# Non-fatal ping — used by fallback.sh which handles its own state machine.
|
||||
# Returns 0 if reachable, 1 if not — does NOT exit.
|
||||
# Non-fatal reachability checks — used by fallback.sh, fallback_test.sh, conf_cache_watchdog.sh.
|
||||
# Return 0 if reachable, 1 if not. Never exit.
|
||||
#
|
||||
# Retried, because a single two-packet ping is too thin a basis for the decision it feeds.
|
||||
# fallback.sh enters FALLBACK on ONE false from ping_remote — no strike count, by design, since
|
||||
# every second of a real outage is downtime — so one dropped pair of ICMP packets performed a
|
||||
# full failover: Tier 1 containers up, DDNS moved, operator notified, then a staged handback to
|
||||
# undo it. Observed 2026-08-22 04:20 on a dry run: HOST2 had 3 weeks uptime and never missed a
|
||||
# beat in its own loop, while this host declared it down and recovered 35 seconds later.
|
||||
#
|
||||
# Success returns immediately, so the healthy path — which is every cycle but the rare one —
|
||||
# costs exactly what it did before. Only a failure pays for the retries.
|
||||
#
|
||||
# worst case = tries × (ping -c2 -W3 ≈ 4s) + (tries-1) × retry delay
|
||||
# at the defaults: 3 × 4s + 2 × 2s = 16s, comfortably inside FALLBACK_CHECK_INTERVAL=30
|
||||
#
|
||||
# Raising retries past that budget would let one cycle overrun the next; the loop sleeps AFTER
|
||||
# its work, so it would stretch the interval rather than overlap, but the detection latency is
|
||||
# what the DDNS TTL is racing.
|
||||
_ping_retry() {
|
||||
local target="$1"
|
||||
local tries="${FALLBACK_PROBE_RETRIES:-3}"
|
||||
local gap="${FALLBACK_PROBE_RETRY_DELAY:-2}"
|
||||
local i
|
||||
for (( i = 1; i <= tries; i++ )); do
|
||||
ping -c2 -W3 "$target" &>/dev/null && return 0
|
||||
[[ "$i" -lt "$tries" ]] && sleep "$gap"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
ping_remote() {
|
||||
ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null
|
||||
_ping_retry "$REMOTE_SERVER"
|
||||
}
|
||||
|
||||
# Non-fatal external connectivity check — used by fallback.sh.
|
||||
# Returns 0 if internet reachable, 1 if not — does NOT exit.
|
||||
ping_internet() {
|
||||
ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null
|
||||
_ping_retry "${EXTERNAL_IP:-8.8.8.8}"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user