Say fallback, not failover — the synonym was silently matching nothing in three renderers, plus retrying probes and a 2-minute handback window

This commit is contained in:
Gmer4Lfe
2026-08-22 13:43:30 -04:00
parent f1ba418aa8
commit ac17be2cd9
9 changed files with 85 additions and 29 deletions
+3 -1
View File
@@ -95,7 +95,7 @@
# ━━━ Identity ━━━
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
# SSH key used for all server-to-server operations — rsync, failover, conf sync.
# SSH key used for all server-to-server operations — rsync, fallback, conf sync.
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
@@ -487,6 +487,8 @@
HOSTN_RADARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated anime root
HOSTN_RADARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Radarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Movies") — blank disables the download orphan cleaner for Radarr
HOSTN_RADARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Radarr's container sees it (e.g. "/downloads/Completed/Movies") — needed to trigger import scans on held folders
HOSTN_LIDARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Lidarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Music") — blank disables the download orphan cleaner for Lidarr
HOSTN_LIDARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Lidarr's container sees it (e.g. "/downloads/Completed/Music") — needed to trigger import scans on held folders
declare -A HOSTN_RADARR_PATH_MAP=(
# ["/movies"]="/mnt/user/Movies"
+33 -10
View File
@@ -372,7 +372,7 @@
# Scripts launched by array_started.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
# Continuous scripts (failover) run until array stops.
# Continuous scripts (fallback) run until array stops.
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
# watchdog_orchestrator.sh — NOT launched here.
ARRAY_START_SCRIPTS=(
@@ -490,9 +490,9 @@
"Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files
"Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files
# Daily, not weekly — DOWNLOAD_ORPHAN_AGE gates each folder on its own mtime, so
# frequency costs nothing in safety, but a weekly run clears a whole week of
# newly-eligible orphans at once and can exceed DOWNLOAD_ORPHAN_MAX_DELETE_GB, which
# aborts the pass entirely and rolls the backlog into an even larger next run.
# frequency costs nothing in safety, while a weekly run makes each pass a whole
# week of newly-eligible orphans at once and far more likely to spill past
# DOWNLOAD_ORPHAN_MAX_DELETE_GB into a deferred remainder.
"Arrs_Stack/arr_download_orphan_cleaner.sh" # sweep orphaned completed downloads out of the SAB Completed folders — deletes junk + already-imported leftovers, triggers import scans for genuinely-missing content
# Runs after cleanup, not before — frees disk space from orphans/junk first, so the
# searches triggered below (for relocated hasFile=false/episodeFileCount=0 entries)
@@ -848,7 +848,23 @@
EXTERNAL_IP="8.8.8.8"
FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s)
# Cost measured 2026-08-22: ~15ms CPU and 784 bytes per cycle
# per host (~2 MB/day, half of it across the mesh). The 2s of
# wall time per cycle is ping waiting, not work.
FALLBACK_HANDBACK_STRIKES=4 # consecutive healthy checks before initiating handback
# 4×30s = 2min. Raising this costs nothing — it requires more
# of the checks that already happen, it does not add any.
# Any failed check resets the counter, so the window is a
# genuine "stayed up this long", not "was up this often".
# Entry is deliberately unguarded — one false from ping_remote enters FALLBACK, with no strike
# count, because every second of a real outage is downtime and DDNS TTL is racing it. The
# confidence comes from the probe itself instead: see FALLBACK_PROBE_* below and _ping_retry()
# in common.sh.
FALLBACK_PROBE_RETRIES=3 # reachability attempts before declaring a host down
FALLBACK_PROBE_RETRY_DELAY=2 # seconds between those attempts
# Success returns on the first try, so a healthy cycle costs
# nothing extra. Only a failure pays: 3×4s + 2×2s = 16s worst
# case, which must stay under FALLBACK_CHECK_INTERVAL.
FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
FALLBACK_ENABLED=false # set true once both servers are configured and paired
# false = suppresses "not running" warnings in status scripts
@@ -1238,7 +1254,7 @@
LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db"
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma" "wv" "ape" "aiff" "aif" "alac" "dsf" "dff" "mpc" "tta")
LIDARR_PROTECTED_PATTERNS=(
# Metadata
"*.nfo" "*.tbn"
@@ -1362,10 +1378,17 @@
# Per-host dirs: HOST*_SONARR_DOWNLOAD_DIR / HOST*_RADARR_DOWNLOAD_DIR (+ _CONTAINER_DIR).
DOWNLOAD_ORPHAN_CLEANER_ENABLED=true
DOWNLOAD_ORPHAN_AGE=7 # days — entries younger than this may be mid-import, never touched
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives)
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # abort delete pass over this — a partial queue fetch would classify
# live downloads as orphans, and a huge total is that failure's symptom;
# --i-know-what-im-doing overrides for known backlogs
DOWNLOAD_ORPHAN_KEEP_MARKER=".vv-keep" # a file by this name inside a download folder pins it —
# never classified, never deleted. For lossless rips the
# library holds only as MP3, which REDUNDANT would sweep
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives) — Sonarr/Radarr
DOWNLOAD_ORPHAN_MIN_AUDIO_MB=2 # same test for Lidarr — a 50M floor would mark every album folder
# as junk, since single tracks rarely reach it
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # per-run delete budget — a backlog above this drains
# safest-first (JUNK, REDUNDANT, then UNMATCHED) up to
# the budget and defers the rest to the next run.
# Live downloads are protected by DOWNLOAD_ORPHAN_AGE,
# not by this; --i-know-what-im-doing clears it in one pass
# ━━━ Arr Content Classification (radarr/sonarr_classification_scan.sh) ━━━
#