From cb7d7a688df27a29c7b9c9bafbb9f2b96449e83c Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 1 Jun 2026 20:04:17 -0400 Subject: [PATCH] =?UTF-8?q?rename:=20failover=20=E2=86=92=20fallback=20thr?= =?UTF-8?q?oughout=20codebase=20=E2=80=94=20this=20system=20is=20Fallback,?= =?UTF-8?q?=20state=20is=20FALLBACK;=20preserve=20conventional=20failover?= =?UTF-8?q?=20comparisons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Docker_Essentials/docker_daily_restart.sh | 0 Docker_Essentials/docker_update.sh | 6 +-- Docker_Essentials/docker_update_remaining.sh | 0 Docker_Essentials/docker_weekly_restart.sh | 0 Docker_Essentials/downloaders_reset.sh | 0 Fallback/Manual-Fallback.md | 8 ++-- Fallback/README-Fallback.md | 4 +- Fallback/fallback_test.sh | 0 Kernel/decision_engine.sh | 0 Manual.md | 4 +- Media/arrs_failed_stalled_recovery.sh | 0 Media/lidarr_cleanup.sh | 0 Media/lidarr_missing_art.sh | 0 Media/media_cleaner.sh | 0 Media/media_shares_permissions.sh | 0 Media/radarr_cleanup.sh | 0 Media/radarr_tmdb_removed.sh | 0 Media/sonarr_cleanup.sh | 0 Media/sonarr_tvdb_removed.sh | 0 Monitors/README-Monitors.md | 2 +- Monitors/backup_verify.sh | 0 Monitors/bandwidth_monitor.sh | 0 Monitors/cert_monitor.sh | 0 Monitors/emby_session_report.sh | 0 Monitors/smart_health.sh | 0 Monitors/system_tuning_monitor.sh | 0 Monitors/weekly_health_digest.sh | 2 +- Monitors/zfs_memory_snapshot.sh | 0 Notes_To-Do.md | 2 +- .../continuous_scripts_status.sh | 0 Orchestrators/README-Orchestrators.md | 6 +-- Orchestrators/critical_sync_maintenance.sh | 0 Orchestrators/daily_sync_maintenance.sh | 0 .../intermediate_sync_maintenance.sh | 0 Orchestrators/monthly_maintenance.sh | 0 Orchestrators/transcode_management.sh | 0 Orchestrators/weekly_sync_maintenance.sh | 0 Partnership/partnership_manager.sh | 6 +-- Plugin/unraid/run_job.sh | 0 README-User_Script_Plug-in.md | 48 +++++++++---------- README.md | 46 +++++++++--------- Rsync/Manual-Rsync.md | 4 +- Rsync/rsync.sh | 0 Tools/Manual-Tools.md | 2 +- Tools/README-Tools.md | 10 ++-- Tools/bulk_permissions_repair.sh | 0 Tools/container_data_export.sh | 0 Tools/emby_database_repair.sh | 0 Tools/emby_to_radarr_sync.sh | 0 Tools/emby_to_sonarr_sync.sh | 0 Tools/fallback_state_reset.sh | 8 ++-- Tools/recreate_shares.sh | 0 Tools/watchdog_skip_list_manager.sh | 0 Tools/zfs_pool_scrub.sh | 0 Transcodes/transcode_cleanup.sh | 0 Transcodes/transcode_manager.sh | 0 common.sh | 10 ++-- git_pull_execute.sh | 2 +- load_config.sh | 0 unRAID_Essentials/clear_logs.sh | 0 unRAID_Essentials/server_reboot.sh | 0 user_script_plug-in.sh | 0 62 files changed, 85 insertions(+), 85 deletions(-) mode change 100644 => 100755 Docker_Essentials/docker_daily_restart.sh mode change 100644 => 100755 Docker_Essentials/docker_update.sh mode change 100644 => 100755 Docker_Essentials/docker_update_remaining.sh mode change 100644 => 100755 Docker_Essentials/docker_weekly_restart.sh mode change 100644 => 100755 Docker_Essentials/downloaders_reset.sh mode change 100644 => 100755 Fallback/fallback_test.sh mode change 100644 => 100755 Kernel/decision_engine.sh mode change 100644 => 100755 Media/arrs_failed_stalled_recovery.sh mode change 100644 => 100755 Media/lidarr_cleanup.sh mode change 100644 => 100755 Media/lidarr_missing_art.sh mode change 100644 => 100755 Media/media_cleaner.sh mode change 100644 => 100755 Media/media_shares_permissions.sh mode change 100644 => 100755 Media/radarr_cleanup.sh mode change 100644 => 100755 Media/radarr_tmdb_removed.sh mode change 100644 => 100755 Media/sonarr_cleanup.sh mode change 100644 => 100755 Media/sonarr_tvdb_removed.sh mode change 100644 => 100755 Monitors/backup_verify.sh mode change 100644 => 100755 Monitors/bandwidth_monitor.sh mode change 100644 => 100755 Monitors/cert_monitor.sh mode change 100644 => 100755 Monitors/emby_session_report.sh mode change 100644 => 100755 Monitors/smart_health.sh mode change 100644 => 100755 Monitors/system_tuning_monitor.sh mode change 100644 => 100755 Monitors/weekly_health_digest.sh mode change 100644 => 100755 Monitors/zfs_memory_snapshot.sh mode change 100644 => 100755 Old_Arch_Still_Works/continuous_scripts_status.sh mode change 100644 => 100755 Orchestrators/critical_sync_maintenance.sh mode change 100644 => 100755 Orchestrators/daily_sync_maintenance.sh mode change 100644 => 100755 Orchestrators/intermediate_sync_maintenance.sh mode change 100644 => 100755 Orchestrators/monthly_maintenance.sh mode change 100644 => 100755 Orchestrators/transcode_management.sh mode change 100644 => 100755 Orchestrators/weekly_sync_maintenance.sh mode change 100644 => 100755 Partnership/partnership_manager.sh mode change 100644 => 100755 Plugin/unraid/run_job.sh mode change 100644 => 100755 Rsync/rsync.sh mode change 100644 => 100755 Tools/bulk_permissions_repair.sh mode change 100644 => 100755 Tools/container_data_export.sh mode change 100644 => 100755 Tools/emby_database_repair.sh mode change 100644 => 100755 Tools/emby_to_radarr_sync.sh mode change 100644 => 100755 Tools/emby_to_sonarr_sync.sh mode change 100644 => 100755 Tools/recreate_shares.sh mode change 100644 => 100755 Tools/watchdog_skip_list_manager.sh mode change 100644 => 100755 Tools/zfs_pool_scrub.sh mode change 100644 => 100755 Transcodes/transcode_cleanup.sh mode change 100644 => 100755 Transcodes/transcode_manager.sh mode change 100644 => 100755 load_config.sh mode change 100644 => 100755 unRAID_Essentials/clear_logs.sh mode change 100644 => 100755 unRAID_Essentials/server_reboot.sh mode change 100644 => 100755 user_script_plug-in.sh diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh old mode 100644 new mode 100755 diff --git a/Docker_Essentials/docker_update.sh b/Docker_Essentials/docker_update.sh old mode 100644 new mode 100755 index 2688648..0fb8647 --- a/Docker_Essentials/docker_update.sh +++ b/Docker_Essentials/docker_update.sh @@ -43,7 +43,7 @@ # # Version Ownership # Fallback containers are excluded from remainder mode. This server only runs -# them during a failover. The remote server owns their version — if remainder +# them during a fallback. The remote server owns their version — if remainder # updates them independently and a handback occurs, the remote's older image # may not handle data written by the newer version. # @@ -72,7 +72,7 @@ # # Fallback Exclusion # Remainder mode excludes containers owned by the remote server's update cycle -# to prevent version divergence across the failover boundary. +# to prevent version divergence across the fallback boundary. # # Running-Only Filter # Stopped containers excluded from remainder mode — intentionally down. @@ -179,7 +179,7 @@ if [[ "$REMAINDER_MODE" == true ]]; then unset _weekly_str _weekly_arr # Fallback coverage containers — owned by the remote server's update cycle. - # This server runs them during failover but should never update them independently. + # This server runs them during fallback but should never update them independently. # Updating them here risks version divergence: if remote's writeback after handback # encounters data written by a newer version, it may not handle it correctly. for _tier in 1 2 3 4; do diff --git a/Docker_Essentials/docker_update_remaining.sh b/Docker_Essentials/docker_update_remaining.sh old mode 100644 new mode 100755 diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh old mode 100644 new mode 100755 diff --git a/Docker_Essentials/downloaders_reset.sh b/Docker_Essentials/downloaders_reset.sh old mode 100644 new mode 100755 diff --git a/Fallback/Manual-Fallback.md b/Fallback/Manual-Fallback.md index 0c98791..74c2a08 100644 --- a/Fallback/Manual-Fallback.md +++ b/Fallback/Manual-Fallback.md @@ -352,7 +352,7 @@ ssh root@[HOST2-tailscale-ip] "docker inspect Emby --format '{{.State.Status}}'" ### 4. Critical Data Mirrored -These shares must exist on HOST2 with current data from HOST1 before failover is needed: +These shares must exist on HOST2 with current data from HOST1 before fallback is needed: ``` /mnt/user/appdata-Fallback/Critical-Data # auth stack — NPM, LLDAP, Authelia, certs @@ -371,7 +371,7 @@ Sync is maintained continuously by `daily_sync_maintenance.sh` critical-data pro ### 5. DDNS TTL Set to 1 Minute Set in your DDNS provider settings. Higher TTL means users continue hitting the old IP -for longer after failover. At 5-minute TTL, users can be hitting a downed server for up +for longer after fallback. At 5-minute TTL, users can be hitting a downed server for up to 5 minutes before DNS switches. ### 6. Both Servers Running fallback.sh @@ -398,7 +398,7 @@ current state, outage duration if not NORMAL, Tailscale reachability, and whethe fallback.sh is running. **Weekly health digest** (`weekly_health_digest.sh`) — reads the state file. If -`DIGEST_SMART_ON_FAILOVER=true` and state is not NORMAL, it sends a notification even +`DIGEST_SMART_ON_FALLBACK=true` and state is not NORMAL, it sends a notification even in smart mode — a non-NORMAL state at digest time needs attention. **Direct check:** @@ -412,7 +412,7 @@ cat /boot/config/fallback_state.db # raw state file ## ━━━ PROCEDURES ━━━ -### Running the Failover Test +### Running the Fallback Test > This starts and stops real containers on both servers. Users will experience a brief > service interruption. Always run `--dry-run` first. diff --git a/Fallback/README-Fallback.md b/Fallback/README-Fallback.md index d623bb0..eb9aced 100644 --- a/Fallback/README-Fallback.md +++ b/Fallback/README-Fallback.md @@ -1,6 +1,6 @@ # ━━━━━ FALLBACK ━━━━━ -Mutual automatic failover between two independent unRAID servers. When one goes down the +Mutual automatic fallback between two independent unRAID servers. When one goes down the other starts its containers, cuts over DNS, and keeps users online. When it comes back everything hands back in the correct sequence — covering DDNS stops, containers stop, rsync writeback runs, containers start on the primary, primary DDNS starts last — so users hit the @@ -45,7 +45,7 @@ The fix: stop containers before syncing. The outage window is only the rsync dur typically minutes. Clean static source at full bandwidth, predictable state every time. **No Way to Validate the System Before Needing It** -A failover system that has never been tested is not a failover system — it is a hope. +A fallback system that has never been tested is not a fallback system — it is a hope. The fix: `fallback_test.sh` — a controlled simulation using an iptables DROP rule to make the remote appear unreachable, triggering the full sequence without taking anything offline. A safety trap removes the rule on any exit — crash, error, ctrl-c, or clean completion. diff --git a/Fallback/fallback_test.sh b/Fallback/fallback_test.sh old mode 100644 new mode 100755 diff --git a/Kernel/decision_engine.sh b/Kernel/decision_engine.sh old mode 100644 new mode 100755 diff --git a/Manual.md b/Manual.md index f17e91d..b448a5e 100644 --- a/Manual.md +++ b/Manual.md @@ -415,7 +415,7 @@ cat /boot/config/fallback_state.db --- -## ━━━ STEP 8: TEST FAILOVER ━━━ +## ━━━ STEP 8: TEST FALLBACK ━━━ Before relying on the system, confirm it actually triggers. `fallback_test.sh` simulates an outage using `iptables` — no real downtime, no real data changes. @@ -601,7 +601,7 @@ The remote server is unreachable on the first check. Common causes: ```bash # Confirm which state fallback is in cat /boot/config/fallback_state.db -# If stuck in FAILOVER after remote comes back: reset state +# If stuck in FALLBACK after remote comes back: reset state bash Tools/fallback_state_reset.sh ``` diff --git a/Media/arrs_failed_stalled_recovery.sh b/Media/arrs_failed_stalled_recovery.sh old mode 100644 new mode 100755 diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh old mode 100644 new mode 100755 diff --git a/Media/lidarr_missing_art.sh b/Media/lidarr_missing_art.sh old mode 100644 new mode 100755 diff --git a/Media/media_cleaner.sh b/Media/media_cleaner.sh old mode 100644 new mode 100755 diff --git a/Media/media_shares_permissions.sh b/Media/media_shares_permissions.sh old mode 100644 new mode 100755 diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh old mode 100644 new mode 100755 diff --git a/Media/radarr_tmdb_removed.sh b/Media/radarr_tmdb_removed.sh old mode 100644 new mode 100755 diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh old mode 100644 new mode 100755 diff --git a/Media/sonarr_tvdb_removed.sh b/Media/sonarr_tvdb_removed.sh old mode 100644 new mode 100755 diff --git a/Monitors/README-Monitors.md b/Monitors/README-Monitors.md index 6088e2f..b3aaacb 100644 --- a/Monitors/README-Monitors.md +++ b/Monitors/README-Monitors.md @@ -83,7 +83,7 @@ drive SMART attributes, ZFS pool state, ARC statistics, kernel memory pressure. Monitors/ ← observes and reports (this folder) Docker_Essentials/ ← acts on containers (docker_watchdog starts/stops) unRAID_Essentials/ ← acts on the server (stability_watchdog, inotify_tuning) -Fallback/ ← acts on the full stack (failover, handback) +Fallback/ ← acts on the full stack (fallback, handback) Rsync/ ← calls bandwidth_monitor.sh (auto-logs each sync) ``` diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh old mode 100644 new mode 100755 diff --git a/Monitors/bandwidth_monitor.sh b/Monitors/bandwidth_monitor.sh old mode 100644 new mode 100755 diff --git a/Monitors/cert_monitor.sh b/Monitors/cert_monitor.sh old mode 100644 new mode 100755 diff --git a/Monitors/emby_session_report.sh b/Monitors/emby_session_report.sh old mode 100644 new mode 100755 diff --git a/Monitors/smart_health.sh b/Monitors/smart_health.sh old mode 100644 new mode 100755 diff --git a/Monitors/system_tuning_monitor.sh b/Monitors/system_tuning_monitor.sh old mode 100644 new mode 100755 diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh old mode 100644 new mode 100755 index 70dc32c..796ad1d --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -198,7 +198,7 @@ FINDINGS=() # notable but not critical ISSUES=() # need attention DIGEST_LINES=() # full report lines -# ── Failover State ──────────────────────────────────────────────────────────────────────────── +# ── fallback State ──────────────────────────────────────────────────────────────────────────── if [[ -f "$FALLBACK_STATE_FILE" ]]; then FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ -n "$FALLBACK_STATE" ]]; then diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh old mode 100644 new mode 100755 diff --git a/Notes_To-Do.md b/Notes_To-Do.md index b11ff00..bcab091 100755 --- a/Notes_To-Do.md +++ b/Notes_To-Do.md @@ -18,7 +18,7 @@ claude -. fix failover strike list timing, maybe 30 seconds. them a t 90 seconds 3 stike triggers. just gotta test buffer. never had the strike system +. fix fallback strike list timing, maybe 30 seconds. them a t 90 seconds 3 stike triggers. just gotta test buffer. never had the strike system . verify silent toggle switches back on good notifications diff --git a/Old_Arch_Still_Works/continuous_scripts_status.sh b/Old_Arch_Still_Works/continuous_scripts_status.sh old mode 100644 new mode 100755 diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index 289a904..2020828 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -658,7 +658,7 @@ DAILY_MAINTENANCE_SCRIPTS=( --- -### ── Relationship to Failover Writeback ────────────────────────────────────── +### ── Relationship to Fallback Writeback ────────────────────────────────────── ```bash # ───────────────────────────────────────────────────────────────────────────── @@ -668,7 +668,7 @@ DAILY_MAINTENANCE_SCRIPTS=( # Normal (daily_sync_maintenance.sh): # HOST1 → pushes Movies, Tv_Shows → HOST2 # -# Tier 4 failover writeback (HOST1 returns after 24hr+ outage): +# Tier 4 fallback writeback (HOST1 returns after 24hr+ outage): # HOST2 → pushes Movies, Tv_Shows → HOST1 # (HOST2 was running HOST1's arrs and accumulated content) # @@ -710,7 +710,7 @@ maintenance block before the 7am coffee report. # ───────────────────────────────────────────────────────────────────────────── # Containers stop BEFORE sync — clean static source, full bandwidth. # Containers start AFTER sync — on fresh data, in dependency order. -# DDNS and failover continue running throughout — only managed containers stop. +# DDNS and fallback continue running throughout — only managed containers stop. # # 1. Pre-flight checks — connectivity, remote Docker daemon, remote rootfs # 2. Stop local containers — Emby + auth stack stopped on this server diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh old mode 100644 new mode 100755 diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh old mode 100644 new mode 100755 diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh old mode 100644 new mode 100755 diff --git a/Orchestrators/monthly_maintenance.sh b/Orchestrators/monthly_maintenance.sh old mode 100644 new mode 100755 diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh old mode 100644 new mode 100755 diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh old mode 100644 new mode 100755 diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh old mode 100644 new mode 100755 index c9d650d..e28b07a --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -582,7 +582,7 @@ derive_partner_folder_name() { local hostname="$1" local short="${hostname,,}" [[ "$short" == unraid-* ]] && short="${short:7}" - # Capitalise first char for readability: jayred365 → Jayred365-Failover + # Capitalise first char for readability: jayred365 → Jayred365-fallback echo "${short^}-Fallback" } @@ -733,7 +733,7 @@ folderview3_remove_partner_folder() { return 0 } -# Gather all partner failover containers for this server (all tiers) +# Gather all partner fallback containers for this server (all tiers) gather_partner_fallback_containers() { local out_var="$1" eval "${out_var}=()" @@ -1570,7 +1570,7 @@ if (vv_write_conf_raw('master.conf', \$master)) { warn "DRY RUN — would write ACTIVE state and push to remote" fi - # FolderView3 — create partner folder with this server's failover containers for remote + # FolderView3 — create partner folder with this server's fallback containers for remote if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then echo "" echo "━━━ $ICON_CONTAINERS FolderView3 Integration ━━━" diff --git a/Plugin/unraid/run_job.sh b/Plugin/unraid/run_job.sh old mode 100644 new mode 100755 diff --git a/README-User_Script_Plug-in.md b/README-User_Script_Plug-in.md index 1e50383..5ef93e8 100644 --- a/README-User_Script_Plug-in.md +++ b/README-User_Script_Plug-in.md @@ -35,7 +35,7 @@ User Scripts Enhanced — Kurotaku ``` 1. Every script entry → Run in Background: YES Without background mode the plugin UI freezes waiting for the script to complete. - Long-running scripts and continuous loops (failover, watchdogs) require background. + Long-running scripts and continuous loops (fallback, watchdogs) require background. There are no exceptions — set background on everything. 2. Run as root: YES @@ -66,7 +66,7 @@ Create these categories in User Scripts Enhanced before adding any entries: ``` Orchestrators — scheduled maintenance windows and startup launcher -Failover — mutual failover state machine and test tools +Fallback — mutual fallback state machine and test tools Monitors — weekly reports, health checks, trend tracking Rsync — manual per-profile sync triggers Docker_Essentials — container management tools (manual use) @@ -119,12 +119,12 @@ ramdisk_setup.sh — creates /mnt/ramdisk_transcodes and symlink before docker_network_connect.sh— ensures custom networks exist + containers connected stability_watchdog.sh ← last-resort — three-tier server stability watchdog docker_watchdog.sh ← continuous — two-tier container healing watchdog -fallback.sh ← continuous — mutual failover state machine +fallback.sh ← continuous — mutual fallback state machine ``` **After running:** One-shot scripts exit cleanly. Three continuous processes run in the background until -the array stops. Check with: `pgrep -f "stability_watchdog\|docker_watchdog\|failover"` +the array stops. Check with: `pgrep -f "stability_watchdog\|docker_watchdog\|fallback"` --- @@ -291,7 +291,7 @@ in one orchestrated sequence. --- -### ── rsync_emby_failover ───────────────────────────────────────────────────── +### ── rsync_emby_fallback ───────────────────────────────────────────────────── > **Superseded by `critical_sync_maintenance`.** Delete this User Script from the > plugin. The emby-fallback dirty sync now runs as step 2 inside @@ -413,7 +413,7 @@ Background: YES ``` **What it does:** -Reads state files from across the ecosystem — watchdog strikes, failover state, skip +Reads state files from across the ecosystem — watchdog strikes, fallback state, skip list, bandwidth history, transcode stats, cert expiry — and compiles them into a digest notification. Reads only, writes nothing. @@ -421,7 +421,7 @@ digest notification. Reads only, writes nothing. ``` always — sends every run regardless of content (daily digest, even when healthy) smart — sends only when something worth reporting is found - triggers: watchdog strikes, non-NORMAL failover, cert warnings, bandwidth spikes + triggers: watchdog strikes, non-NORMAL fallback, cert warnings, bandwidth spikes weekly — sends once per week on DIGEST_DAY (Sunday), silent all other days ``` @@ -642,7 +642,7 @@ that survive rsync updates without parser changes. --- ## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -## 🔄 FAILOVER +## 🔄 FALLBACK ## ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ > **fallback.sh is started by array_started.sh — not scheduled separately.** @@ -650,11 +650,11 @@ that survive rsync updates without parser changes. --- -### ── failover_status ───────────────────────────────────────────────────────── +### ── fallback_status ───────────────────────────────────────────────────────── ``` -Category: Failover -Name: failover_status +Category: Fallback +Name: fallback_status Schedule: None — run manually Background: NO (runs and exits — status check) ``` @@ -665,21 +665,21 @@ Background: NO (runs and exits — status check) ``` **What it does:** -Shows current failover state at a glance without affecting the running loop. -State (NORMAL/FAILOVER/NO_INTERNET/DARK). Which tier containers are active. -Outage duration if in FAILOVER. Handback strike count if remote has returned. +Shows current fallback state at a glance without affecting the running loop. +State (NORMAL/FALLBACK/NO_INTERNET/DARK). Which tier containers are active. +Outage duration if in FALLBACK. Handback strike count if remote has returned. DDNS containers currently running. Remote server Tailscale visibility. -**Use before any manual failover intervention.** Understanding the current state +**Use before any manual fallback intervention.** Understanding the current state before acting prevents making things worse. --- -### ── failover_test ──────────────────────────────────────────────────────────── +### ── fallback_test ──────────────────────────────────────────────────────────── ``` -Category: Failover -Name: failover_test +Category: Fallback +Name: fallback_test Schedule: None — run manually (during a planned maintenance window) Background: YES — test takes several minutes ``` @@ -690,7 +690,7 @@ Background: YES — test takes several minutes ``` **What it does:** -Validates the full failover lifecycle — detection, Tier 1 container start, handback, +Validates the full fallback lifecycle — detection, Tier 1 container start, handback, container stop, DNS return — using an iptables rule to simulate remote unreachability. No real outage. Safety trap removes the iptables rule on any exit condition including ctrl-c and crashes — remote is always reachable after the test. @@ -702,7 +702,7 @@ ctrl-c and crashes — remote is always reachable after the test. **For a live test** (real container lifecycle, brief service interruption): Change `--dry-run` to nothing and run during a maintenance window. -**8 phases:** pre-flight → block remote → detect FAILOVER → verify Tier 1 started +**8 phases:** pre-flight → block remote → detect FALLBACK → verify Tier 1 started → restore remote → wait for handback → verify NORMAL restored → report pass/fail. --- @@ -711,7 +711,7 @@ Change `--dry-run` to nothing and run during a maintenance window. > **Not yet built.** `fallback_state_reset.sh` is a planned utility — not in `Fallback/` yet. > -> **Manual workaround** when state file is stuck in FAILOVER after a failed test or +> **Manual workaround** when state file is stuck in FALLBACK after a failed test or > forced kill of fallback.sh: > ```bash > # Verify first — right containers on right server, DDNS correct, fallback.sh not running @@ -721,7 +721,7 @@ Change `--dry-run` to nothing and run during a maintenance window. > Resets state only — does NOT start or stop any containers. **When you need this:** -State file stuck in FAILOVER after a test that didn't complete cleanly. After killing +State file stuck in FALLBACK after a test that didn't complete cleanly. After killing fallback.sh directly (not via plugin Abort button). After a failed handback. **Verify before resetting:** @@ -1615,7 +1615,7 @@ Wrong startup setup (old approach): docker_syslog_filter — At Startup of Array stability_watchdog — Every 15 Minutes via watchdog_orchestrator docker_watchdog — At Startup of Array - failover — At Startup of Array + fallback — At Startup of Array (unpredictable order, no coordination) ``` @@ -1680,7 +1680,7 @@ array_start --dry-run verify all startup scripts found and sequenced co transcode_management --status confirm ramdisk mounted and symlink pointing at ramdisk critical_sync_maintenance verify Critical-Data sync completes to HOST2 daily_sync_maintenance --dry-run verify remote connectivity and share list -failover_status confirm NORMAL state on both servers +fallback_status confirm NORMAL state on both servers docker_watchdog_status confirm required containers all running stability_watchdog_status confirm no threshold alerts on first check ``` \ No newline at end of file diff --git a/README.md b/README.md index c0346c3..9917003 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ **A complete self-healing, self-maintaining, mutually-redundant two-server unRAID ecosystem.** One codebase runs on both servers. Both servers run their own lives independently. When one goes down the other covers it — not because of conventional -failover infrastructure, but because both servers already have copies of everything +fallback infrastructure, but because both servers already have copies of everything that matters and both know how to run it. > **This is not conventional failover.** There is no primary and standby. There is no @@ -38,7 +38,7 @@ HOST2 — unRAID-Jayred365 **Hardware does not need to match.** Everything is accessed through `/mnt/user/` — unRAID's unified share layer. HOST1 has a Threadripper with ZFS. HOST2 has a completely -different CPU and fewer drives. A failover container on HOST2 mounts `/mnt/user/Movies` +different CPU and fewer drives. A fallback container on HOST2 mounts `/mnt/user/Movies` and sees mirrored data because the share names match. The hardware underneath is irrelevant. @@ -60,7 +60,7 @@ break silently. # # The convention: "no suffix = this container participates in the shared stack" # - Emby # the shared Emby — runs on HOST1, picked up by HOST2 on failover + Emby # the shared Emby — runs on HOST1, picked up by HOST2 on fallback NginxProxyManager # shared NPM — runs on both, same config via sync Lldap-Gmer4Lfe # shared LLDAP — same user directory, both servers Authelia # shared auth — both servers, same config @@ -93,7 +93,7 @@ codebase work on both servers without modification. Immich-Gmer4Lfe # Gmer4Lfe's photo library — personal ``` -User-specific containers are covered during failover (see Tiered Failover below) but +User-specific containers are covered during fallback (see Tiered Fallback below) but are distinct from the shared stack. They are never confused with each other because the suffix makes the owner unambiguous. Scripts targeting `Emby` and scripts targeting `Emby-Gmer4Lfe` are targeting different containers for different purposes. @@ -113,7 +113,7 @@ the suffix makes the owner unambiguous. Scripts targeting `Emby` and scripts tar # → looks for exactly those names in docker ps # # A container named differently than expected is not found — no error, no restart, -# no failover coverage. It silently does not happen. +# no fallback coverage. It silently does not happen. # Keep names consistent on both servers for shared containers. # ───────────────────────────────────────────────────────────────────────────── ``` @@ -235,11 +235,11 @@ where determines whether you need encryption and whether sync failure matters. --- -### ── Shared Failover Shares — Sync Required ───────────────────────────────── +### ── Shared Fallback Shares — Sync Required ───────────────────────────────── ```bash # ───────────────────────────────────────────────────────────────────────────── -# These shares enable failover — HOST2 must have current copies for the covered +# These shares enable fallback — HOST2 must have current copies for the covered # services to work correctly when they start on HOST2. # /mnt/user/Media_Server/Emby # Emby userdata — watch history, playstates @@ -248,7 +248,7 @@ where determines whether you need encryption and whether sync failure matters. /mnt/user/Tv_Shows # media library content /mnt/user/Music # media library content # -# Sync failure here = failover runs but with stale data (acceptable) or missing +# Sync failure here = fallback runs but with stale data (acceptable) or missing # data (problematic). These sync every night via daily_sync_maintenance.sh. # Critical-Data and Emby also sync every 30 minutes via critical_sync_maintenance.sh. # ───────────────────────────────────────────────────────────────────────────── @@ -256,12 +256,12 @@ where determines whether you need encryption and whether sync failure matters. --- -### ── Non-Failover Shares — Sync as Backup, No Encryption Required ─────────── +### ── Non-Fallback Shares — Sync as Backup, No Encryption Required ─────────── ```bash # ───────────────────────────────────────────────────────────────────────────── -# Some shares don't participate in failover — their container only runs on one -# server and is covered during failover from scratch (or not at all). +# Some shares don't participate in fallback — their container only runs on one +# server and is covered during fallback from scratch (or not at all). # These shares can sync as offsite backup without encryption. # Content is not sensitive — anyone with access to HOST2 could read it. # @@ -323,7 +323,7 @@ Two fully independent servers, each running their own household's services. Each server has its own domain, its own users, its own media library management. Each server is the primary for its own services every single day. -The "failover" component is additive: +The "fallback" component is additive: When HOST1 goes down → HOST2 starts HOST1's containers on top of its own When HOST2 comes back → HOST1's containers hand back cleanly HOST2 never stopped running its own services during this entire time @@ -378,7 +378,7 @@ This is not standby. This is mutual coverage between peers. --- -## ━━━ HOW FAILOVER ACTUALLY WORKS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +## ━━━ HOW FALLBACK ACTUALLY WORKS ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ `fallback.sh` runs on both servers as a continuous background process. Every 2 minutes it pings two things: the other server's Tailscale IP, and `8.8.8.8`. The combination @@ -387,7 +387,7 @@ of those two answers determines the current state. ``` Remote ping PASS + Internet PASS → NORMAL — both servers running independently -Remote ping FAIL + Internet PASS → FAILOVER — remote is down, start covering it +Remote ping FAIL + Internet PASS → FALLBACK — remote is down, start covering it Tier 1 immediately: DDNS + Emby + Auth + VaultWarden + Live TV + Gitea Tier 2 at 4 hours: NextCloud + Immich + Jellyseerr Tier 3 at 12 hours: AdGuard + Uptime monitoring @@ -410,12 +410,12 @@ DDNS last. Split-brain DNS is prevented by design. ## ━━━ THE AUTH STACK — WARM ON BOTH SERVERS ALWAYS ━━━━━━━━━━━━━━━━━━━━━━━━━━ NPM, LLDAP, Authelia, MariaDB, Redis — all running actively on both servers at all -times. Not cold standby. Not started at failover. Running right now, serving each +times. Not cold standby. Not started at fallback. Running right now, serving each server's own domain. ```bash # ───────────────────────────────────────────────────────────────────────────── -# Why this eliminates the failover dead zone: +# Why this eliminates the fallback dead zone: # # Cold standby approach: HOST1 goes down → DNS switches to HOST2 → HOST2 starts # auth containers → 30-60 seconds before auth is ready → users hit the auth wall @@ -454,7 +454,7 @@ Source of truth: Config auth stack config ──→ mirrored every 30 min Emby state watch history, playstates──→ dirty sync every 30 min -Failover coverage: +Fallback coverage: If HOST1 down: — starts Emby, VaultWarden, Dispatcharr, all arr stack serves Gmer4Lfe.com via DDNS @@ -489,7 +489,7 @@ varaverk/ ├── Rsync/ ← Core rsync script — called per share with profile │ README: README-Rsync_Setup.md │ -├── Fallback/ ← Mutual automatic failover — continuous background process +├── Fallback/ ← Mutual automatic fallback — continuous background process │ README: README-Fallback.md │ ├── Watchdogs/ ← All watchdog scripts: resource, docker, system, stability @@ -564,7 +564,7 @@ Array start (Varaverk event hook): → php_fpm_max_children.sh WebGUI tuning before first request → ramdisk_setup.sh create ramdisk before Emby starts → docker_network_connect.sh connect containers to extra networks - → fallback.sh continuous — mutual failover state machine + → fallback.sh continuous — mutual fallback state machine Every 15 minutes: watchdog_orchestrator.sh fires each watchdog in sequence @@ -782,9 +782,9 @@ If you're setting this up from scratch on two servers: 4. Run initial sync to populate HOST2 bash daily_sync_maintenance.sh --log -5. Verify failover works before relying on it - bash failover_test.sh --dry-run first - bash failover_test.sh when ready — uses iptables simulation, no real downtime +5. Verify fallback works before relying on it + bash fallback_test.sh --dry-run first + bash fallback_test.sh when ready — uses iptables simulation, no real downtime ``` --- @@ -802,7 +802,7 @@ If you're setting this up from scratch on two servers: | [README-Monitors.md](Monitors/README-Monitors.md) | Cert monitoring, SMART health, bandwidth tracking, health digest profiles | | [README-Partnership.md](Partnership/README-Partnership.md) | Auth stack sharing, onboard/offboard/transfer lifecycle, deferred offboard | | [Manual.md](Manual.md) | Complete setup guide — Tailscale, SSH keys, git clone, master.conf, Varaverk scheduler | -| [README-Tools.md](Tools/README-Tools.md) | failover_state_reset, skip list manager, emby_database_repair, container export | +| [README-Tools.md](Tools/README-Tools.md) | fallback_state_reset, skip list manager, emby_database_repair, container export | | [README-Unraid_Essentials.md](unRAID_Essentials/README-Unraid_Essentials.md) | `git_pull_execute.sh`, `server_reboot.sh`, `mover_stop.sh`, `user_scripts_stop.sh` | --- diff --git a/Rsync/Manual-Rsync.md b/Rsync/Manual-Rsync.md index c208e3e..627e78b 100644 --- a/Rsync/Manual-Rsync.md +++ b/Rsync/Manual-Rsync.md @@ -68,7 +68,7 @@ Terminal access — unRAID UI → Tools → Terminal, or SSH Optional but recommended: ``` Gitea (Docker container on HOST1) — self-hosted git for the script repository -Working Emby installation — for transcode management and failover +Working Emby installation — for transcode management and fallback ``` --- @@ -366,7 +366,7 @@ PROFILES["emby-fallback_REMOTE_RESTART_CONTAINERS"]=("Emby") - Why: WAL is checkpointed on clean shutdown — safe to copy everything - Result: HOST2 gets a gold-standard Emby state once per week -The two profiles work together. emby-fallback keeps HOST2 current for immediate failover. +The two profiles work together. emby-fallback keeps HOST2 current for immediate fallback. emby gives HOST2 full fidelity once per week. Neither alone is sufficient. --- diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh old mode 100644 new mode 100755 diff --git a/Tools/Manual-Tools.md b/Tools/Manual-Tools.md index dd9b33c..52083ef 100644 --- a/Tools/Manual-Tools.md +++ b/Tools/Manual-Tools.md @@ -572,7 +572,7 @@ docker_watchdog Memory-monitored containers Recent restart history + skip list -failover (fallback.sh) +fallback (fallback.sh) Current state (NORMAL / FALLBACK / HANDBACK) Tier flags and timestamps Remote Tailscale visibility diff --git a/Tools/README-Tools.md b/Tools/README-Tools.md index 23d7890..70dceec 100644 --- a/Tools/README-Tools.md +++ b/Tools/README-Tools.md @@ -13,8 +13,8 @@ the scheduled maintenance model. These scripts sit ready for the moment you need ## ━━━ THE PROBLEM THAT BUILT THIS ━━━ -**Failover State Stuck After Testing** -Run a failover test, something exits uncleanly, state file shows `FALLBACK`. +**Fallback State Stuck After Testing** +Run a fallback test, something exits uncleanly, state file shows `FALLBACK`. `fallback.sh` resumes and reads FALLBACK — starts containers it shouldn't, makes decisions based on a state that doesn't reflect reality. Manual recovery means knowing the exact file format and every field to reset. At 2am after a failed test. @@ -90,7 +90,7 @@ unRAID_Essentials/ ← regular system maintenance — scheduled Docker_Essentials/ ← regular container management — scheduled Monitors/ ← regular health reporting — scheduled Orchestrators/ ← regular maintenance windows — scheduled -Fallback/ ← automated failover/handback — event-driven +Fallback/ ← automated fallback/handback — event-driven Tools/ ← situational utilities — run when needed ``` @@ -116,7 +116,7 @@ The relationship is one-way: Tools act on state that other scripts have written. | Script | What It Fixes | When to Run | |--------|--------------|-------------| -| `fallback_state_reset.sh` | State file stuck in FALLBACK after test or failed handback | After failover testing or manual intervention | +| `fallback_state_reset.sh` | State file stuck in FALLBACK after test or failed handback | After fallback testing or manual intervention | | `watchdog_skip_list_manager.sh` | Container stuck on watchdog skip list after fixing root cause | After fixing a container that hit the restart loop limit | | `bulk_permissions_repair.sh` | Files owned by wrong user after admin copy or bad container config | When arr operations fail due to permissions | | `container_data_export.sh` | Need a clean backup before a risky container update or migration | Before major updates, appdata migrations, or container removals | @@ -143,7 +143,7 @@ Situation arises ┌──────────────────────────────────────────────────────────────────────┐ │ Tools/ Run directly when needed │ │ │ -│ fallback_state_reset.sh ◄── after failover test / failed handback│ +│ fallback_state_reset.sh ◄── after fallback test / failed handback│ │ watchdog_skip_list_manager ◄── after fixing a crash-looping container│ │ bulk_permissions_repair ◄── wrong ownership after copy or rsync │ │ container_data_export ◄── before a risky update or migration │ diff --git a/Tools/bulk_permissions_repair.sh b/Tools/bulk_permissions_repair.sh old mode 100644 new mode 100755 diff --git a/Tools/container_data_export.sh b/Tools/container_data_export.sh old mode 100644 new mode 100755 diff --git a/Tools/emby_database_repair.sh b/Tools/emby_database_repair.sh old mode 100644 new mode 100755 diff --git a/Tools/emby_to_radarr_sync.sh b/Tools/emby_to_radarr_sync.sh old mode 100644 new mode 100755 diff --git a/Tools/emby_to_sonarr_sync.sh b/Tools/emby_to_sonarr_sync.sh old mode 100644 new mode 100755 diff --git a/Tools/fallback_state_reset.sh b/Tools/fallback_state_reset.sh index 922728a..220aa48 100755 --- a/Tools/fallback_state_reset.sh +++ b/Tools/fallback_state_reset.sh @@ -1,12 +1,12 @@ #!/bin/bash # ============================================================================================== -# ============================= Failover State Reset =========================================== +# ============================= fallback State Reset =========================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Resets the fallback state file to NORMAL and clears all tier flags. Use when -# the state file is stuck after failover testing, a failed handback, manual +# the state file is stuck after fallback testing, a failed handback, manual # intervention, or fallback.sh being killed mid-cycle. # # Writes a fresh state file with: @@ -17,8 +17,8 @@ # resumes from NORMAL on its next cycle. # # WARNING: Only run when you have verified the stack is actually in a normal -# state — right containers on the right server, DDNS correct, no active failover -# in progress. Resetting state during a real failover causes fallback.sh to stop +# state — right containers on the right server, DDNS correct, no active fallback +# in progress. Resetting state during a real fallback causes fallback.sh to stop # covering the remote server until the next detection cycle. # # ============================================================================================== diff --git a/Tools/recreate_shares.sh b/Tools/recreate_shares.sh old mode 100644 new mode 100755 diff --git a/Tools/watchdog_skip_list_manager.sh b/Tools/watchdog_skip_list_manager.sh old mode 100644 new mode 100755 diff --git a/Tools/zfs_pool_scrub.sh b/Tools/zfs_pool_scrub.sh old mode 100644 new mode 100755 diff --git a/Transcodes/transcode_cleanup.sh b/Transcodes/transcode_cleanup.sh old mode 100644 new mode 100755 diff --git a/Transcodes/transcode_manager.sh b/Transcodes/transcode_manager.sh old mode 100644 new mode 100755 diff --git a/common.sh b/common.sh index fa5915a..8841067 100755 --- a/common.sh +++ b/common.sh @@ -48,7 +48,7 @@ # translate_path() added — container → host path for arr cleanup scripts # check_arr_version() added — API version safety gate before arr operations # check_api() added — pre-flight API reachability check -# ping_remote(), ping_internet() — non-fatal ping for failover use +# ping_remote(), ping_internet() — non-fatal ping for fallback use # check_local_array(), check_remote_array() — array health checks # check_remote_docker() — Docker daemon health check # get_unraid_temp_thresholds() — reads thresholds from dynamix.cfg @@ -159,7 +159,7 @@ ICON_LOCK="🔏" # script instance lock — acquired/released ICON_RAM="💨" # ramdisk operations — fast ephemeral storage ICON_LINK="🔗" # symlink state and management -# Failover Operations +# fallback Operations ICON_FALLBACK="🔀" # fallback state changes and operations # Docker Network Operations @@ -656,7 +656,7 @@ resolve_tailscale_ip() { # ============================================================================================== # Fatal connectivity check — used by rsync and other scripts that must abort if unreachable. -# For failover use ping_remote() which returns status without exiting. +# For fallback use ping_remote() which returns status without exiting. check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then @@ -739,7 +739,7 @@ is_vm_manager_enabled() { # Verifies local /mnt/user is mounted and has shares. # Non-fatal — returns status for caller to decide. -# Used by failover before starting remote containers locally. +# Used by fallback before starting remote containers locally. check_local_array() { log "Checking local array..." if ! mountpoint -q /mnt/user 2>/dev/null; then @@ -1660,7 +1660,7 @@ check_api() { # ============================================================================================== # Verifies local and remote servers are running compatible unRAID versions before any # remote operation. Version mismatches can mean changed APIs, commands, or behaviours -# that silently break remote container operations, rsync, or failover logic. +# that silently break remote container operations, rsync, or fallback logic. # # Reads /etc/unraid-version on both sides — format: version="7.2.3" # diff --git a/git_pull_execute.sh b/git_pull_execute.sh index 6320576..eeb3956 100755 --- a/git_pull_execute.sh +++ b/git_pull_execute.sh @@ -25,7 +25,7 @@ # New server gets only its own conf ✅ # # ── GITEA LOCATION DETECTION ────────────────────────────────────────────────────────────────── -# Detects where Gitea is running at runtime — works through failover: +# Detects where Gitea is running at runtime — works through fallback: # Gitea local → connects via local IP # Gitea remote → connects via Tailscale IP # Both fail → falls back to GITEA_DOMAIN if configured diff --git a/load_config.sh b/load_config.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/clear_logs.sh b/unRAID_Essentials/clear_logs.sh old mode 100644 new mode 100755 diff --git a/unRAID_Essentials/server_reboot.sh b/unRAID_Essentials/server_reboot.sh old mode 100644 new mode 100755 diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh old mode 100644 new mode 100755