From aa6d1657254ef08ac109ae3a843d3ad232ec7e5f Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 18 May 2026 22:04:07 -0400 Subject: [PATCH] feat: add fallback.sh --stop for clean array shutdown Fallback runs as a bare subprocess (cmdline: bash .../Fallback/fallback.sh), not via /tmp/user.scripts, so user_scripts_stop.sh cannot find it. Adding --stop mode closes that gap in the array shutdown sequence. fallback.sh --stop: reads lock file (/tmp/unraid_locks/fallback.lock) to get the running PID, sends SIGTERM (hits the existing trap that sets FALLBACK_RUNNING=false), waits up to 10s, SIGKILLs if still alive. master.conf: insert Fallback/fallback.sh --stop as step 2 in ARRAY_STOP_SCRIPTS (after user_scripts_stop, before rsync_stop). user_script_plug-in.sh: update ARRAY STOPPING block to show 5-step order and add v2.5 changelog entry. --- Fallback/fallback.sh | 298 ++++++++++++++++++++++++++++++----------- master.conf | 5 +- user_script_plug-in.sh | 4 + 3 files changed, 226 insertions(+), 81 deletions(-) diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh index 271a2ff..29d54b9 100755 --- a/Fallback/fallback.sh +++ b/Fallback/fallback.sh @@ -1,100 +1,204 @@ #!/bin/bash # ============================================================================================== -# ================================= Failover =================================================== +# ================================= Fallback =================================================== # ============================================================================================== -# Mutual container failover between two unRAID servers. -# Each server runs this script independently — no direct coordination between servers. -# All decisions based solely on two pings: remote reachable + internet reachable. # -# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── -# Both servers run this script continuously as a background task via User Scripts. -# Every FALLBACK_CHECK_INTERVAL seconds each server: -# 1. Pings the remote server -# 2. Pings the internet -# 3. Determines its current state -# 4. Takes the appropriate action +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Mutual container fallback between two unRAID servers. Runs continuously as a +# background process — started at array start by array_start.sh. Each server +# runs this script independently with no coordination between servers. # -# No SSH signaling, no shared state files, no coordination — each server acts autonomously -# based only on what it can see from its own network perspective. +# All decisions are based solely on two pings per cycle: remote Tailscale IP +# reachable + internet reachable. No SSH signaling, no shared state, no election +# algorithm. Each server acts entirely from its own network perspective. # -# ── STATES ──────────────────────────────────────────────────────────────────────────────────── -# NORMAL — remote up, internet up -# Own containers only. DDNS ON. Silent operation. +# Never requires human intervention during normal fallback and handback. +# Stop only via User Scripts Abort — do NOT kill directly (state file may corrupt). # -# FALLBACK — remote down, internet up -# Start remote containers locally — tiered by outage duration. -# Remote DDNS started immediately (Tier 1). -# Own containers keep running — failover is additive. +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== # -# NO_INTERNET — internet down (remote may be up or down) -# Stop own DDNS immediately — can't update DNS without internet. -# Do not start remote containers — no internet = no point. -# Wait for recovery. +# Each cycle (every FALLBACK_CHECK_INTERVAL seconds): +# 1. Ping remote Tailscale IP — is the other server reachable? +# 2. Ping internet (EXTERNAL_IP) — do I have internet? +# 3. Determine state from the two results +# 4. Take the action for that state # -# DARK — remote down AND internet down -# Same actions as NO_INTERNET. -# Cannot determine if remote is truly down or just unreachable. +# States: +# NORMAL — remote up, internet up — own containers, own DDNS on, silent +# FALLBACK — remote down, internet up — start tier containers, DDNS over +# NO_INTERNET — internet down — stop own DDNS immediately, wait +# DARK — remote down AND internet down — same actions as NO_INTERNET # -# ── DDNS RULES — ABSOLUTE ───────────────────────────────────────────────────────────────────── -# Each server owns its own DDNS — ON when that server has internet. -# Script controls DDNS exclusively — network state NEVER auto-starts DDNS. -# DDNS only starts after full handback sequence confirms containers are up. -# One DDNS per domain active at all times — never two, never zero for long. -# 1 minute TTL + 90s strike window = minimal user impact on failover. +# FALLBACK escalation (tiered by outage duration): +# Tier 1 — immediate — vital services + Live TV (cannot wait) +# Tier 2 — after HOST*_TIER2_DELAY (default 4hr) — shared productivity +# Tier 3 — after HOST*_TIER3_DELAY (default 12hr) — secondary services +# Tier 4 — after HOST*_TIER4_DELAY (default 24hr) — arrs + downloaders # -# ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── -# When remote returns after FALLBACK: -# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks -# 2. Pre-flight checks — version parity, remote array, remote Docker daemon -# 3. Staged reverse-tier handback — Tier 4 → Tier 3 → Tier 2 → Tier 1 -# Each stage: stop local tier containers → rsync writeback → start on remote -# Emby (Tier 1) stays on fallback serving users until all higher tiers are done -# 4. DDNS handoff — immediately before Tier 1 goes down (not at start) -# 5. Tier 1 handback — stop local → rsync → start remote -# 6. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up +# Handback (when remote returns after FALLBACK): +# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks +# 2. Pre-flight checks — version parity, remote array, remote Docker daemon +# 3. Staged reverse: Tier 4→3→2 — Emby stays on covering server throughout +# Each tier: stop local → rsync writeback → start on remote +# 4. DDNS handoff — stop remote DDNS before Tier 1 goes down +# 5. Tier 1 handback — stop local vital services, rsync writeback, start remote +# 6. Start remote DDNS — DNS cuts back ONLY after containers confirmed running # 7. Return to NORMAL # -# ── TIERED FALLBACK ─────────────────────────────────────────────────────────────────────────── -# Tier 1 — Immediate — vital services + Live TV — cannot wait -# Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr) -# Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr) -# Tier 4 — HOST*_TIER4_DELAY — arrs + downloaders (default 24hr) -# Tier delays configurable per host in master_host*.conf +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== # -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# FALLBACK_ENABLED gate — exits cleanly if disabled in master.conf -# Version parity check — refuses handback if unRAID versions mismatch -# Remote Docker daemon — checks remote daemon before issuing any remote commands -# Timeout protection — all docker and SSH commands wrapped in timeouts -# Container verification — verifies containers came up after start (failover critical) -# MY_ID-based routing — all tier/DDNS/writeback arrays selected via MY_ID not hostname -# Command validation — validates unRAID notify script before use -# Silent by default — state transitions warn(), routine cycle checks log() +# Pure Ping-Only Detection +# No SSH signaling, no election algorithm, no shared state between servers. +# Each server makes all decisions from its own vantage point. A server that +# cannot reach the remote and has internet acts — no coordination needed. # -# ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── -# HOST*_DDNS_CONTAINERS — DDNS containers this host manages -# FALLBACK_HOST*_STOP_ON_NO_NET — containers stopped on internet loss -# FALLBACK_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other -# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes -# HOST*_TIER1_WRITEBACK_DELAY — skip Tier 1 writeback if outage under this -# FALLBACK_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier +# DDNS Absolute Control +# This script is the sole authority over DDNS containers. Network state +# returning is not permission to start DDNS — only the completion of the +# full handback sequence grants that. Eliminates the split-brain DNS window +# where two servers update the same domain with different IPs simultaneously. # -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# FALLBACK_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.) -# FALLBACK_CHECK_INTERVAL — seconds between checks (default 30) -# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback (default 3 = 90s) -# FALLBACK_STATE_FILE — /boot/config path — survives reboots -# FALLBACK_RSYNC_ENABLED — gate for writeback rsync jobs -# EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8) +# Staged Reverse-Tier Handback +# Emby stays on the covering server serving users while Tiers 4→3→2 hand +# back. The user impact window (Emby down) is only the Tier 1 rsync +# duration — typically minutes. Users are served continuously until the +# final step. # -# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# fallback.sh — normal start (continuous loop) -# fallback.sh --dry-run — preview state changes without acting on containers -# fallback.sh --status — show current state and exit -# fallback.sh --log — verbose cycle output +# Strike Confirmation +# Handback requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks. +# Brief network recovery during an ongoing outage would otherwise trigger a +# failed handback: containers stop on the covering server, remote goes down +# again mid-rsync. Strikes prevent this. +# +# MY_ID-Based Routing +# All tier arrays, DDNS lists, and writeback paths are selected via MY_ID +# set by detect_hosts() — not hostname string comparison. The same script +# and same config file handle both directions symmetrically. +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Enforcement +# Docker and SSH operations require root. +# +# FALLBACK_ENABLED Gate +# Exits cleanly when disabled — safe to run on servers being rebuilt without +# triggering spurious fallback actions. +# +# Version Parity Check +# Refuses handback if remote unRAID version doesn't match. A mismatch may +# mean the remote is not fully ready or on an incompatible version. +# +# Remote Docker Daemon Check +# SSH-checks the remote Docker daemon is responding before any remote commands. +# Reachable on Tailscale does not mean Docker is ready. +# +# Container Verification +# After every local_start(), verifies the container came up and stayed running. +# Catches crashes-on-start before declaring fallback successful. +# +# Timeout Protection +# All docker commands (DOCKER_TIMEOUT=15s) and SSH commands (SSH_TIMEOUT=10s) +# wrapped in timeouts. A hung daemon does not block the state machine. +# +# Single Instance Lock +# acquire_lock() prevents two instances running simultaneously. Both would +# modify containers and DDNS independently — conflict is certain. +# +# Silent by Default +# State transitions: warn() — always visible. Healthy routine cycles: log() +# — suppressed unless --log. Produces no output on clean cycles. +# +# ============================================================================================== +# STATE FILES +# ============================================================================================== +# +# FALLBACK_STATE_FILE — /boot/config/fallback_state.db (survives reboots) +# Keys: state, fallback_start, handback_strikes, tier2_started, +# tier3_started, tier4_started. Lives on /boot/ intentionally — if the +# server was in FALLBACK when it rebooted, it resumes FALLBACK on restart. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master_host*.conf +# +# HOST*_DDNS_CONTAINERS +# DDNS containers this host manages — stopped on internet loss, started +# as the last step of handback +# +# FALLBACK_HOST*_STOP_ON_NO_NET +# Containers stopped when this host loses internet +# +# FALLBACK_HOST*_COVERS_HOST*_TIER1–4 +# Containers this host starts for the remote when remote is down, by tier. +# Variable pattern: FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${N} +# +# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY +# Minutes after FALLBACK entry before activating each tier. +# References the remote host's ID: HOST1_TIER2_DELAY for HOST1 outage. +# (defaults: 240 / 720 / 1440) +# +# HOST*_TIER1_WRITEBACK_DELAY +# Minimum outage minutes before Tier 1 writeback runs. Skip for short +# outages where primary state is cleaner. (default: 60) +# +# FALLBACK_HOST*_WRITEBACK_TIER1–4 +# Paths rsynced back to remote on handback, per tier. +# Tier 4 auto-uses HOST*_DAILY_SYNC_SHARES — no separate list needed. +# Variable pattern: FALLBACK_${REMOTE_ID}_WRITEBACK_TIER${N} +# +# master.conf +# +# FALLBACK_ENABLED +# false = exit cleanly (e.g. remote server being rebuilt). (default: false) +# +# FALLBACK_CHECK_INTERVAL +# Seconds between connectivity checks. (default: 30) +# +# FALLBACK_HANDBACK_STRIKES +# Consecutive remote-up checks required before handback begins. (default: 3) +# +# FALLBACK_STATE_FILE +# State file path — /boot/config/fallback_state.db — survives reboots. +# +# FALLBACK_RSYNC_ENABLED +# Gate for writeback rsync jobs during handback. (default: true) +# +# EXTERNAL_IP +# IP pinged for internet check. (default: 8.8.8.8) +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# fallback.sh +# Normal start — continuous loop. Start via User Scripts or array_start.sh. +# +# fallback.sh --stop +# Gracefully stop the running instance (SIGTERM → wait 10s → SIGKILL). +# Used by array_stopping.sh during array shutdown. +# +# fallback.sh --dry-run +# Preview state changes without starting or stopping containers or DDNS. +# +# fallback.sh --status +# Show current state, identity, DDNS containers, outage duration if in +# FALLBACK, tier flags, strike counter. Then exit. +# +# fallback.sh --log +# Verbose output on every decision in every cycle. +# +# To stop: use `fallback.sh --stop` or click Abort in User Scripts. +# Do NOT kill -9 directly — state file may corrupt if mid-write. # -# ── TO STOP THIS SCRIPT ─────────────────────────────────────────────────────────────────────── -# Click Abort in unRAID User Scripts — do NOT kill directly, state file may corrupt. # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -103,6 +207,42 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" +# ── Stop mode — runs before acquire_lock so we can target the holding instance ──────────────── +if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then + LOCKFILE="${LOCK_DIR}/fallback.lock" + if [[ ! -f "$LOCKFILE" ]]; then + log "No fallback.sh lock found — not running" + exit 0 + fi + lock_content=$(cat "$LOCKFILE" 2>/dev/null) + target_pid="${lock_content%%:*}" + if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then + warn "Stale lock — fallback.sh not running (PID $target_pid gone) — clearing" + rm -f "$LOCKFILE" + exit 0 + fi + warn "Stopping fallback.sh (PID $target_pid)..." + kill -TERM "$target_pid" 2>/dev/null || true + waited=0 + while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 10 ]]; do + sleep 1 + (( waited++ )) + done + if kill -0 "$target_pid" 2>/dev/null; then + warn "SIGTERM ignored — sending SIGKILL to fallback.sh (PID $target_pid)" + kill -KILL "$target_pid" 2>/dev/null || true + sleep 2 + if kill -0 "$target_pid" 2>/dev/null; then + error "Failed to kill fallback.sh (PID $target_pid)" + exit 1 + fi + warn "Force-stopped: fallback.sh (PID $target_pid) ✅" + else + warn "Stopped: fallback.sh (PID $target_pid) ✅" + fi + exit 0 +fi + # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== diff --git a/master.conf b/master.conf index 183e965..ce9d153 100644 --- a/master.conf +++ b/master.conf @@ -201,10 +201,10 @@ PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron -# SSH key lifecycle — managed by Initial_run/ssh_setup.sh. +# SSH key lifecycle — managed by Partnership/ssh_setup.sh. # Key named after this server: hostname lowercased, unraid- prefix stripped. # unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation -# Run Initial_run/partnership_onboard.sh to generate, copy, and update conf automatically. +# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically. SSH_MAX_STRIKES=5 # consecutive SSH auth failures before critical notify SSH_STRIKE_RESET_HRS=24 # hours since last failure before strike counter resets @@ -269,6 +269,7 @@ # Order matters: user scripts first (prevents new ops), then data movement, then containers. ARRAY_STOP_SCRIPTS=( "unRAID_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops + "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) "unRAID_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) "unRAID_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) "Docker_Essentials/docker_container_stop.sh" # stop all containers last diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 873d980..6873be6 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -88,6 +88,9 @@ # resource_watchdog.sh: three-level pressure reduction (soft/medium/hard). # docker_container_stop.sh: sequential verified container stop for shutdown. # Path fixes: git_pull_execute.sh (root), continuous_scripts_status.sh (Tools/). +# v2.5 — fallback.sh --stop: added to ARRAY_STOP_SCRIPTS as step 2 (after user_scripts_stop, +# before rsync_stop) — fallback runs as a bare subprocess, not via /tmp/user.scripts, +# so it is not caught by user_scripts_stop.sh. # ============================================================================================== @@ -134,6 +137,7 @@ # # Runs in order: # user_scripts_stop.sh stop background user scripts first (prevents new operations) +# fallback.sh --stop gracefully stop fallback (not caught by user_scripts_stop) # rsync_stop.sh --rsync-only kill rsync without triggering container recovery # mover_stop.sh stop mover after rsync (both write to same paths) # docker_container_stop.sh stop all containers one-by-one with verification