Bring script headers onto the template and close safeguard gaps

Headers claimed protections the code never had, and several destructive paths had no
guard against a collapsed config value.

Fallback/fallback.sh

@@ -89,7 +89,50 @@
 #
 # FALLBACK_ENABLED Gate
 #   Exits cleanly when disabled — safe to run on servers being rebuilt without
-#   triggering spurious fallback actions.
+#   triggering spurious fallback actions. Fail-closed: anything that is not exactly
+#   "true" counts as disabled, so a malformed toggle cannot grant this script DDNS
+#   authority and cross-server container control by accident.
+#
+# Docker Presence Check
+#   Verifies the docker binary exists before the state machine starts.
+#
+# Remote IP Resolution
+#   resolve_remote_ip() must resolve the partner before any SSH operation, so a
+#   remote command can never be issued against an unresolved or stale address.
+#
+# Asymmetric Failover / Handback
+#   Entering FALLBACK is immediate — a down partner means users are already affected.
+#   Returning requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks. The
+#   asymmetry is deliberate: protecting fast costs a few minutes of redundant
+#   coverage, handing back fast on a flapping partner costs a second outage.
+#
+# DDNS Excluded From Tier Loops
+#   The Tier 1 stop and start loops explicitly skip any container that is also in
+#   REMOTE_DDNS_CONTAINERS. DDNS is sequenced by the handoff and cutover steps alone,
+#   so ordinary tier processing can never move DNS at the wrong moment.
+#
+# Writeback Delay Gate
+#   Tier writeback rsync only runs once the outage has exceeded that tier's writeback
+#   delay. A brief blip does not trigger a full data writeback, which would cost more
+#   than the outage it is compensating for.
+#
+# FALLBACK_RSYNC_ENABLED Gate
+#   Writeback is skipped entirely when disabled, and the skip is announced rather than
+#   silent — containers still hand back, but nobody is left assuming data moved.
+#
+# Play State Sync Before Cutover
+#   Handback retries play_state_sync up to PLAY_SYNC_HANDBACK_RETRIES times before DNS
+#   cuts over, so users land on current watch state. Exhausting retries warns and
+#   proceeds — stale resume positions are not worth holding DNS on a downed service.
+#
+# Partnership Suspend Abort
+#   If partnership goes inactive mid-fallback, _abort_fallback_containers() stops the
+#   fallback containers and returns to NORMAL rather than leaving this host serving a
+#   partner it is no longer paired with.
+#
+# Container Verify Wait
+#   CONTAINER_VERIFY_WAIT seconds elapse after each start before the running check, so
+#   a container that starts and immediately crashes is caught rather than counted up.
 #
 # Version Parity Check
 #   Refuses handback if remote unRAID version doesn't match. A mismatch may
@@ -271,8 +314,11 @@ if [[ "$EUID" -ne 0 ]]; then
     exit 1
 fi
 
-# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt)
-if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
+# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt).
+# Fail-closed: anything that isn't exactly "true" disables fallback. Matching only the
+# literal "false" would let a typo ("no", "0", "FALSE") hand this script DDNS authority
+# and cross-server container control on a toggle nobody meant to set.
+if [[ "${FALLBACK_ENABLED:-false}" != "true" ]]; then
     warn "FALLBACK_ENABLED=false — fallback monitoring disabled"
     warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready"
     exit 0
@@ -289,8 +335,6 @@ detect_hosts
 require_partnership
 resolve_remote_ip
 
-# Validate unRAID notify script — used throughout for state change notifications
-
 [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"
 
 # Timeout for all docker and SSH docker commands

Fallback/fallback_test.sh

@@ -41,9 +41,23 @@
 # ==============================================================================================
 #
 # iptables Safety Trap
-#   The DROP rule is removed via trap on ANY exit — normal completion, crash, error,
-#   ctrl-c. Remote connectivity is always restored regardless of test outcome.
-#   You cannot accidentally leave the remote permanently blocked.
+#   The DROP rule is removed via an EXIT trap that fires on normal completion, error
+#   exit, script crash, ctrl-c (SIGINT) and SIGTERM — verified, not assumed. Remote
+#   connectivity is restored regardless of test outcome.
+#
+# Stale Rule Sweep
+#   The trap above cannot cover SIGKILL or a power cut, which are the only ways a DROP
+#   rule survives the test. One stranded that way makes fallback.sh see the partner as
+#   permanently down and hold FALLBACK indefinitely, so pre-flight clears any leftover
+#   rule before doing anything else — including before the reachability check, which
+#   would otherwise fail and blame the network for the test's own residue.
+#
+# Root Enforcement
+#   iptables and container control require root.
+#
+# iptables Presence Check
+#   platform_require_cmd confirms iptables exists before the test begins — there is no
+#   point entering a connectivity simulation that cannot simulate anything.
 #
 # FALLBACK_ENABLED Gate
 #   Aborts if FALLBACK_ENABLED=false. Testing a disabled fallback system is
@@ -139,6 +153,28 @@ cleanup() {
 
 trap cleanup EXIT
 
+# ── Stale rule sweep — the one case the trap above cannot cover ───────────────────────────────
+# The EXIT trap fires on normal exit, error, ctrl-c and SIGTERM, but not on SIGKILL or a power
+# cut. A DROP rule stranded that way makes fallback.sh see the partner as permanently down and
+# sit in FALLBACK indefinitely — so clear any leftover from a previous run before starting.
+_clear_stale_block() {
+    [[ -z "${REMOTE_SERVER:-}" ]] && return
+    local removed=0
+    while iptables -C OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null; do
+        if [[ "$DRY_RUN" == true ]]; then
+            warn "DRY RUN — would remove stale iptables block on $REMOTE_SERVER"
+            return
+        fi
+        iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null || break
+        (( removed++ ))
+    done
+    if [[ "$removed" -gt 0 ]]; then
+        warn "$ICON_SHIELD Removed $removed stale iptables block(s) on $REMOTE_SERVER from a previous run"
+        notify "Fallback test cleared $removed stale iptables block(s) on $(hostname) — a previous test was killed before cleanup" \
+            "Fallback Test" "warning"
+    fi
+}
+
 # ==============================================================================================
 # ━━━ Setup ━━━
 # ==============================================================================================
@@ -149,7 +185,8 @@ if [[ "$EUID" -ne 0 ]]; then
 fi
 
 # FALLBACK_ENABLED gate — no point testing if fallback is disabled
-if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
+# Fail-closed, matching fallback.sh — anything not exactly "true" counts as disabled.
+if [[ "${FALLBACK_ENABLED:-false}" != "true" ]]; then
     warn "FALLBACK_ENABLED=false — fallback test aborted"
     warn "Enable fallback in master.conf before running this test"
     exit 0
@@ -238,6 +275,11 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━
 echo ""
 echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
 
+# Must run before the reachability check below — a stale DROP rule from a killed run makes
+# the partner look unreachable, and the test would abort blaming the network for its own
+# leftover.
+_clear_stale_block
+
 # Remote reachable
 if ping_remote; then
     log "$REMOTE_SERVER_NAME is reachable"