#!/bin/bash
# ==============================================================================================
# ================================= Fallback ===================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Mutual container fallback between two unRAID servers. Runs continuously as a
# background process — started at array start by array_started.sh. Each server
# runs this script independently with no coordination between servers.
#
# All decisions are based solely on two pings per cycle: remote Tailscale IP
# reachable + internet reachable. No SSH signaling, no shared state, no election
# algorithm. Each server acts entirely from its own network perspective.
#
# Never requires human intervention during normal fallback and handback.
# Stop only via `fallback.sh --stop` — do NOT kill directly (state file may corrupt).
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Each cycle (every FALLBACK_CHECK_INTERVAL seconds):
#   1. Ping remote Tailscale IP — is the other server reachable?
#   2. Ping internet (EXTERNAL_IP) — do I have internet?
#   3. Determine state from the two results
#   4. Take the action for that state
#
# States:
#   NORMAL       — remote up, internet up — own containers, own DDNS on, silent
#   FALLBACK     — remote down, internet up — start tier containers, DDNS over
#   NO_INTERNET  — internet down — stop own DDNS immediately, wait
#   DARK         — remote down AND internet down — same actions as NO_INTERNET
#
# FALLBACK escalation (tiered by outage duration):
#   Tier 1 — immediate — vital services + Live TV (cannot wait)
#   Tier 2 — after HOST*_TIER2_DELAY (default 4hr) — shared productivity
#   Tier 3 — after HOST*_TIER3_DELAY (default 12hr) — secondary services
#   Tier 4 — after HOST*_TIER4_DELAY (default 24hr) — arrs + downloaders
#
# Handback (when remote returns after FALLBACK):
#   1. Strike confirmation  — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
#   2. Pre-flight checks    — version parity, remote array, remote Docker daemon
#   3. Staged reverse: Tier 4→3→2 — Emby stays on covering server throughout
#      Each tier: stop local → rsync writeback → start on remote
#   4. DDNS handoff         — stop remote DDNS before Tier 1 goes down
#   5. Tier 1 handback      — stop local vital services, rsync writeback, start remote
#   6. Start remote DDNS    — DNS cuts back ONLY after containers confirmed running
#   7. Return to NORMAL
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Pure Ping-Only Detection
#   No SSH signaling, no election algorithm, no shared state between servers.
#   Each server makes all decisions from its own vantage point. A server that
#   cannot reach the remote and has internet acts — no coordination needed.
#
# DDNS Absolute Control
#   This script is the sole authority over DDNS containers. Network state
#   returning is not permission to start DDNS — only the completion of the
#   full handback sequence grants that. Eliminates the split-brain DNS window
#   where two servers update the same domain with different IPs simultaneously.
#
# Staged Reverse-Tier Handback
#   Emby stays on the covering server serving users while Tiers 4→3→2 hand
#   back. The user impact window (Emby down) is only the Tier 1 rsync
#   duration — typically minutes. Users are served continuously until the
#   final step.
#
# Strike Confirmation
#   Handback requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks.
#   Brief network recovery during an ongoing outage would otherwise trigger a
#   failed handback: containers stop on the covering server, remote goes down
#   again mid-rsync. Strikes prevent this.
#
# MY_ID-Based Routing
#   All tier arrays, DDNS lists, and writeback paths are selected via MY_ID
#   set by detect_hosts() — not hostname string comparison. The same script
#   and same config file handle both directions symmetrically.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Docker and SSH operations require root.
#
# FALLBACK_ENABLED Gate
#   Exits cleanly when disabled — safe to run on servers being rebuilt without
#   triggering spurious fallback actions.
#
# Version Parity Check
#   Refuses handback if remote unRAID version doesn't match. A mismatch may
#   mean the remote is not fully ready or on an incompatible version.
#
# Remote Docker Daemon Check
#   SSH-checks the remote Docker daemon is responding before any remote commands.
#   Reachable on Tailscale does not mean Docker is ready.
#
# Container Verification
#   After every local_start(), verifies the container came up and stayed running.
#   Catches crashes-on-start before declaring fallback successful.
#
# Timeout Protection
#   All docker commands (DOCKER_TIMEOUT=15s) and SSH commands (SSH_TIMEOUT=10s)
#   wrapped in timeouts. A hung daemon does not block the state machine.
#
# Single Instance Lock
#   acquire_lock() prevents two instances running simultaneously. Both would
#   modify containers and DDNS independently — conflict is certain.
#
# Host Detection
#   detect_hosts() resolves MY_ID / REMOTE_ID from master.conf at startup.
#   Exits if the host cannot be identified — prevents running on an unknown machine.
#
# Partnership Gate
#   When FALLBACK_PARTNERSHIP_REQUIRED=true, fallback is only permitted when the
#   local partnership DB reports an active partnership. If partnership goes inactive,
#   a grace timer starts. After FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes the loop
#   suspends: no new FALLBACK is entered, and any live fallback containers are
#   stopped cleanly. Resumes automatically when partnership becomes active again.
#   Set FALLBACK_PARTNERSHIP_REQUIRED=false to run fallback without a partnership.
#
# Silent by Default
#   State transitions: warn() — always visible. Healthy routine cycles: log()
#   — suppressed unless --log. Produces no output on clean cycles.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
#   FALLBACK_STATE_FILE   — /boot/config/fallback_state.db (survives reboots)
#       Keys: state, fallback_start, handback_strikes, tier2_started,
#       tier3_started, tier4_started, partnership_suspended, partner_lost_at.
#       Lives on /boot/ intentionally — if the server was in FALLBACK when it
#       rebooted, it resumes FALLBACK on restart.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_DDNS_CONTAINERS
#       DDNS containers this host manages — stopped on internet loss, started
#       as the last step of handback
#
#   FALLBACK_HOST*_STOP_ON_NO_NET
#       Containers stopped when this host loses internet
#
#   FALLBACK_HOST*_COVERS_HOST*_TIER1–4
#       Containers this host starts for the remote when remote is down, by tier.
#       Variable pattern: FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${N}
#
#   HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY
#       Minutes after FALLBACK entry before activating each tier.
#       References the remote host's ID: HOST1_TIER2_DELAY for HOST1 outage.
#       (defaults: 240 / 720 / 1440)
#
#   HOST*_TIER1_WRITEBACK_DELAY
#       Minimum outage minutes before Tier 1 writeback runs. Skip for short
#       outages where primary state is cleaner. (default: 60)
#
#   FALLBACK_HOST*_WRITEBACK_TIER1–4
#       Paths rsynced back to remote on handback, per tier.
#       Tier 4 auto-uses HOST*_DAILY_SYNC_SHARES — no separate list needed.
#       Variable pattern: FALLBACK_${REMOTE_ID}_WRITEBACK_TIER${N}
#
# master.conf
#
#   FALLBACK_ENABLED
#       false = exit cleanly (e.g. remote server being rebuilt). (default: false)
#
#   FALLBACK_PARTNERSHIP_REQUIRED
#       true = fallback is gated on an active partnership — suspends when no active
#       partner is found after FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes.
#       false = fallback runs standalone regardless of partnership state. (default: true)
#
#   FALLBACK_PARTNERSHIP_SUSPEND_AFTER
#       Minutes without an active partnership before fallback suspends itself.
#       Grace period: fallback continues normally during this window.
#       0 = suspend immediately when partnership goes inactive. (default: 120)
#
#   FALLBACK_CHECK_INTERVAL
#       Seconds between connectivity checks. (default: 30)
#
#   FALLBACK_HANDBACK_STRIKES
#       Consecutive remote-up checks required before handback begins. (default: 3)
#
#   FALLBACK_STATE_FILE
#       State file path — /boot/config/fallback_state.db — survives reboots.
#
#   FALLBACK_RSYNC_ENABLED
#       Gate for writeback rsync jobs during handback. (default: true)
#
#   EXTERNAL_IP
#       IP pinged for internet check. (default: 8.8.8.8)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# fallback.sh
#     Normal start — continuous loop. Started by array_started.sh at array start.
#
# fallback.sh --stop
#     Gracefully stop the running instance (SIGTERM → wait 10s → SIGKILL).
#     Used by array_stopping.sh during array shutdown.
#
# fallback.sh --dry-run
#     Preview state changes without starting or stopping containers or DDNS.
#
# fallback.sh --status
#     Show current state, identity, DDNS containers, outage duration if in
#     FALLBACK, tier flags, strike counter. Then exit.
#
# fallback.sh --log
#     Verbose output on every decision in every cycle.
#
# To stop: use `fallback.sh --stop`. Do NOT kill -9 — state file may corrupt if mid-write.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
    LOCKFILE="${LOCK_DIR}/fallback.lock"
    if [[ ! -f "$LOCKFILE" ]]; then
        log "No fallback.sh lock found — not running"
        exit 0
    fi
    lock_content=$(cat "$LOCKFILE" 2>/dev/null)
    target_pid="${lock_content%%:*}"
    if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
        warn "Stale lock — fallback.sh not running (PID $target_pid gone) — clearing"
        rm -f "$LOCKFILE"
        exit 0
    fi
    warn "Stopping fallback.sh (PID $target_pid)..."
    kill -TERM "$target_pid" 2>/dev/null || true
    waited=0
    while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 10 ]]; do
        sleep 1
        (( waited++ ))
    done
    if kill -0 "$target_pid" 2>/dev/null; then
        warn "SIGTERM ignored — sending SIGKILL to fallback.sh (PID $target_pid)"
        kill -KILL "$target_pid" 2>/dev/null || true
        sleep 2
        if kill -0 "$target_pid" 2>/dev/null; then
            error "Failed to kill fallback.sh (PID $target_pid)"
            exit 1
        fi
        warn "Force-stopped: fallback.sh (PID $target_pid) ✅"
    else
        warn "Stopped: fallback.sh (PID $target_pid) ✅"
    fi
    exit 0
fi

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt)
if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
    warn "FALLBACK_ENABLED=false — fallback monitoring disabled"
    warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready"
    exit 0
fi

acquire_lock   # strict single instance — notifies on collision

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts
resolve_remote_ip

# Validate unRAID notify script — used throughout for state change notifications

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"

# Timeout for all docker and SSH docker commands
DOCKER_TIMEOUT=15
SSH_TIMEOUT=10
CONTAINER_VERIFY_WAIT=5   # seconds after start before verifying container is up

log "$ICON_GEAR Config: check-interval=${FALLBACK_CHECK_INTERVAL}s handback-strikes=${FALLBACK_HANDBACK_STRIKES} partnership-required=${FALLBACK_PARTNERSHIP_REQUIRED:-true} rsync-enabled=${FALLBACK_RSYNC_ENABLED:-true}"
log "$ICON_GEAR Timeouts: docker=${DOCKER_TIMEOUT}s ssh=${SSH_TIMEOUT}s verify-wait=${CONTAINER_VERIFY_WAIT}s"

# ==============================================================================================
# ── STATE FILE HELPERS ────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# State file on /boot/config — survives reboots.
# Format: key=value one per line.
# Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started

state_get() {
    grep "^${1}=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2
}

state_set() {
    local key="$1" value="$2"
    if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then
        # Escape sed replacement metacharacters: \ first, then & and |
        local safe="${value//\\/\\\\}"; safe="${safe//&/\\&}"; safe="${safe//|/\\|}"
        sed -i "s|^${key}=.*|${key}=${safe}|" "$FALLBACK_STATE_FILE"
    else
        echo "${key}=${value}" >> "$FALLBACK_STATE_FILE"
    fi
}

state_init() {
    mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
    [[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
    [[ -z "$(state_get state)" ]]                  && state_set state "NORMAL"
    [[ -z "$(state_get fallback_start)" ]]         && state_set fallback_start "0"
    [[ -z "$(state_get handback_strikes)" ]]       && state_set handback_strikes "0"
    [[ -z "$(state_get tier2_started)" ]]          && state_set tier2_started "false"
    [[ -z "$(state_get tier3_started)" ]]          && state_set tier3_started "false"
    [[ -z "$(state_get tier4_started)" ]]          && state_set tier4_started "false"
    [[ -z "$(state_get partnership_suspended)" ]]  && state_set partnership_suspended "false"
    [[ -z "$(state_get partner_lost_at)" ]]        && state_set partner_lost_at "0"
}

# ==============================================================================================
# ── PARTNERSHIP GATE HELPERS ──────────────────────────────────────────────────────────────────
# ==============================================================================================

# Returns 0 (true) if the local partnership DB reports an ACTIVE partnership.
# Reads /boot/config/partnership_<hostname>.db — no subprocess, no SSH.
check_partnership_active() {
    local state_file="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
    local state
    state=$(grep "^state=" "$state_file" 2>/dev/null | cut -d= -f2)
    [[ "$state" == "ACTIVE" ]]
}

# Stop all tiers that were started during a fallback and clear their state flags.
# Called when the partnership gate suspends an in-progress fallback.
_abort_fallback_containers() {
    for tier in 4 3 2 1; do
        [[ "$(state_get "tier${tier}_started")" != "true" ]] && continue
        warn "Aborting Tier $tier containers — partnership suspended"
        local containers
        read -r -a containers <<< "$(get_tier_containers "$tier")"
        for container in "${containers[@]}"; do
            [[ -n "$container" ]] && local_stop "$container"
        done
        state_set "tier${tier}_started" "false"
    done
}

# ==============================================================================================
# ── CONTAINER HELPERS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# All docker commands wrapped in DOCKER_TIMEOUT.
# All SSH docker commands wrapped in SSH_TIMEOUT.
# Verification after start — fallback is critical, confirm containers came up.

# Start a container locally and verify it came up
local_start() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
        "$container" 2>/dev/null)
    if [[ "$status" == "true" ]]; then
        log "$container already running locally"
        return 0
    fi
    log "$ICON_START Starting $container locally..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would start $container locally"
        return 0
    fi
    if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
        sleep "$CONTAINER_VERIFY_WAIT"
        local post_status
        post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
            "$container" 2>/dev/null)
        if [[ "$post_status" == "true" ]]; then
            log "$ICON_STARTED $container started and running ✅"
            return 0
        else
            error "$container started but crashed immediately"
            notify "$container failed to stay running during fallback on $(hostname)" \
                "Fallback" "warning"
            return 1
        fi
    else
        error "Failed to start $container locally"
        notify "Failed to start $container locally during fallback on $(hostname)" \
            "Fallback" "warning"
        return 1
    fi
}

# Stop a container locally
local_stop() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
        "$container" 2>/dev/null)
    if [[ "$status" != "true" ]]; then
        log "$container already stopped locally"
        return 0
    fi
    log "$ICON_STOP Stopping $container locally..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would stop $container locally"
        return 0
    fi
    timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
        log "$ICON_STOPPED $container stopped" || \
        error "Failed to stop $container locally"
}

# Start a container on remote via SSH and verify it came up
remote_start() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
        $container 2>/dev/null" 2>/dev/null)
    if [[ "$status" == "true" ]]; then
        log "$container already running on $REMOTE_SERVER_NAME"
        return 0
    fi
    log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
        return 0
    fi
    if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then
        sleep "$CONTAINER_VERIFY_WAIT"
        local post_status
        post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \
            "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
            $container 2>/dev/null" 2>/dev/null)
        if [[ "$post_status" == "true" ]]; then
            log "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅"
            return 0
        else
            error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
            notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \
                "Fallback" "warning"
            return 1
        fi
    else
        error "Failed to start $container on $REMOTE_SERVER_NAME"
        notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \
            "Fallback" "warning"
        return 1
    fi
}

# Stop a container on remote via SSH
remote_stop() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
        $container 2>/dev/null" 2>/dev/null)
    if [[ "$status" != "true" ]]; then
        log "$container already stopped on $REMOTE_SERVER_NAME"
        return 0
    fi
    log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
        return 0
    fi
    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
        log "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
        error "Failed to stop $container on $REMOTE_SERVER_NAME"
}

# ==============================================================================================
# ── DDNS HELPERS ──────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# DDNS managed exclusively by this script — never by network state alone.
# Stopping DDNS on internet loss prevents split-brain DNS updates.

local_ddns_stop() {
    warn "$ICON_NET Stopping local DDNS — internet lost"
    for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
        local_stop "$container"
    done
}

local_ddns_start() {
    warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up"
    for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
        local_start "$container"
    done
}

remote_ddns_stop() {
    warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync"
    for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
        remote_stop "$container"
    done
}

# ==============================================================================================
# ── TIERED FALLBACK HELPERS ───────────────────────────────────────────────────────────────────
# ==============================================================================================
# All routing uses MY_ID — not hostname string comparison.
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
# "This server covers the other" — MY_ID selects which tier arrays to use.

get_tier_containers() {
    local tier="$1"
    local remote_id="${REMOTE_ID}"
    local var_name="FALLBACK_${MY_ID}_COVERS_${remote_id}_TIER${tier}"
    eval "echo \"\${${var_name}[@]:-}\""
}

get_tier2_delay() {
    local var_name="${REMOTE_ID}_TIER2_DELAY"
    echo "${!var_name:-240}"
}

get_tier3_delay() {
    local var_name="${REMOTE_ID}_TIER3_DELAY"
    echo "${!var_name:-720}"
}

get_tier4_delay() {
    local var_name="${REMOTE_ID}_TIER4_DELAY"
    echo "${!var_name:-1440}"
}

get_tier1_writeback_delay() {
    local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY"
    echo "${!var_name:-60}"
}

get_writeback_jobs_for_tier() {
    local tier="$1"
    local remote_id="${REMOTE_ID}"
    if [[ "$tier" -eq 4 ]]; then
        # Tier 4 — daily sync shares of the remote + any extra writeback paths
        local shares_var="${remote_id}_DAILY_SYNC_SHARES"
        local extra_var="FALLBACK_${remote_id}_WRITEBACK_TIER4"
        eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\""
    else
        local var_name="FALLBACK_${remote_id}_WRITEBACK_TIER${tier}"
        eval "echo \"\${${var_name}[@]:-}\""
    fi
}

# Select DDNS arrays based on MY_ID
set_ddns_arrays() {
    local local_ddns_var="${MY_ID}_DDNS_CONTAINERS"
    local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS"
    eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")"
    eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")"
}

# Select internet-loss stop list based on MY_ID
get_stop_on_no_net() {
    local var_name="FALLBACK_${MY_ID}_STOP_ON_NO_NET"
    eval "echo \"\${${var_name}[@]:-}\""
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    state_init
    set_ddns_arrays
    CURRENT_STATE=$(state_get state)
    FALLBACK_START_TS=$(state_get fallback_start)
    TIER2=$(state_get tier2_started)
    TIER3=$(state_get tier3_started)
    TIER4=$(state_get tier4_started)
    STRIKES=$(state_get handback_strikes)

    local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")

    echo ""
    echo "━━━━━ $ICON_SUMMARY FALLBACK STATUS ━━━━━"
    echo "$ICON_HOST    My ID:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_HOST    Remote ID:   $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
    echo "$ICON_GEAR    OS ver:      $local_ver"
    echo "$ICON_FALLBACK State:      $CURRENT_STATE"
    echo "$ICON_NET     Local DDNS:  ${LOCAL_DDNS_CONTAINERS[*]:-none}"
    echo "$ICON_NET     Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}"
    echo "$ICON_TIME    Interval:    ${FALLBACK_CHECK_INTERVAL}s"
    echo "$ICON_FALLBACK Strikes:    $STRIKES / $FALLBACK_HANDBACK_STRIKES"

    if [[ "$CURRENT_STATE" == "FALLBACK" && "$FALLBACK_START_TS" -gt 0 ]]; then
        ELAPSED=$(( ($(date +%s) - FALLBACK_START_TS) / 60 ))
        echo "$ICON_TIME    Outage:      ${ELAPSED}min"
        echo "$ICON_FALLBACK Tier 2:     $TIER2 (delay: $(get_tier2_delay)min)"
        echo "$ICON_FALLBACK Tier 3:     $TIER3 (delay: $(get_tier3_delay)min)"
        echo "$ICON_FALLBACK Tier 4:     $TIER4 (delay: $(get_tier4_delay)min)"
    fi

    echo "$ICON_GEAR    Enabled:     $FALLBACK_ENABLED"
    echo "$ICON_GEAR    Dry Run:     $DRY_RUN"

    if [[ "${FALLBACK_PARTNERSHIP_REQUIRED:-true}" == "true" ]]; then
        SUSPENDED=$(state_get partnership_suspended)
        PLOST=$(state_get partner_lost_at)
        if [[ "$SUSPENDED" == "true" ]]; then
            echo "$ICON_GEAR    Partnership: SUSPENDED (no active partnership)"
        elif [[ -n "$PLOST" && "$PLOST" != "0" ]]; then
            PLOST_MIN=$(( ($(date +%s) - PLOST) / 60 ))
            echo "$ICON_GEAR    Partnership: GRACE (${PLOST_MIN}min / ${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min until suspend)"
        else
            echo "$ICON_GEAR    Partnership: ACTIVE"
        fi
    else
        echo "$ICON_GEAR    Partnership: REQUIRED=false (standalone mode)"
    fi

    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Called when remote returns after FALLBACK state.
# CRITICAL sequencing — do not reorder without understanding the consequences.
# Each step must succeed before proceeding — aborts and retries next cycle on failure.

run_handback() {
    echo ""
    echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    warn "$REMOTE_SERVER_NAME has returned — beginning staged handback"

    # ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
    echo ""
    echo "━━━ $ICON_SHIELD Pre-flight ━━━"

    if ! check_unraid_version_parity; then
        warn "Version parity check failed — aborting handback, will retry next cycle"
        state_set handback_strikes 0
        return 1
    fi

    if ! check_remote_array; then
        warn "Remote array not ready — aborting handback, will retry next cycle"
        state_set handback_strikes 0
        return 1
    fi

    if ! check_remote_docker_daemon; then
        warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
        state_set handback_strikes 0
        return 1
    fi

    echo "  Pre-flight checks passed ✅"

    local outage_minutes
    outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
    local tier1_wb_delay t2_delay t3_delay t4_delay
    tier1_wb_delay=$(get_tier1_writeback_delay)
    t2_delay=$(get_tier2_delay)
    t3_delay=$(get_tier3_delay)
    t4_delay=$(get_tier4_delay)

    warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1"

    # ── Staged handback helper ────────────────────────────────────────────────────────────────
    # Each tier: stop local containers → rsync writeback → start on remote.
    # Skips tiers that were never activated. Emby (Tier 1) stays on fallback
    # serving users until all higher tiers complete.
    handback_tier() {
        local tier="$1" threshold="$2" label="$3"

        [[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0

        echo ""
        echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━"

        local containers
        read -r -a containers <<< "$(get_tier_containers "$tier")"

        echo "  Stopping local Tier $tier containers..."
        for container in "${containers[@]}"; do
            [[ -n "$container" ]] && local_stop "$container"
        done

        if ! check_rsync_enabled "FALLBACK"; then
            warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback"
        else
            local jobs
            read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
            if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then
                echo "  Rsync writeback — outage ${outage_minutes}min >= ${threshold}min"
                for job in "${jobs[@]}"; do
                    [[ -z "$job" ]] && continue
                    log "Syncing: $job"
                    [[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
                        || warn "DRY RUN — would rsync: $job"
                done
            else
                log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
            fi
        fi

        echo "  Starting Tier $tier on $REMOTE_SERVER_NAME..."
        for container in "${containers[@]}"; do
            [[ -n "$container" ]] && remote_start "$container"
        done

        state_set "tier${tier}_started" "false"
        warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME"
    }

    # ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ──────────────────────────────────
    handback_tier 4 "$t4_delay" "media shares + edge cases"
    handback_tier 3 "$t3_delay" "secondary services"
    handback_tier 2 "$t2_delay" "productivity services"

    # ── Step 5: DDNS handoff — immediately before Tier 1 goes down ──────────────────────────
    # Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks.
    # Prevents split-brain DNS during Tier 1 rsync window.
    echo ""
    echo "━━━ $ICON_NET DDNS Handoff ━━━"
    remote_ddns_stop
    sleep 5

    # ── Step 6: Tier 1 handback — vital services + Emby ─────────────────────────────────────
    echo ""
    echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━"

    read -r -a t1 <<< "$(get_tier_containers 1)"

    echo "  Stopping local Tier 1 containers..."
    for container in "${t1[@]}"; do
        [[ -z "$container" ]] && continue
        local is_ddns=false
        for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
            [[ "$container" == "$ddns" ]] && is_ddns=true && break
        done
        [[ "$is_ddns" == false ]] && local_stop "$container"
    done

    if ! check_rsync_enabled "FALLBACK"; then
        warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback"
    else
        local t1_jobs
        read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)"
        if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then
            echo "  Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min"
            for job in "${t1_jobs[@]}"; do
                [[ -z "$job" ]] && continue
                log "Syncing: $job"
                if [[ "$DRY_RUN" == false ]]; then
                    if [[ "$(basename "$job")" == "Emby" ]]; then
                        bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback
                    else
                        bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
                    fi
                else
                    warn "DRY RUN — would rsync: $job"
                fi
            done
        else
            log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min"
        fi
    fi

    echo "  Starting Tier 1 on $REMOTE_SERVER_NAME..."
    sleep 10  # give databases time to initialise before apps
    for container in "${t1[@]}"; do
        [[ -z "$container" ]] && continue
        local is_ddns=false
        for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
            [[ "$container" == "$ddns" ]] && is_ddns=true && break
        done
        [[ "$is_ddns" == false ]] && remote_start "$container"
    done

    # ── Step 7: DNS Cutover — final step ────────────────────────────────────────────────────
    # DNS cuts over ONLY after Tier 1 containers confirmed up
    echo ""
    echo "━━━ $ICON_NET DNS Cutover ━━━"
    sleep 5
    for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
        remote_start "$container"
    done
    warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"

    # ── Step 8: Return to NORMAL ─────────────────────────────────────────────────────────────
    echo ""
    state_set state "NORMAL"
    state_set fallback_start "0"
    state_set handback_strikes "0"
    state_set tier2_started "false"
    state_set tier3_started "false"
    state_set tier4_started "false"

    # Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback
    local_ddns_start

    warn "$ICON_DONE Handback complete — returned to NORMAL"
    notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
        "Fallback" "normal"
}

# ==============================================================================================
# ━━━ Main State Machine ━━━
# ==============================================================================================
state_init
set_ddns_arrays

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_FALLBACK FALLBACK — $(date '+%Y-%m-%d %H:%M:%S')"
echo "  $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "  $ICON_NET  Remote IP: $REMOTE_SERVER"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

FALLBACK_RUNNING=true
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
    SIGTERM SIGINT

while [[ "$FALLBACK_RUNNING" == true ]]; do
    NOW=$(date +%s)
    CURRENT_STATE=$(state_get state)

    # ── Connectivity checks ──────────────────────────────────────────────────────────────────
    REMOTE_UP=false
    INTERNET_UP=false

    ping_remote   && REMOTE_UP=true
    ping_internet && INTERNET_UP=true

    log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"

    # ── Partnership gate ──────────────────────────────────────────────────────────────────────
    # When FALLBACK_PARTNERSHIP_REQUIRED=true, a fallback without an active partner is
    # meaningless — who are we covering for? The gate suspends the loop after a grace
    # period and aborts any live fallback containers. Clears automatically when partnership
    # is restored.
    if [[ "${FALLBACK_PARTNERSHIP_REQUIRED:-true}" == "true" ]]; then
        if check_partnership_active; then
            # Partnership healthy — clear any accumulated grace state
            if [[ "$(state_get partnership_suspended)" == "true" ]]; then
                warn "Active partnership restored — re-enabling fallback monitoring"
                state_set partnership_suspended "false"
                state_set partner_lost_at "0"
                notify "Fallback re-enabled on $(hostname) — active partnership restored" \
                    "Fallback" "normal"
            elif [[ "$(state_get partner_lost_at)" != "0" ]]; then
                log "Partnership recovered within grace window — resetting timer"
                state_set partner_lost_at "0"
            fi
        else
            PARTNER_LOST_AT=$(state_get partner_lost_at)
            if [[ -z "$PARTNER_LOST_AT" || "$PARTNER_LOST_AT" == "0" ]]; then
                warn "No active partnership detected — grace timer started (${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min before suspend)"
                state_set partner_lost_at "$NOW"
                PARTNER_LOST_AT=$NOW
            fi

            PARTNER_LOST_MIN=$(( (NOW - PARTNER_LOST_AT) / 60 ))
            SUSPEND_AFTER=${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}

            if [[ "$(state_get partnership_suspended)" == "true" || "$PARTNER_LOST_MIN" -ge "$SUSPEND_AFTER" ]]; then
                # Suspend — abort any live fallback containers, then skip this cycle entirely
                if [[ "$(state_get partnership_suspended)" != "true" ]]; then
                    warn "No active partnership for ${PARTNER_LOST_MIN}min — suspending fallback"
                    state_set partnership_suspended "true"
                    notify "Fallback suspended on $(hostname) — no active partnership for ${PARTNER_LOST_MIN}min" \
                        "Fallback" "warning"
                fi
                if [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
                    warn "Aborting in-progress fallback — partnership suspended"
                    _abort_fallback_containers
                    state_set state "NORMAL"
                    state_set fallback_start "0"
                    state_set handback_strikes "0"
                    notify "Fallback aborted on $(hostname) — no active partnership, containers stopped" \
                        "Fallback" "warning"
                fi
                log "Partnership suspended — skipping fallback cycle"
                sleep "$FALLBACK_CHECK_INTERVAL" & wait $!
                continue
            else
                log "No active partnership — ${PARTNER_LOST_MIN}min elapsed / ${SUSPEND_AFTER}min until suspend"
            fi
        fi
    fi

    # ════════════════════════════════════════════════════════════════
    # NORMAL STATE
    # ════════════════════════════════════════════════════════════════
    if [[ "$CURRENT_STATE" == "NORMAL" ]]; then

        if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
            log "$ICON_SUCCESS NORMAL — all systems up"

        elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
            # Remote is down — enter FALLBACK immediately
            echo ""
            echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
            warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"

            state_set state "FALLBACK"
            state_set fallback_start "$NOW"
            state_set handback_strikes "0"
            state_set tier2_started "false"
            state_set tier3_started "false"
            state_set tier4_started "false"

            echo ""
            echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
            read -r -a tier1 <<< "$(get_tier_containers 1)"
            for container in "${tier1[@]}"; do
                [[ -n "$container" ]] && local_start "$container"
            done

            notify "FALLBACK started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \
                "Fallback" "warning"

        elif [[ "$INTERNET_UP" == false ]]; then
            # Lost internet — enter NO_INTERNET
            echo ""
            echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
            warn "Internet connectivity lost — stopping local DDNS"

            state_set state "NO_INTERNET"
            local_ddns_stop

            # Stop containers configured to stop on internet loss
            read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
            for container in "${stop_on_no_net[@]}"; do
                [[ -n "$container" ]] && local_stop "$container"
            done

            notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \
                "Fallback" "warning"
        fi

    # ════════════════════════════════════════════════════════════════
    # FALLBACK STATE
    # ════════════════════════════════════════════════════════════════
    elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then

        FALLBACK_START_TS=$(state_get fallback_start)
        ELAPSED_MIN=$(( (NOW - FALLBACK_START_TS) / 60 ))

        if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
            # Remote returned — increment handback strikes
            STRIKES=$(state_get handback_strikes)
            STRIKES=$(( STRIKES + 1 ))
            state_set handback_strikes "$STRIKES"

            warn "$ICON_FALLBACK $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FALLBACK_HANDBACK_STRIKES"

            if [[ "$STRIKES" -ge "$FALLBACK_HANDBACK_STRIKES" ]]; then
                run_handback
            fi

        elif [[ "$INTERNET_UP" == false ]]; then
            # Lost internet during fallback — enter DARK
            echo ""
            echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
            warn "Lost internet during fallback — entering DARK state"
            state_set state "DARK"
            local_ddns_stop
            notify "DARK state on $(hostname) — lost internet during fallback" \
                "Fallback" "warning"

        else
            # Still in FALLBACK state — check tier escalation
            state_set handback_strikes "0"

            TIER2_DELAY=$(get_tier2_delay)
            if [[ "$(state_get tier2_started)" == "false" && \
                  "$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
                echo ""
                echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
                read -r -a tier2 <<< "$(get_tier_containers 2)"
                for container in "${tier2[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                state_set tier2_started "true"
                notify "Fallback Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \
                    "Fallback" "warning"
            fi

            TIER3_DELAY=$(get_tier3_delay)
            if [[ "$(state_get tier3_started)" == "false" && \
                  "$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
                echo ""
                echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
                read -r -a tier3 <<< "$(get_tier_containers 3)"
                for container in "${tier3[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                state_set tier3_started "true"
                notify "Fallback Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \
                    "Fallback" "warning"
            fi

            TIER4_DELAY=$(get_tier4_delay)
            if [[ "$(state_get tier4_started)" == "false" && \
                  "$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
                echo ""
                echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
                read -r -a tier4 <<< "$(get_tier_containers 4)"
                for container in "${tier4[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                state_set tier4_started "true"
                notify "Fallback Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \
                    "Fallback" "warning"
            fi

            log "$ICON_FALLBACK FALLBACK — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
        fi

    # ════════════════════════════════════════════════════════════════
    # NO_INTERNET STATE
    # ════════════════════════════════════════════════════════════════
    elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then

        if [[ "$INTERNET_UP" == true ]]; then
            echo ""
            echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

            if [[ "$REMOTE_UP" == true ]]; then
                warn "Remote is up — returning to NORMAL"
                state_set state "NORMAL"
                local_ddns_start
                notify "Internet recovered on $(hostname) — returning to NORMAL" \
                    "Fallback" "normal"
            else
                warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
                state_set state "FALLBACK"
                state_set fallback_start "$NOW"
                state_set handback_strikes "0"
                state_set tier2_started "false"
                state_set tier3_started "false"
                state_set tier4_started "false"

                echo ""
                echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
                read -r -a tier1 <<< "$(get_tier_containers 1)"
                for container in "${tier1[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FALLBACK" \
                    "Fallback" "warning"
            fi
        else
            log "$ICON_NET NO_INTERNET — waiting for connectivity"
        fi

    # ════════════════════════════════════════════════════════════════
    # DARK STATE
    # ════════════════════════════════════════════════════════════════
    elif [[ "$CURRENT_STATE" == "DARK" ]]; then

        if [[ "$INTERNET_UP" == true ]]; then
            echo ""
            echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

            if [[ "$REMOTE_UP" == true ]]; then
                warn "Remote up, internet up — transitioning through FALLBACK for handback"
                # Was in FALLBACK state before DARK — route through FALLBACK state for handback
                state_set state "FALLBACK"
                state_set handback_strikes "0"
            else
                warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
                state_set state "FALLBACK"
                state_set fallback_start "$NOW"
                state_set handback_strikes "0"
                state_set tier2_started "false"
                state_set tier3_started "false"
                state_set tier4_started "false"

                read -r -a tier1 <<< "$(get_tier_containers 1)"
                for container in "${tier1[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
            fi
            notify "Emerging from DARK state on $(hostname)" "Fallback" "warning"
        else
            log "$ICON_FALLBACK DARK — no internet, no remote — waiting"
        fi
    fi

    # ── Sleep until next check ───────────────────────────────────────────────────────────────
    log "Next check in ${FALLBACK_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
    sleep "$FALLBACK_CHECK_INTERVAL" &
    wait $!

done