#!/bin/bash
# ==============================================================================================
# ================================= Failover ===================================================
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs this script independently — no direct coordination between servers.
# All decisions based solely on two pings: remote reachable + internet reachable.
#
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
# Both servers run this script continuously as a background task via User Scripts.
# Every FAILOVER_CHECK_INTERVAL seconds each server:
#   1. Pings the remote server
#   2. Pings the internet
#   3. Determines its current state
#   4. Takes the appropriate action
#
# No SSH signaling, no shared state files, no coordination — each server acts autonomously
# based only on what it can see from its own network perspective.
#
# ── STATES ────────────────────────────────────────────────────────────────────────────────────
#   NORMAL      — remote up, internet up
#                 Own containers only. DDNS ON. Silent operation.
#
#   FAILOVER    — remote down, internet up
#                 Start remote containers locally — tiered by outage duration.
#                 Remote DDNS started immediately (Tier 1).
#                 Own containers keep running — failover is additive.
#
#   NO_INTERNET — internet down (remote may be up or down)
#                 Stop own DDNS immediately — can't update DNS without internet.
#                 Do not start remote containers — no internet = no point.
#                 Wait for recovery.
#
#   DARK        — remote down AND internet down
#                 Same actions as NO_INTERNET.
#                 Cannot determine if remote is truly down or just unreachable.
#
# ── DDNS RULES — ABSOLUTE ─────────────────────────────────────────────────────────────────────
#   Each server owns its own DDNS — ON when that server has internet.
#   Script controls DDNS exclusively — network state NEVER auto-starts DDNS.
#   DDNS only starts after full handback sequence confirms containers are up.
#   One DDNS per domain active at all times — never two, never zero for long.
#   1 minute TTL + 1 minute check interval = minimal user impact on failover.
#
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
#   When remote returns after FAILOVER:
#   1. Strike confirmation     — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
#   2. Pre-flight checks       — version parity, remote array, remote Docker daemon
#   3. Stop remote DDNS first  — prevents split brain DNS during rsync
#   4. Stop remote containers  — clean state before rsync
#   5. Tiered rsync writeback  — skip if outage under threshold
#   6. Start remote containers — in dependency order with verification
#   7. Start remote DDNS last  — DNS cuts back ONLY after containers confirmed up
#   8. Return to NORMAL
#
# ── TIERED FAILOVER ───────────────────────────────────────────────────────────────────────────
#   Tier 1 — Immediate         — vital services + Live TV — cannot wait
#   Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr)
#   Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr)
#   Tier 4 — HOST*_TIER4_DELAY — arrs + downloaders (default 24hr)
#   Tier delays configurable per host in master_host*.conf
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   FAILOVER_ENABLED gate   — exits cleanly if disabled in master.conf
#   Version parity check    — refuses handback if unRAID versions mismatch
#   Remote Docker daemon    — checks remote daemon before issuing any remote commands
#   Timeout protection      — all docker and SSH commands wrapped in timeouts
#   Container verification  — verifies containers came up after start (failover critical)
#   MY_ID-based routing     — all tier/DDNS/writeback arrays selected via MY_ID not hostname
#   Command validation      — validates unRAID notify script before use
#   Silent by default       — state transitions warn(), routine cycle checks log()
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
#   HOST*_DDNS_CONTAINERS              — DDNS containers this host manages
#   FAILOVER_HOST*_STOP_ON_NO_NET      — containers stopped on internet loss
#   FAILOVER_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other
#   HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes
#   HOST*_TIER1_WRITEBACK_DELAY        — skip Tier 1 writeback if outage under this
#   FAILOVER_HOST*_WRITEBACK_TIER1-4   — paths synced back on handback per tier
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   FAILOVER_ENABLED           — false = exit cleanly (HOST2 being rebuilt etc.)
#   FAILOVER_CHECK_INTERVAL    — seconds between checks
#   FAILOVER_HANDBACK_STRIKES  — consecutive remote-up checks before handback
#   FAILOVER_STATE_FILE        — /boot/config path — survives reboots
#   FAILOVER_RSYNC_ENABLED     — gate for writeback rsync jobs
#   EXTERNAL_IP                — IP to ping for internet check (default 8.8.8.8)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   failover.sh              — normal start (continuous loop)
#   failover.sh --dry-run    — preview state changes without acting on containers
#   failover.sh --status     — show current state and exit
#   failover.sh --log        — verbose cycle output
#
# ── TO STOP THIS SCRIPT ───────────────────────────────────────────────────────────────────────
#   Click Abort in unRAID User Scripts — do NOT kill directly, state file may corrupt.
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# FAILOVER_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt)
if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then
    warn "FAILOVER_ENABLED=false — failover monitoring disabled"
    warn "Set FAILOVER_ENABLED=true in master.conf when both servers are ready"
    exit 0
fi

acquire_lock   # strict single instance — notifies on collision

detect_hosts
resolve_remote_ip

# Validate unRAID notify script — used throughout for state change notifications
validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"

# Timeout for all docker and SSH docker commands
DOCKER_TIMEOUT=15
SSH_TIMEOUT=10
CONTAINER_VERIFY_WAIT=5   # seconds after start before verifying container is up

# ==============================================================================================
# ── STATE FILE HELPERS ────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# State file on /boot/config — survives reboots.
# Format: key=value one per line.
# Keys: state, failover_start, handback_strikes, tier2_started, tier3_started, tier4_started

state_get() {
    grep "^${1}=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2
}

state_set() {
    local key="$1" value="$2"
    if grep -q "^${key}=" "$FAILOVER_STATE_FILE" 2>/dev/null; then
        sed -i "s|^${key}=.*|${key}=${value}|" "$FAILOVER_STATE_FILE"
    else
        echo "${key}=${value}" >> "$FAILOVER_STATE_FILE"
    fi
}

state_init() {
    mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
    [[ ! -f "$FAILOVER_STATE_FILE" ]] && touch "$FAILOVER_STATE_FILE"
    [[ -z "$(state_get state)" ]]             && state_set state "NORMAL"
    [[ -z "$(state_get failover_start)" ]]    && state_set failover_start "0"
    [[ -z "$(state_get handback_strikes)" ]]  && state_set handback_strikes "0"
    [[ -z "$(state_get tier2_started)" ]]     && state_set tier2_started "false"
    [[ -z "$(state_get tier3_started)" ]]     && state_set tier3_started "false"
    [[ -z "$(state_get tier4_started)" ]]     && state_set tier4_started "false"
}

# ==============================================================================================
# ── CONTAINER HELPERS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# All docker commands wrapped in DOCKER_TIMEOUT.
# All SSH docker commands wrapped in SSH_TIMEOUT.
# Verification after start — failover is critical, confirm containers came up.

# Start a container locally and verify it came up
local_start() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
        "$container" 2>/dev/null)
    if [[ "$status" == "true" ]]; then
        log "$container already running locally"
        return 0
    fi
    log "$ICON_START Starting $container locally..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would start $container locally"
        return 0
    fi
    if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
        sleep "$CONTAINER_VERIFY_WAIT"
        local post_status
        post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
            "$container" 2>/dev/null)
        if [[ "$post_status" == "true" ]]; then
            log "$ICON_STARTED $container started and running ✅"
            return 0
        else
            error "$container started but crashed immediately"
            notify "$container failed to stay running during failover on $(hostname)" \
                "Failover" "warning"
            return 1
        fi
    else
        error "Failed to start $container locally"
        notify "Failed to start $container locally during failover on $(hostname)" \
            "Failover" "warning"
        return 1
    fi
}

# Stop a container locally
local_stop() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
        "$container" 2>/dev/null)
    if [[ "$status" != "true" ]]; then
        log "$container already stopped locally"
        return 0
    fi
    log "$ICON_STOP Stopping $container locally..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would stop $container locally"
        return 0
    fi
    timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
        log "$ICON_STOPPED $container stopped" || \
        error "Failed to stop $container locally"
}

# Start a container on remote via SSH and verify it came up
remote_start() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
        $container 2>/dev/null" 2>/dev/null)
    if [[ "$status" == "true" ]]; then
        log "$container already running on $REMOTE_SERVER_NAME"
        return 0
    fi
    log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
        return 0
    fi
    if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then
        sleep "$CONTAINER_VERIFY_WAIT"
        local post_status
        post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \
            "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
            $container 2>/dev/null" 2>/dev/null)
        if [[ "$post_status" == "true" ]]; then
            log "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅"
            return 0
        else
            error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
            notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \
                "Failover" "warning"
            return 1
        fi
    else
        error "Failed to start $container on $REMOTE_SERVER_NAME"
        notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \
            "Failover" "warning"
        return 1
    fi
}

# Stop a container on remote via SSH
remote_stop() {
    local container="$1"
    [[ -z "$container" ]] && return
    local status
    status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
        $container 2>/dev/null" 2>/dev/null)
    if [[ "$status" != "true" ]]; then
        log "$container already stopped on $REMOTE_SERVER_NAME"
        return 0
    fi
    log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
        return 0
    fi
    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
        root@"$REMOTE_SERVER" \
        "timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
        log "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
        error "Failed to stop $container on $REMOTE_SERVER_NAME"
}

# ==============================================================================================
# ── DDNS HELPERS ──────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# DDNS managed exclusively by this script — never by network state alone.
# Stopping DDNS on internet loss prevents split-brain DNS updates.

local_ddns_stop() {
    warn "$ICON_NET Stopping local DDNS — internet lost"
    for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
        local_stop "$container"
    done
}

local_ddns_start() {
    warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up"
    for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
        local_start "$container"
    done
}

remote_ddns_stop() {
    warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync"
    for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
        remote_stop "$container"
    done
}

# ==============================================================================================
# ── TIERED FAILOVER HELPERS ───────────────────────────────────────────────────────────────────
# ==============================================================================================
# All routing uses MY_ID — not hostname string comparison.
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
# "This server covers the other" — MY_ID selects which tier arrays to use.

get_tier_containers() {
    local tier="$1"
    local remote_id="${REMOTE_ID}"
    local var_name="FAILOVER_${MY_ID}_RUNS_FOR_${remote_id}_TIER${tier}"
    eval "echo \"\${${var_name}[@]:-}\""
}

get_tier2_delay() {
    local var_name="${REMOTE_ID}_TIER2_DELAY"
    echo "${!var_name:-240}"
}

get_tier3_delay() {
    local var_name="${REMOTE_ID}_TIER3_DELAY"
    echo "${!var_name:-720}"
}

get_tier4_delay() {
    local var_name="${REMOTE_ID}_TIER4_DELAY"
    echo "${!var_name:-1440}"
}

get_tier1_writeback_delay() {
    local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY"
    echo "${!var_name:-60}"
}

get_writeback_jobs_for_tier() {
    local tier="$1"
    local remote_id="${REMOTE_ID}"
    if [[ "$tier" -eq 4 ]]; then
        # Tier 4 — daily sync shares of the remote + any extra writeback paths
        local shares_var="${remote_id}_DAILY_SYNC_SHARES"
        local extra_var="FAILOVER_${remote_id}_WRITEBACK_TIER4"
        eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\""
    else
        local var_name="FAILOVER_${remote_id}_WRITEBACK_TIER${tier}"
        eval "echo \"\${${var_name}[@]:-}\""
    fi
}

# Select DDNS arrays based on MY_ID
set_ddns_arrays() {
    local local_ddns_var="${MY_ID}_DDNS_CONTAINERS"
    local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS"
    eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")"
    eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")"
}

# Select internet-loss stop list based on MY_ID
get_stop_on_no_net() {
    local var_name="FAILOVER_${MY_ID}_STOP_ON_NO_NET"
    eval "echo \"\${${var_name}[@]:-}\""
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    state_init
    set_ddns_arrays
    CURRENT_STATE=$(state_get state)
    FAILOVER_START_TS=$(state_get failover_start)
    TIER2=$(state_get tier2_started)
    TIER3=$(state_get tier3_started)
    TIER4=$(state_get tier4_started)
    STRIKES=$(state_get handback_strikes)

    local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")

    echo ""
    echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━"
    echo "$ICON_HOST    My ID:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_HOST    Remote ID:   $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
    echo "$ICON_GEAR    unRAID ver:  $local_ver"
    echo "$ICON_FAILOVER State:      $CURRENT_STATE"
    echo "$ICON_NET     Local DDNS:  ${LOCAL_DDNS_CONTAINERS[*]:-none}"
    echo "$ICON_NET     Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}"
    echo "$ICON_TIME    Interval:    ${FAILOVER_CHECK_INTERVAL}s"
    echo "$ICON_FAILOVER Strikes:    $STRIKES / $FAILOVER_HANDBACK_STRIKES"

    if [[ "$CURRENT_STATE" == "FAILOVER" && "$FAILOVER_START_TS" -gt 0 ]]; then
        ELAPSED=$(( ($(date +%s) - FAILOVER_START_TS) / 60 ))
        echo "$ICON_TIME    Outage:      ${ELAPSED}min"
        echo "$ICON_FAILOVER Tier 2:     $TIER2 (delay: $(get_tier2_delay)min)"
        echo "$ICON_FAILOVER Tier 3:     $TIER3 (delay: $(get_tier3_delay)min)"
        echo "$ICON_FAILOVER Tier 4:     $TIER4 (delay: $(get_tier4_delay)min)"
    fi

    echo "$ICON_GEAR    Enabled:     $FAILOVER_ENABLED"
    echo "$ICON_GEAR    Dry Run:     $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Called when remote returns after FAILOVER state.
# CRITICAL sequencing — do not reorder without understanding the consequences.
# Each step must succeed before proceeding — aborts and retries next cycle on failure.

run_handback() {
    echo ""
    echo "━━━ $ICON_FAILOVER Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    warn "$REMOTE_SERVER_NAME has returned — beginning handback"

    # ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
    echo ""
    echo "━━━ $ICON_SHIELD Pre-flight ━━━"

    # Version parity — refuse handback if unRAID versions mismatch
    if ! check_unraid_version_parity; then
        warn "Version parity check failed — aborting handback, will retry next cycle"
        state_set handback_strikes 0
        return 1
    fi

    # Remote array must be mounted before rsync
    if ! check_remote_array; then
        warn "Remote array not ready — aborting handback, will retry next cycle"
        state_set handback_strikes 0
        return 1
    fi

    # Remote Docker daemon must be responsive before issuing container commands
    if ! check_remote_docker_daemon; then
        warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
        state_set handback_strikes 0
        return 1
    fi

    log "Pre-flight checks passed ✅"

    # ── Step 2: Stop remote DDNS FIRST ──────────────────────────────────────────────────────
    # CRITICAL — prevents split-brain DNS while data is being synced
    echo ""
    echo "━━━ $ICON_NET DDNS Handoff ━━━"
    remote_ddns_stop
    sleep 5  # brief pause to ensure DDNS stop propagates before rsync

    # ── Step 3: Stop remote containers ──────────────────────────────────────────────────────
    # Clean state before rsync — no dirty writes during transfer window
    echo ""
    echo "━━━ $ICON_STOP Stop Remote Containers ━━━"

    local ALL_FAILOVER_CONTAINERS=()

    # Collect all started tiers in reverse order (highest tier stops first)
    if [[ "$(state_get tier4_started)" == "true" ]]; then
        read -r -a t4 <<< "$(get_tier_containers 4)"
        ALL_FAILOVER_CONTAINERS+=("${t4[@]}")
    fi
    if [[ "$(state_get tier3_started)" == "true" ]]; then
        read -r -a t3 <<< "$(get_tier_containers 3)"
        ALL_FAILOVER_CONTAINERS+=("${t3[@]}")
    fi
    if [[ "$(state_get tier2_started)" == "true" ]]; then
        read -r -a t2 <<< "$(get_tier_containers 2)"
        ALL_FAILOVER_CONTAINERS+=("${t2[@]}")
    fi

    # Tier 1 always started — stop last (DDNS already handled above)
    read -r -a t1 <<< "$(get_tier_containers 1)"
    for container in "${t1[@]}"; do
        local is_ddns=false
        for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
            [[ "$container" == "$ddns" ]] && is_ddns=true && break
        done
        [[ "$is_ddns" == false ]] && ALL_FAILOVER_CONTAINERS+=("$container")
    done

    for container in "${ALL_FAILOVER_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        local_stop "$container"
    done

    log "All failover containers stopped locally"

    # ── Step 4: Rsync writeback ──────────────────────────────────────────────────────────────
    # Tiered writeback with skip window — short outages do not benefit from writeback.
    # After a short outage primary's clean nightly state is more reliable than dirty
    # accumulation — skip writeback entirely for outages under the threshold.
    echo ""
    echo "━━━ $ICON_SYNC Rsync Writeback ━━━"

    local outage_minutes
    outage_minutes=$(( ($(date +%s) - $(state_get failover_start)) / 60 ))
    local tier1_wb_delay
    tier1_wb_delay=$(get_tier1_writeback_delay)

    warn "Outage duration: ${outage_minutes}min"

    if ! check_rsync_enabled "FAILOVER"; then
        warn "FAILOVER_RSYNC_ENABLED=false — skipping all writeback jobs"
        warn "Handback will complete without syncing state back to primary"
    else
        run_writeback_tier() {
            local tier="$1" threshold="$2" label="$3"
            local jobs
            read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
            [[ ${#jobs[@]} -eq 0 ]] && return

            if [[ "$outage_minutes" -ge "$threshold" ]]; then
                warn "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min"
                for job in "${jobs[@]}"; do
                    [[ -z "$job" ]] && continue
                    log "Syncing: $job"
                    if [[ "$DRY_RUN" == false ]]; then
                        if [[ "$(basename "$job")" == "Emby" ]]; then
                            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-failover
                        else
                            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
                        fi
                    else
                        warn "DRY RUN — would rsync: $job"
                    fi
                done
            else
                log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min — primary state is cleaner"
            fi
        }

        local t2_delay t3_delay t4_delay
        t2_delay=$(get_tier2_delay)
        t3_delay=$(get_tier3_delay)
        t4_delay=$(get_tier4_delay)

        run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack"
        run_writeback_tier 2 "$t2_delay"       "productivity services"
        run_writeback_tier 3 "$t3_delay"       "secondary services"
        run_writeback_tier 4 "$t4_delay"       "media shares + edge cases"
    fi

    log "Writeback complete"

    # ── Step 5: Start remote containers ─────────────────────────────────────────────────────
    # Start in tier order with dependency awareness — databases before apps
    echo ""
    echo "━━━ $ICON_START Start Remote Containers ━━━"

    # Tier 1 — excluding DDNS (handled separately as final step)
    for container in "${t1[@]}"; do
        [[ -z "$container" ]] && continue
        local is_ddns=false
        for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
            [[ "$container" == "$ddns" ]] && is_ddns=true && break
        done
        [[ "$is_ddns" == false ]] && remote_start "$container"
    done

    sleep 10  # give databases time to initialise before apps

    [[ "$(state_get tier2_started)" == "true" ]] && \
        for container in "${t2[@]}"; do
            [[ -n "$container" ]] && remote_start "$container"
        done

    [[ "$(state_get tier3_started)" == "true" ]] && \
        for container in "${t3[@]}"; do
            [[ -n "$container" ]] && remote_start "$container"
        done

    [[ "$(state_get tier4_started)" == "true" ]] && \
        for container in "${t4[@]}"; do
            [[ -n "$container" ]] && remote_start "$container"
        done

    log "Remote containers started"

    # ── Step 6: Start remote DDNS LAST ──────────────────────────────────────────────────────
    # DNS cuts over ONLY after containers confirmed up — this is the final step
    echo ""
    echo "━━━ $ICON_NET DNS Cutover ━━━"
    sleep 5  # brief pause to ensure containers are accepting connections
    for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
        remote_start "$container"
    done
    warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"

    # ── Step 7: Return to NORMAL ─────────────────────────────────────────────────────────────
    echo ""
    state_set state "NORMAL"
    state_set failover_start "0"
    state_set handback_strikes "0"
    state_set tier2_started "false"
    state_set tier3_started "false"
    state_set tier4_started "false"

    warn "$ICON_DONE Handback complete — returned to NORMAL"
    notify "Failover handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
        "Failover" "normal"
}

# ==============================================================================================
# ━━━ Main State Machine ━━━
# ==============================================================================================
state_init
set_ddns_arrays

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_FAILOVER FAILOVER — $(date '+%Y-%m-%d %H:%M:%S')"
echo "  $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "  $ICON_NET  Remote IP: $REMOTE_SERVER"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

FAILOVER_RUNNING=true
trap 'FAILOVER_RUNNING=false; warn "Failover received shutdown signal — stopping cleanly"; exit 0' \
    SIGTERM SIGINT

while [[ "$FAILOVER_RUNNING" == true ]]; do
    NOW=$(date +%s)
    CURRENT_STATE=$(state_get state)

    # ── Connectivity checks ──────────────────────────────────────────────────────────────────
    REMOTE_UP=false
    INTERNET_UP=false

    ping_remote   && REMOTE_UP=true
    ping_internet && INTERNET_UP=true

    log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"

    # ════════════════════════════════════════════════════════════════
    # NORMAL STATE
    # ════════════════════════════════════════════════════════════════
    if [[ "$CURRENT_STATE" == "NORMAL" ]]; then

        if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
            log "$ICON_SUCCESS NORMAL — all systems up"

        elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
            # Remote is down — enter FAILOVER
            echo ""
            echo "━━━ $ICON_FAILOVER Entering FAILOVER — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
            warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting failover"

            state_set state "FAILOVER"
            state_set failover_start "$NOW"
            state_set handback_strikes "0"
            state_set tier2_started "false"
            state_set tier3_started "false"
            state_set tier4_started "false"

            echo ""
            echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
            read -r -a tier1 <<< "$(get_tier_containers 1)"
            for container in "${tier1[@]}"; do
                [[ -n "$container" ]] && local_start "$container"
            done

            notify "FAILOVER started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \
                "Failover" "warning"

        elif [[ "$INTERNET_UP" == false ]]; then
            # Lost internet — enter NO_INTERNET
            echo ""
            echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
            warn "Internet connectivity lost — stopping local DDNS"

            state_set state "NO_INTERNET"
            local_ddns_stop

            # Stop containers configured to stop on internet loss
            read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
            for container in "${stop_on_no_net[@]}"; do
                [[ -n "$container" ]] && local_stop "$container"
            done

            notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \
                "Failover" "warning"
        fi

    # ════════════════════════════════════════════════════════════════
    # FAILOVER STATE
    # ════════════════════════════════════════════════════════════════
    elif [[ "$CURRENT_STATE" == "FAILOVER" ]]; then

        FAILOVER_START_TS=$(state_get failover_start)
        ELAPSED_MIN=$(( (NOW - FAILOVER_START_TS) / 60 ))

        if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
            # Remote returned — increment handback strikes
            STRIKES=$(state_get handback_strikes)
            STRIKES=$(( STRIKES + 1 ))
            state_set handback_strikes "$STRIKES"

            warn "$ICON_FAILOVER $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FAILOVER_HANDBACK_STRIKES"

            if [[ "$STRIKES" -ge "$FAILOVER_HANDBACK_STRIKES" ]]; then
                run_handback
            fi

        elif [[ "$INTERNET_UP" == false ]]; then
            # Lost internet during failover — enter DARK
            echo ""
            echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
            warn "Lost internet during failover — entering DARK state"
            state_set state "DARK"
            local_ddns_stop
            notify "DARK state on $(hostname) — lost internet during failover" \
                "Failover" "warning"

        else
            # Still in failover — check tier escalation
            state_set handback_strikes "0"

            TIER2_DELAY=$(get_tier2_delay)
            if [[ "$(state_get tier2_started)" == "false" && \
                  "$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
                echo ""
                echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
                read -r -a tier2 <<< "$(get_tier_containers 2)"
                for container in "${tier2[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                state_set tier2_started "true"
                notify "Failover Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \
                    "Failover" "warning"
            fi

            TIER3_DELAY=$(get_tier3_delay)
            if [[ "$(state_get tier3_started)" == "false" && \
                  "$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
                echo ""
                echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
                read -r -a tier3 <<< "$(get_tier_containers 3)"
                for container in "${tier3[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                state_set tier3_started "true"
                notify "Failover Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \
                    "Failover" "warning"
            fi

            TIER4_DELAY=$(get_tier4_delay)
            if [[ "$(state_get tier4_started)" == "false" && \
                  "$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
                echo ""
                echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
                read -r -a tier4 <<< "$(get_tier_containers 4)"
                for container in "${tier4[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                state_set tier4_started "true"
                notify "Failover Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \
                    "Failover" "warning"
            fi

            log "$ICON_FAILOVER FAILOVER — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
        fi

    # ════════════════════════════════════════════════════════════════
    # NO_INTERNET STATE
    # ════════════════════════════════════════════════════════════════
    elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then

        if [[ "$INTERNET_UP" == true ]]; then
            echo ""
            echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

            if [[ "$REMOTE_UP" == true ]]; then
                warn "Remote is up — returning to NORMAL"
                state_set state "NORMAL"
                local_ddns_start
                notify "Internet recovered on $(hostname) — returning to NORMAL" \
                    "Failover" "normal"
            else
                warn "Internet back but $REMOTE_SERVER_NAME still down — entering FAILOVER"
                state_set state "FAILOVER"
                state_set failover_start "$NOW"
                state_set handback_strikes "0"
                state_set tier2_started "false"
                state_set tier3_started "false"
                state_set tier4_started "false"

                echo ""
                echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
                read -r -a tier1 <<< "$(get_tier_containers 1)"
                for container in "${tier1[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
                notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FAILOVER" \
                    "Failover" "warning"
            fi
        else
            log "$ICON_NET NO_INTERNET — waiting for connectivity"
        fi

    # ════════════════════════════════════════════════════════════════
    # DARK STATE
    # ════════════════════════════════════════════════════════════════
    elif [[ "$CURRENT_STATE" == "DARK" ]]; then

        if [[ "$INTERNET_UP" == true ]]; then
            echo ""
            echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

            if [[ "$REMOTE_UP" == true ]]; then
                warn "Remote up, internet up — transitioning through FAILOVER for handback"
                # Was in failover before DARK — route through FAILOVER state for handback
                state_set state "FAILOVER"
                state_set handback_strikes "0"
            else
                warn "Internet back but $REMOTE_SERVER_NAME still down — entering FAILOVER"
                state_set state "FAILOVER"
                state_set failover_start "$NOW"
                state_set handback_strikes "0"
                state_set tier2_started "false"
                state_set tier3_started "false"
                state_set tier4_started "false"

                read -r -a tier1 <<< "$(get_tier_containers 1)"
                for container in "${tier1[@]}"; do
                    [[ -n "$container" ]] && local_start "$container"
                done
            fi
            notify "Emerging from DARK state on $(hostname)" "Failover" "warning"
        else
            log "$ICON_FAILOVER DARK — no internet, no remote — waiting"
        fi
    fi

    # ── Sleep until next check ───────────────────────────────────────────────────────────────
    log "Next check in ${FAILOVER_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
    sleep "$FAILOVER_CHECK_INTERVAL" &
    wait $!

done