#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Failover Script --------------------------------------------
# -----------------------------------------------------------------------------------------------
# Mutual container failover between two unRAID servers.
# Runs continuously on BOTH servers — each operates fully autonomously.
# No coordination between servers — decisions based solely on ping results.
#
# States:
#   NORMAL      — remote up, internet up — own containers only
#   FAILOVER    — remote down, internet up — own + remote's containers (additive)
#   NO_INTERNET — internet down — stop public-facing containers
#   DARK        — remote down + internet down — same as NO_INTERNET
#
# Handback (remote returns after FAILOVER):
#   Strike confirmation → pre-flight checks → rsync → start on remote → stop locally
#
# All configuration in Master.conf under Failover section.
# Run via User Scripts plugin as a background task — runs continuously until stopped.
# Supports --dry-run to walk through logic without taking any action.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

detect_hosts
resolve_remote_ip

# Select correct arrays based on which server we are
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
    FAILOVER_START_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
    FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST1_STOP_ON_NO_NET[@]}")
    FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST1_RSYNC_JOBS[@]}")
else
    FAILOVER_START_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
    FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST2_STOP_ON_NO_NET[@]}")
    FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST2_RSYNC_JOBS[@]}")
fi

info "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}"
info "$ICON_STOP     Stop on no-net:      ${FAILOVER_STOP_ON_NO_NET[*]}"
info "$ICON_SYNC     Rsync jobs:          ${FAILOVER_RSYNC_JOBS[*]}"

# Ensure state file directory exists
touch "$FAILOVER_STATE_FILE" 2>/dev/null || {
    error "Cannot create state file: $FAILOVER_STATE_FILE"
    exit 1
}

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━"
    echo "$ICON_HOST      Local:              $LOCAL_SERVER_NAME"
    echo "$ICON_NET       Remote:             $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
    echo "$ICON_PING      External IP:        $EXTERNAL_IP"
    echo "$ICON_TIME      Check interval:     ${FAILOVER_CHECK_INTERVAL}s"
    echo "$ICON_RETRY     Handback strikes:   $FAILOVER_HANDBACK_STRIKES"
    echo "$ICON_FAILOVER  Failover containers: ${FAILOVER_START_CONTAINERS[*]}"
    echo "$ICON_STOP      Stop on no-net:     ${FAILOVER_STOP_ON_NO_NET[*]}"
    echo "$ICON_SYNC      Rsync jobs:         ${FAILOVER_RSYNC_JOBS[*]}"
    echo "$ICON_GEAR      State file:         $FAILOVER_STATE_FILE"
    echo "$ICON_GEAR      Dry Run:            $DRY_RUN"

    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        echo ""
        echo "━━━ Persisted State ━━━"
        cat "$FAILOVER_STATE_FILE"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be started or stopped"

# -----------------------------------------------------------------------------------------------
# STATE FILE HELPERS
# State persists to /boot/ so it survives reboots.
# On restart the script re-evaluates from scratch using live ping results.
# State file is reference only — pings are always the source of truth.
# -----------------------------------------------------------------------------------------------

get_state() {
    grep -E "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2
}

set_state() {
    local new_state="$1"
    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    cat > "$FAILOVER_STATE_FILE" <<EOF
state=$new_state
last_change=$timestamp
host=$LOCAL_SERVER_NAME
EOF
    log "State set to $new_state"
}

get_handback_strikes() {
    grep -E "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2
}

set_handback_strikes() {
    local count="$1"
    if grep -q "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null; then
        sed -i "s/^handback_strikes=.*/handback_strikes=$count/" "$FAILOVER_STATE_FILE"
    else
        echo "handback_strikes=$count" >> "$FAILOVER_STATE_FILE"
    fi
}

# -----------------------------------------------------------------------------------------------
# CONTAINER HELPERS — LOCAL operations for failover
# These operate on the LOCAL server unlike stop_containers/start_containers in common.sh
# which operate on the REMOTE server via SSH.
# -----------------------------------------------------------------------------------------------

# Start a container locally — skip if already running
start_local_container() {
    local container="$1"

    local status
    status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")

    if [[ "$status" == "true" ]]; then
        log "$container already running locally — skipping"
        return 0
    fi

    if [[ "$status" == "unknown" ]]; then
        warn "$container not found on this host — skipping"
        return 1
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would start $container locally"
        return 0
    fi

    echo "$ICON_START Starting $container locally..."
    if docker start "$container" >/dev/null 2>&1; then
        echo "$ICON_STARTED $container started"
        return 0
    else
        error "Failed to start $container locally"
        return 1
    fi
}

# Stop a container locally — skip if already stopped
stop_local_container() {
    local container="$1"

    local status
    status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")

    if [[ "$status" == "false" ]]; then
        log "$container already stopped locally — skipping"
        return 0
    fi

    if [[ "$status" == "unknown" ]]; then
        log "$container not found on this host — skipping"
        return 0
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would stop $container locally"
        return 0
    fi

    echo "$ICON_STOP Stopping $container locally..."
    if docker stop "$container" >/dev/null 2>&1; then
        echo "$ICON_STOPPED $container stopped"
        return 0
    else
        error "Failed to stop $container locally"
        return 1
    fi
}

# Start a container on the remote server via SSH
start_remote_container() {
    local container="$1"

    local status
    status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
        "docker inspect -f '{{.State.Running}}' $container 2>/dev/null || echo unknown" 2>/dev/null)

    if [[ "$status" == "true" ]]; then
        log "$container already running on remote — skipping"
        return 0
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
        return 0
    fi

    echo "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
    if ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
        "docker start $container" >/dev/null 2>&1; then
        echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME"
        return 0
    else
        error "Failed to start $container on $REMOTE_SERVER_NAME"
        return 1
    fi
}

# -----------------------------------------------------------------------------------------------
# STATE ACTIONS
# Each state has a corresponding action function.
# These are idempotent — safe to call on every loop iteration.
# -----------------------------------------------------------------------------------------------

# NORMAL — remote is up, internet is up
# Ensure failover containers are stopped locally (cleanup after returning from FAILOVER)
action_normal() {
    local prev_state="$1"

    if [[ "$prev_state" == "FAILOVER" ]]; then
        info "$ICON_FAILOVER Returning from FAILOVER — cleaning up local failover containers"
        for c in "${FAILOVER_START_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            stop_local_container "$c"
        done
        notify "Failover ended on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME is back online" "Failover" "normal"
    fi

    # Ensure no-net containers are running (they may have been stopped)
    if [[ "$prev_state" == "NO_INTERNET" || "$prev_state" == "DARK" ]]; then
        info "$ICON_START Internet restored — starting previously stopped containers"
        for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do
            [[ -z "$c" ]] && continue
            start_local_container "$c"
        done
        notify "Internet restored on $LOCAL_SERVER_NAME — normal containers restarted" "Failover" "normal"
    fi
}

# FAILOVER — remote is down, internet is up
# Start remote's containers locally (additive — own containers keep running)
action_failover() {
    local prev_state="$1"

    if [[ "$prev_state" != "FAILOVER" ]]; then
        # Just entered failover — notify and start containers
        info "$ICON_FAILOVER Entering FAILOVER — $REMOTE_SERVER_NAME is unreachable"

        # Check local array before starting containers
        if ! check_local_array; then
            error "Local array not ready — cannot start failover containers safely"
            notify "Failover triggered but local array not ready on $LOCAL_SERVER_NAME" "Failover" "warning"
            return 1
        fi

        # Check local docker daemon
        if ! timeout 10 docker ps >/dev/null 2>&1; then
            error "Local Docker daemon not responding — cannot start failover containers"
            notify "Failover triggered but Docker not ready on $LOCAL_SERVER_NAME" "Failover" "warning"
            return 1
        fi

        notify "Failover ACTIVE on $LOCAL_SERVER_NAME — starting ${REMOTE_SERVER_NAME} containers: ${FAILOVER_START_CONTAINERS[*]}" "Failover" "warning"
    fi

    # Start each failover container — idempotent, skips if already running
    for c in "${FAILOVER_START_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue
        start_local_container "$c"
    done
}

# NO_INTERNET / DARK — this server has no internet
# Stop public-facing containers — no point serving if offline
action_no_internet() {
    local prev_state="$1"

    if [[ "$prev_state" != "NO_INTERNET" && "$prev_state" != "DARK" ]]; then
        warn "$ICON_WARN Internet lost on $LOCAL_SERVER_NAME — stopping public containers"
        notify "Internet lost on $LOCAL_SERVER_NAME — stopping public-facing containers" "Failover" "warning"
    fi

    for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do
        [[ -z "$c" ]] && continue
        stop_local_container "$c"
    done

    # If we were in FAILOVER, also stop the failover containers
    if [[ "$prev_state" == "FAILOVER" ]]; then
        info "Was in FAILOVER — stopping failover containers too"
        for c in "${FAILOVER_START_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            stop_local_container "$c"
        done
    fi
}

# HANDBACK — remote has returned, ready to hand containers back
# Confirmed by FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
action_handback() {
    info "$ICON_FAILOVER Initiating handback to $REMOTE_SERVER_NAME..."
    notify "Handback starting on $LOCAL_SERVER_NAME — syncing data to $REMOTE_SERVER_NAME" "Failover" "normal"

    # Pre-flight checks before rsync
    echo ""
    echo "━━━ $ICON_SHIELD Handback Pre-flight ━━━"

    if ! check_remote_array; then
        error "Remote array not ready — deferring handback"
        notify "Handback deferred on $LOCAL_SERVER_NAME — remote array not ready" "Failover" "warning"
        set_handback_strikes 0
        return 1
    fi
    success "Remote array ready"

    if ! check_remote_docker; then
        error "Remote Docker not ready — deferring handback"
        notify "Handback deferred on $LOCAL_SERVER_NAME — remote Docker not ready" "Failover" "warning"
        set_handback_strikes 0
        return 1
    fi
    success "Remote Docker ready"

    REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
        "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
    if [[ -n "$REMOTE_USAGE" ]] && [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
        error "Remote rootfs ${REMOTE_USAGE}% full — deferring handback"
        notify "Handback deferred — remote rootfs ${REMOTE_USAGE}% full on $REMOTE_SERVER_NAME" "Failover" "warning"
        set_handback_strikes 0
        return 1
    fi
    success "Remote rootfs healthy"

    # Rsync data back via rsync.sh — uses existing profile system
    echo ""
    echo "━━━ $ICON_SYNC Handback Rsync ━━━"
    local rsync_failed=false

    for path in "${FAILOVER_RSYNC_JOBS[@]}"; do
        [[ -z "$path" ]] && continue
        info "Syncing $path → $REMOTE_SERVER_NAME"

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would rsync $path"
        else
            if bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$path" --no-log; then
                success "Rsync complete: $path"
            else
                error "Rsync failed: $path"
                rsync_failed=true
            fi
        fi
    done

    if [[ "$rsync_failed" == true ]]; then
        error "One or more rsync jobs failed — deferring container handback"
        notify "Handback rsync failed on $LOCAL_SERVER_NAME — containers not handed back yet" "Failover" "warning"
        set_handback_strikes 0
        return 1
    fi

    # Start containers on remote
    echo ""
    echo "━━━ $ICON_START $ICON_CONTAINERS Start on Remote ━━━"
    local start_failed=false

    for c in "${FAILOVER_START_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue
        if ! start_remote_container "$c"; then
            start_failed=true
        fi
    done

    if [[ "$start_failed" == true ]]; then
        error "One or more containers failed to start on remote — not stopping local copies"
        notify "Handback partial failure on $LOCAL_SERVER_NAME — some containers failed to start on $REMOTE_SERVER_NAME" "Failover" "warning"
        set_handback_strikes 0
        return 1
    fi

    # Stop local failover containers — only after remote confirmed started
    echo ""
    echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Local Failover ━━━"
    for c in "${FAILOVER_START_CONTAINERS[@]}"; do
        [[ -z "$c" ]] && continue
        stop_local_container "$c"
    done

    notify "Handback complete on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME has containers back" "Failover" "normal"
    set_handback_strikes 0
    return 0
}

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_FAILOVER Main Loop ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_FAILOVER Failover Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST  Local:    $LOCAL_SERVER_NAME"
echo "$ICON_NET   Remote:   $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "$ICON_TIME  Interval: ${FAILOVER_CHECK_INTERVAL}s"
echo ""

notify "Failover script started on $LOCAL_SERVER_NAME — monitoring $REMOTE_SERVER_NAME" "Failover" "normal"

LOOP_COUNT=0

while true; do
    LOOP_COUNT=$((LOOP_COUNT + 1))
    CURRENT_STATE=$(get_state)
    [[ -z "$CURRENT_STATE" ]] && CURRENT_STATE="UNKNOWN"

    echo ""
    echo "━━━ $ICON_PING Check #${LOOP_COUNT} — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "Previous state: $CURRENT_STATE"

    # ── Two ping checks — the only inputs to state decisions ──
    REMOTE_UP=false
    INTERNET_UP=false

    ping_remote  && REMOTE_UP=true
    ping_internet && INTERNET_UP=true

    log "Remote reachable: $REMOTE_UP | Internet reachable: $INTERNET_UP"

    # ── Determine new state ──
    if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
        NEW_STATE="NORMAL"
    elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
        NEW_STATE="FAILOVER"
    elif [[ "$INTERNET_UP" == false ]]; then
        NEW_STATE="NO_INTERNET"
    else
        NEW_STATE="DARK"
    fi

    # ── Handle handback strike system ──
    # When returning from FAILOVER to NORMAL, require consecutive confirmations
    if [[ "$CURRENT_STATE" == "FAILOVER" && "$NEW_STATE" == "NORMAL" ]]; then
        STRIKES=$(get_handback_strikes)
        [[ -z "$STRIKES" ]] && STRIKES=0
        STRIKES=$((STRIKES + 1))
        set_handback_strikes "$STRIKES"

        if [[ "$STRIKES" -lt "$FAILOVER_HANDBACK_STRIKES" ]]; then
            info "$ICON_RETRY Remote appears back — strike $STRIKES/$FAILOVER_HANDBACK_STRIKES — waiting for confirmation"
            NEW_STATE="FAILOVER"  # Stay in FAILOVER until strikes confirmed
        else
            info "$ICON_RETRY Remote confirmed stable ($STRIKES/$FAILOVER_HANDBACK_STRIKES) — initiating handback"
            NEW_STATE="HANDBACK"
        fi
    else
        # Reset handback strikes if we're not in the confirmation window
        set_handback_strikes 0
    fi

    # ── Log state ──
    if [[ "$NEW_STATE" != "$CURRENT_STATE" ]]; then
        echo "$ICON_FAILOVER State: $CURRENT_STATE → $NEW_STATE"
    else
        info "State: $NEW_STATE (unchanged)"
    fi

    # ── Execute state action ──
    case "$NEW_STATE" in
        NORMAL)
            action_normal "$CURRENT_STATE"
            set_state "NORMAL"
            ;;
        FAILOVER)
            action_failover "$CURRENT_STATE"
            set_state "FAILOVER"
            ;;
        NO_INTERNET|DARK)
            action_no_internet "$CURRENT_STATE"
            set_state "$NEW_STATE"
            ;;
        HANDBACK)
            if action_handback; then
                set_state "NORMAL"
                action_normal "FAILOVER"
            else
                # Handback failed — stay in FAILOVER, retry next cycle
                set_state "FAILOVER"
            fi
            ;;
    esac

    echo "$ICON_TIME Next check in ${FAILOVER_CHECK_INTERVAL}s"
    sleep "$FAILOVER_CHECK_INTERVAL"
done