#!/bin/bash
# ==============================================================================================
# ================================= Failover Test ==============================================
# ==============================================================================================
# Controlled simulation of the failover lifecycle — validates the entire failover sequence
# without waiting for a real outage.
#
# ── WHAT THIS SCRIPT IS ───────────────────────────────────────────────────────────────────────
# A test harness only — contains no failover logic.
# All failover logic lives in failover.sh and is exercised by this test.
# Any changes to failover.sh are automatically reflected here.
#
# ── TEST SEQUENCE ─────────────────────────────────────────────────────────────────────────────
#   Phase 1 — Pre-flight         verify both servers reachable, daemons healthy,
#                                 version parity, failover.sh exists, state is NORMAL
#   Phase 2 — Block Remote       iptables rule drops all traffic to remote IP
#   Phase 3 — Failover Detection wait for failover.sh to detect outage and enter FAILOVER
#   Phase 4 — Container Start    verify Tier 1 failover containers started locally
#   Phase 5 — Restore            remove iptables rule, remote becomes reachable
#   Phase 6 — Handback           wait for failover.sh to complete handback to NORMAL
#   Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback
#   Phase 8 — Report             full pass/fail summary per phase
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   FAILOVER_ENABLED gate   — aborts if failover monitoring is disabled
#   iptables safety trap    — rule ALWAYS removed on exit (crash, error, ctrl-c, normal)
#                             remote connectivity always restored regardless of outcome
#   Version parity check    — pre-flight verifies both servers on compatible unRAID versions
#   Remote Docker daemon    — pre-flight verifies remote daemon is responsive
#   DOCKER_TIMEOUT          — all docker calls protected against daemon hangs
#   MY_ID-based routing     — tier containers selected via MY_ID not hostname comparison
#   Command validation      — iptables and notify validated before use
#   Dry-run safe            — full sequence walkthrough without touching iptables or containers
#
# ── WARNING ───────────────────────────────────────────────────────────────────────────────────
# ⚠️  This script starts and stops REAL containers on both servers.
#     Run during a maintenance window — users will experience a brief service interruption.
#     Use --dry-run to walk through the sequence without any real changes.
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   FAILOVER_TEST_BLOCK_WAIT    — seconds to wait for failover.sh to detect outage
#   FAILOVER_TEST_HANDBACK_WAIT — seconds to wait for failover.sh to complete handback
#   FAILOVER_CHECK_INTERVAL     — check interval of the running failover.sh (informational)
#   FAILOVER_HANDBACK_STRIKES   — strikes required before handback (informational)
#   FAILOVER_STATE_FILE         — state file path to read current state
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   failover_test.sh              — run full test sequence
#   failover_test.sh --dry-run    — walk through all phases without changes
#   failover_test.sh --status     — show current failover state and test config
#   failover_test.sh --log        — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh"
DOCKER_TIMEOUT=15

# ==============================================================================================
# ── SAFETY TRAP — always remove iptables rule on exit ─────────────────────────────────────────
# ==============================================================================================
# Fires on normal exit, error exit, ctrl-c, and script crashes.
# Remote connectivity is ALWAYS restored regardless of test outcome.

IPTABLES_RULE_ACTIVE=false

cleanup() {
    if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
        echo ""
        warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
        if [[ "$DRY_RUN" == false ]]; then
            iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
            IPTABLES_RULE_ACTIVE=false
            warn "iptables rule removed — remote connectivity restored"
        else
            warn "DRY RUN — would remove iptables rule"
        fi
    fi
}

trap cleanup EXIT

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# FAILOVER_ENABLED gate — no point testing if failover is disabled
if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then
    warn "FAILOVER_ENABLED=false — failover test aborted"
    warn "Enable failover in master.conf before running this test"
    exit 0
fi

acquire_lock   # strict single instance — modifies iptables and containers

detect_hosts
resolve_remote_ip

# Validate commands used by this script
validate_unraid_cmd \
    "$(which iptables 2>/dev/null || echo /sbin/iptables)" \
    "--version" "iptables" \
    "iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; }

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

if [[ ! -f "$FAILOVER_SCRIPT" ]]; then
    error "failover.sh not found at $FAILOVER_SCRIPT"
    exit 1
fi
log "failover.sh found at $FAILOVER_SCRIPT"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")

    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST      My ID:         $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_HOST      Remote ID:     $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
    echo "$ICON_GEAR      unRAID ver:    $local_ver"
    echo "$ICON_FAILOVER  Block wait:    ${FAILOVER_TEST_BLOCK_WAIT}s"
    echo "$ICON_FAILOVER  Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s"
    echo "$ICON_FAILOVER  Check interval: ${FAILOVER_CHECK_INTERVAL}s"
    echo "$ICON_FAILOVER  Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
    echo "$ICON_GEAR      Dry Run:       $DRY_RUN"

    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
        echo "$ICON_FAILOVER  Current state: ${CURRENT_STATE:-unknown}"
    else
        echo "$ICON_FAILOVER  Current state: no state file"
    fi

    # Show Tier 1 containers for this host
    TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1"
    eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
    echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── PHASE TRACKING ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
PHASES_PASS=()
PHASES_FAIL=()
TOTAL_START=$(date +%s)

phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE  Phase: $1 — PASSED ✅"; }
phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; }

# Get Tier 1 containers for this server's failover responsibility
TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1"
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"

# ==============================================================================================
# ━━━ Phase 1 — Pre-flight ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')"
echo "  $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"

# Remote reachable
if ping_remote; then
    log "$REMOTE_SERVER_NAME is reachable"
else
    error "$REMOTE_SERVER_NAME is not reachable — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Internet reachable
if ping_internet; then
    log "Internet is reachable"
else
    error "No internet connectivity — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Version parity — test may produce misleading results on mismatch
if ! check_unraid_version_parity; then
    error "unRAID version mismatch — test aborted to prevent misleading results"
    phase_fail "Pre-flight"
    exit 1
fi

# Remote Docker daemon — must be responsive before test manipulates containers
if ! check_remote_docker_daemon; then
    error "Remote Docker daemon not responsive — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Failover state must be NORMAL before test
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
    CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
    if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
        error "Failover state is $CURRENT_STATE — must be NORMAL before running test"
        phase_fail "Pre-flight"
        exit 1
    fi
    log "Failover state is NORMAL"
else
    warn "No state file found — assuming NORMAL (first run)"
fi

# Tier 1 containers configured
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
    error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
    error "Check FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1 in master_host*.conf"
    phase_fail "Pre-flight"
    exit 1
fi
log "Tier 1 containers: ${TIER1_CONTAINERS[*]}"

phase_pass "Pre-flight"

# ==============================================================================================
# ━━━ Phase 2 — Block Remote Connectivity ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)"

if [[ "$DRY_RUN" == false ]]; then
    iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
    IPTABLES_RULE_ACTIVE=true

    # Verify block is working
    sleep 2
    if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
        log "Connectivity block confirmed — ping to remote fails as expected"
        phase_pass "Block Remote"
    else
        error "iptables rule did not block connectivity — ping still succeeds"
        phase_fail "Block Remote"
        exit 1
    fi
else
    warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule"
    phase_pass "Block Remote"
fi

# ==============================================================================================
# ━━━ Phase 3 — Failover Detection ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━"
warn "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
log "failover.sh check interval: ${FAILOVER_CHECK_INTERVAL}s"

if [[ "$DRY_RUN" == false ]]; then
    sleep "$FAILOVER_TEST_BLOCK_WAIT"

    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
        if [[ "$NEW_STATE" == "FAILOVER" ]]; then
            log "State changed to FAILOVER — outage detected correctly ✅"
            phase_pass "Failover Detection"
        else
            error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s"
            warn "Is failover.sh running? Check User Scripts plugin"
            phase_fail "Failover Detection"
        fi
    else
        error "No state file found after wait — failover.sh may not be running"
        phase_fail "Failover Detection"
    fi
else
    warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s then check for FAILOVER state"
    phase_pass "Failover Detection"
fi

# ==============================================================================================
# ━━━ Phase 4 — Container Start Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━"
log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}"

if [[ "$DRY_RUN" == false ]]; then
    CONTAINERS_OK=true
    for container in "${TIER1_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
            "$container" 2>/dev/null)
        if [[ "$STATUS" == "true" ]]; then
            log "$ICON_RUNNING $container is running locally ✅"
        else
            error "$ICON_NOT_RUNNING $container is NOT running locally"
            CONTAINERS_OK=false
        fi
    done

    if [[ "$CONTAINERS_OK" == true ]]; then
        phase_pass "Container Start"
    else
        phase_fail "Container Start"
    fi
else
    warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}"
    phase_pass "Container Start"
fi

# ==============================================================================================
# ━━━ Phase 5 — Restore Remote Connectivity ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again"

if [[ "$DRY_RUN" == false ]]; then
    iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
    IPTABLES_RULE_ACTIVE=false

    sleep 3
    if ping_remote; then
        log "$REMOTE_SERVER_NAME is reachable again ✅"
        phase_pass "Restore Connectivity"
    else
        error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
        phase_fail "Restore Connectivity"
    fi
else
    warn "DRY RUN — would remove iptables rule"
    phase_pass "Restore Connectivity"
fi

# ==============================================================================================
# ━━━ Phase 6 — Handback ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━"
warn "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to complete handback..."
log "Requires $FAILOVER_HANDBACK_STRIKES consecutive checks at ${FAILOVER_CHECK_INTERVAL}s"
log "Minimum handback time: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"

if [[ "$DRY_RUN" == false ]]; then
    sleep "$FAILOVER_TEST_HANDBACK_WAIT"

    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
        if [[ "$FINAL_STATE" == "NORMAL" ]]; then
            log "State returned to NORMAL — handback completed ✅"
            phase_pass "Handback"
        else
            error "State is $FINAL_STATE — expected NORMAL after ${FAILOVER_TEST_HANDBACK_WAIT}s"
            warn "Handback may still be in progress — check failover.sh output"
            phase_fail "Handback"
        fi
    else
        error "No state file found"
        phase_fail "Handback"
    fi
else
    warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s then verify NORMAL state"
    phase_pass "Handback"
fi

# ==============================================================================================
# ━━━ Phase 7 — Container Handback Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━"
log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME"

if [[ "$DRY_RUN" == false ]]; then
    HANDBACK_OK=true
    for container in "${TIER1_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
            "$container" 2>/dev/null)
        if [[ "$STATUS" != "true" ]]; then
            log "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
        else
            error "$ICON_RUNNING $container still running locally — handback may have failed"
            HANDBACK_OK=false
        fi
    done

    if [[ "$HANDBACK_OK" == true ]]; then
        phase_pass "Container Handback"
    else
        phase_fail "Container Handback"
    fi
else
    warn "DRY RUN — would verify Tier 1 containers stopped locally after handback"
    phase_pass "Container Handback"
fi

TOTAL_END=$(date +%s)

# ==============================================================================================
# ━━━ Test Report ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━"
echo "$ICON_HOST    My ID:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST    Remote:   $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_TIME    Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
echo ""
echo "  Phase Results:"
for phase in "${PHASES_PASS[@]}"; do
    echo "    $ICON_SUCCESS $phase"
done
for phase in "${PHASES_FAIL[@]}"; do
    echo "    $ICON_ERROR $phase"
done
echo ""

PASS_COUNT=${#PHASES_PASS[@]}
FAIL_COUNT=${#PHASES_FAIL[@]}
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
    warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED"
    notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \
        "Failover Test" "normal"
else
    error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
    notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \
        "Failover Test" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
exit 0