#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Failover Test ----------------------------------------------
# -----------------------------------------------------------------------------------------------
# Controlled simulation of the failover scenario — validates the entire failover lifecycle
# without waiting for a real outage.
#
# This script is a TEST HARNESS only — it does not contain failover logic.
# All failover logic lives in failover.sh and is called directly from here.
# Any changes to failover.sh are automatically reflected in this test.
#
# Test sequence:
#   1. Pre-flight  — verify both servers reachable, failover.sh exists, state is NORMAL
#   2. Block       — add iptables rule dropping all traffic to remote IP
#   3. Detect      — run failover.sh one cycle — confirm FAILOVER state detected
#   4. Start       — verify failover containers started locally
#   5. Restore     — remove iptables rule, remote becomes reachable again
#   6. Handback    — wait for failover.sh to confirm handback strikes and hand back
#   7. Verify      — confirm containers returned to remote, local copies stopped
#   8. Report      — full pass/fail summary per phase
#
# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal.
# Remote connectivity is always restored regardless of test outcome.
#
# ⚠️  This script starts and stops real containers on both servers.
#     Run during a maintenance window — users will experience a brief service interruption.
#     Use --dry-run to walk through the sequence without touching containers or iptables.
#
# All configuration in Master.conf under Failover and Failover Test sections.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh"

# -----------------------------------------------------------------------------------------------
# SAFETY TRAP — always remove iptables rule on exit
# Fires on normal exit, error exit, ctrl-c, and script crashes
# -----------------------------------------------------------------------------------------------
IPTABLES_RULE_ACTIVE=false

cleanup() {
    if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
        echo ""
        warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
        if [[ "$DRY_RUN" == false ]]; then
            iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
            IPTABLES_RULE_ACTIVE=false
            success "iptables rule removed — remote connectivity restored"
        else
            warn "DRY RUN — would remove iptables rule"
        fi
    fi
}

trap cleanup EXIT

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

if ! command -v iptables >/dev/null 2>&1; then
    error "iptables not found — required for connectivity simulation"
    exit 1
fi

success "iptables available"

if [[ ! -f "$FAILOVER_SCRIPT" ]]; then
    error "failover.sh not found at $FAILOVER_SCRIPT"
    exit 1
fi

success "failover.sh found"

detect_hosts
resolve_remote_ip

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST      Local:         $LOCAL_SERVER_NAME"
    echo "$ICON_HOST      Remote:        $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
    echo "$ICON_FAILOVER  Block wait:    ${FAILOVER_TEST_BLOCK_WAIT}s"
    echo "$ICON_FAILOVER  Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s"
    echo "$ICON_GEAR      Dry Run:       $DRY_RUN"

    # Current failover state
    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
        echo "$ICON_FAILOVER  Current state: ${CURRENT_STATE:-unknown}"
    else
        echo "$ICON_FAILOVER  Current state: no state file"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# -----------------------------------------------------------------------------------------------
# PHASE TRACKING
# -----------------------------------------------------------------------------------------------
PHASES_PASS=()
PHASES_FAIL=()
TOTAL_START=$(date +%s)

phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; }
phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; }

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 1 — Pre-flight ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')"
echo "  $ICON_HOST Local:  $LOCAL_SERVER_NAME"
echo "  $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"

# Check remote reachable
info "Checking remote reachability..."
if ping_remote; then
    success "Remote $REMOTE_SERVER_NAME is reachable"
else
    error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Check internet reachable
info "Checking internet connectivity..."
if ping_internet; then
    success "Internet is reachable"
else
    error "No internet connectivity — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Check current failover state is NORMAL
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
    CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
    if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
        error "Failover state is $CURRENT_STATE — must be NORMAL before running test"
        phase_fail "Pre-flight"
        exit 1
    fi
    success "Failover state is NORMAL"
else
    warn "No state file found — assuming NORMAL (first run)"
fi

phase_pass "Pre-flight"

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 2 — Block Remote ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER"

if [[ "$DRY_RUN" == false ]]; then
    iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
    IPTABLES_RULE_ACTIVE=true
    success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable"

    # Verify block is working
    if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
        success "Connectivity block confirmed — ping to remote fails as expected"
        phase_pass "Block Remote"
    else
        error "iptables rule did not block connectivity — ping still succeeds"
        phase_fail "Block Remote"
        exit 1
    fi
else
    warn "DRY RUN — would block $REMOTE_SERVER with iptables"
    phase_pass "Block Remote"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 3 — Failover Detection ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━"
info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s"

if [[ "$DRY_RUN" == false ]]; then
    sleep "$FAILOVER_TEST_BLOCK_WAIT"

    # Check state file updated to FAILOVER
    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
        if [[ "$NEW_STATE" == "FAILOVER" ]]; then
            success "State changed to FAILOVER — outage detected correctly"
            phase_pass "Failover Detection"
        else
            error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s"
            warn "failover.sh may not be running — check User Scripts plugin"
            phase_fail "Failover Detection"
        fi
    else
        error "No state file found after wait — failover.sh may not be running"
        phase_fail "Failover Detection"
    fi
else
    warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state"
    phase_pass "Failover Detection"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 4 — Container Start Verification ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━"

# Determine which containers should have started on this host
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
    EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
else
    EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
fi

if [[ "$DRY_RUN" == false ]]; then
    CONTAINERS_OK=true
    for container in "${EXPECTED_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
        if [[ "$STATUS" == "true" ]]; then
            success "$ICON_RUNNING $container is running locally"
        else
            error "$ICON_NOT_RUNNING $container is NOT running locally"
            CONTAINERS_OK=false
        fi
    done

    if [[ "$CONTAINERS_OK" == true ]]; then
        phase_pass "Container Start"
    else
        phase_fail "Container Start"
    fi
else
    warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}"
    phase_pass "Container Start"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 5 — Restore Connectivity ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
info "Removing iptables block — remote becomes reachable again"

if [[ "$DRY_RUN" == false ]]; then
    iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
    IPTABLES_RULE_ACTIVE=false
    success "iptables rule removed"

    # Verify connectivity restored
    sleep 3
    if ping_remote; then
        success "Remote $REMOTE_SERVER_NAME is reachable again"
        phase_pass "Restore Connectivity"
    else
        error "Remote still unreachable after removing iptables rule"
        phase_fail "Restore Connectivity"
    fi
else
    warn "DRY RUN — would remove iptables rule"
    phase_pass "Restore Connectivity"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 6 — Handback ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━"
info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..."
info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals"
info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"

if [[ "$DRY_RUN" == false ]]; then
    sleep "$FAILOVER_TEST_HANDBACK_WAIT"

    if [[ -f "$FAILOVER_STATE_FILE" ]]; then
        FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
        if [[ "$FINAL_STATE" == "NORMAL" ]]; then
            success "State returned to NORMAL — handback completed"
            phase_pass "Handback"
        else
            error "State is $FINAL_STATE — expected NORMAL after handback wait"
            phase_fail "Handback"
        fi
    else
        error "No state file found"
        phase_fail "Handback"
    fi
else
    warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state"
    phase_pass "Handback"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ PHASE 7 — Container Handback Verification ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━"

if [[ "$DRY_RUN" == false ]]; then
    HANDBACK_OK=true
    for container in "${EXPECTED_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
        if [[ "$STATUS" != "true" ]]; then
            success "$ICON_NOT_RUNNING $container stopped locally — handed back"
        else
            error "$ICON_RUNNING $container still running locally — handback may have failed"
            HANDBACK_OK=false
        fi
    done

    if [[ "$HANDBACK_OK" == true ]]; then
        phase_pass "Container Handback"
    else
        phase_fail "Container Handback"
    fi
else
    warn "DRY RUN — would verify failover containers stopped locally after handback"
    phase_pass "Container Handback"
fi

TOTAL_END=$(date +%s)

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Test Report ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━"
echo "$ICON_HOST      Local:    $LOCAL_SERVER_NAME"
echo "$ICON_HOST      Remote:   $REMOTE_SERVER_NAME"
echo "$ICON_TIME      Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
echo ""
echo "  Phase Results:"
for phase in "${PHASES_PASS[@]}"; do
    echo "    $ICON_SUCCESS $phase"
done
for phase in "${PHASES_FAIL[@]}"; do
    echo "    $ICON_ERROR $phase"
done
echo ""

PASS_COUNT=${#PHASES_PASS[@]}
FAIL_COUNT=${#PHASES_FAIL[@]}
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))

if [[ "$DRY_RUN" == true ]]; then
    echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
    echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED"
    notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal"
else
    echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
    notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
exit 0