Varaverk/Fallback/fallback_test.sh

#!/bin/bash
# ==============================================================================================
# ================================= Fallback Test ==============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Controlled simulation of the fallback lifecycle. Validates the entire sequence
# without waiting for a real outage. Contains no fallback logic — exercises the
# real fallback.sh via an iptables DROP rule on the remote Tailscale IP.
#
# Run during a maintenance window. Users will experience a brief service
# interruption. Use --dry-run to walk through all phases without real changes.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Phase 1 — Pre-flight         Both servers reachable, Docker daemons healthy,
#                               version parity, fallback.sh exists, state NORMAL
# Phase 2 — Block Remote        iptables DROP rule added — remote appears unreachable
# Phase 3 — Fallback Detection  Wait FALLBACK_TEST_BLOCK_WAIT for fallback.sh to
#                               detect the outage and enter FALLBACK state
# Phase 4 — Container Start     Verify Tier 1 containers started locally
# Phase 5 — Restore             iptables rule removed — remote reachable again
# Phase 6 — Handback            Wait FALLBACK_TEST_HANDBACK_WAIT for fallback.sh to
#                               complete full handback and return to NORMAL
# Phase 7 — Container Handback  Verify Tier 1 containers stopped locally
# Report  — Full pass/fail per phase with timing
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Test Harness Only
#   Contains zero fallback logic. All fallback is exercised through fallback.sh.
#   Any change to fallback.sh is automatically reflected in the test result.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# iptables Safety Trap
#   The DROP rule is removed via an EXIT trap that fires on normal completion, error
#   exit, script crash, ctrl-c (SIGINT) and SIGTERM — verified, not assumed. Remote
#   connectivity is restored regardless of test outcome.
#
# Stale Rule Sweep
#   The trap above cannot cover SIGKILL or a power cut, which are the only ways a DROP
#   rule survives the test. One stranded that way makes fallback.sh see the partner as
#   permanently down and hold FALLBACK indefinitely, so pre-flight clears any leftover
#   rule before doing anything else — including before the reachability check, which
#   would otherwise fail and blame the network for the test's own residue.
#
# Root Enforcement
#   iptables and container control require root.
#
# iptables Presence Check
#   platform_require_cmd confirms iptables exists before the test begins — there is no
#   point entering a connectivity simulation that cannot simulate anything.
#
# FALLBACK_ENABLED Gate
#   Aborts if FALLBACK_ENABLED=false. Testing a disabled fallback system is
#   misleading and potentially destructive.
#
# State Must Be NORMAL
#   Pre-flight fails if state is not NORMAL. Running a test during an actual
#   fallback event would interfere with the real event.
#
# Version Parity Check
#   Pre-flight verifies unRAID version parity before any iptables rules are
#   added. A mismatch makes the test result unreliable.
#
# Remote Docker Daemon Check
#   Pre-flight confirms remote Docker daemon is responsive before Phase 2.
#
# Lock Acquisition
#   acquire_lock() prevents concurrent test runs. Running two tests simultaneously
#   would produce conflicting iptables rules and unreliable results.
#
# Host Detection
#   detect_hosts() resolves MY_ID / REMOTE_ID from master.conf at startup.
#   Exits if the host cannot be identified — prevents testing on an unknown machine.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   FALLBACK_TEST_BLOCK_WAIT
#       Seconds to wait in Phase 3 for fallback.sh to detect the outage.
#       Must be > FALLBACK_CHECK_INTERVAL + buffer. At 30s interval: use ≥60s.
#       (shipped default: 150)
#
#   FALLBACK_TEST_HANDBACK_WAIT
#       Seconds to wait in Phase 6 for fallback.sh to complete handback.
#       Must cover: FALLBACK_HANDBACK_STRIKES × FALLBACK_CHECK_INTERVAL + rsync
#       duration + container start time. At 3 strikes × 30s + ~2min rsync +
#       ~1min container start: use ≥240s. (shipped default: 360)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# fallback_test.sh --dry-run
#     Walk through all 7 phases with output but no iptables changes and no
#     container starts/stops. ALWAYS run this before a live test.
#
# fallback_test.sh
#     Full live test — real iptables DROP rule, real container lifecycle.
#     Users will experience a brief service interruption. Run during a
#     maintenance window.
#
# fallback_test.sh --status
#     Show current fallback state and test timing configuration. No test run.
#
# fallback_test.sh --log
#     Verbose output on every check in every phase.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
DOCKER_TIMEOUT=15

# ==============================================================================================
# ── SAFETY TRAP — always remove iptables rule on exit ─────────────────────────────────────────
# ==============================================================================================
# Fires on normal exit, error exit, ctrl-c, and script crashes.
# Remote connectivity is ALWAYS restored regardless of test outcome.

IPTABLES_RULE_ACTIVE=false

cleanup() {
    if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
        echo ""
        warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
        if [[ "$DRY_RUN" == false ]]; then
            iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
            IPTABLES_RULE_ACTIVE=false
            warn "iptables rule removed — remote connectivity restored"
        else
            warn "DRY RUN — would remove iptables rule"
        fi
    fi
}

trap cleanup EXIT

# ── Stale rule sweep — the one case the trap above cannot cover ───────────────────────────────
# The EXIT trap fires on normal exit, error, ctrl-c and SIGTERM, but not on SIGKILL or a power
# cut. A DROP rule stranded that way makes fallback.sh see the partner as permanently down and
# sit in FALLBACK indefinitely — so clear any leftover from a previous run before starting.
_clear_stale_block() {
    [[ -z "${REMOTE_SERVER:-}" ]] && return
    local removed=0
    while iptables -C OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null; do
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would remove stale iptables block on $REMOTE_SERVER"
            return
        fi
        iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null || break
        (( removed++ ))
    done
    if [[ "$removed" -gt 0 ]]; then
        warn "$ICON_SHIELD Removed $removed stale iptables block(s) on $REMOTE_SERVER from a previous run"
        notify "Fallback test cleared $removed stale iptables block(s) on $(hostname) — a previous test was killed before cleanup" \
            "Fallback Test" "warning"
    fi
}

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# FALLBACK_ENABLED gate — no point testing if fallback is disabled
# Fail-closed, matching fallback.sh — anything not exactly "true" counts as disabled.
if [[ "${FALLBACK_ENABLED:-false}" != "true" ]]; then
    warn "FALLBACK_ENABLED=false — fallback test aborted"
    warn "Enable fallback in master.conf before running this test"
    exit 0
fi

acquire_lock   # strict single instance — modifies iptables and containers

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts
resolve_remote_ip

# Validate commands used by this script
platform_require_cmd \
    "$(which iptables 2>/dev/null || echo /sbin/iptables)" \
    "--version" "iptables" \
    "iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; }


if [[ ! -f "$FALLBACK_SCRIPT" ]]; then
    error "fallback.sh not found at $FALLBACK_SCRIPT"
    exit 1
fi
log "fallback.sh found at $FALLBACK_SCRIPT"
log "$ICON_GEAR Config: remote=${REMOTE_SERVER_NAME} (${REMOTE_SERVER}) fallback-script=${FALLBACK_SCRIPT}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")

    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST      My ID:         $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_HOST      Remote ID:     $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
    echo "$ICON_GEAR      OS ver:        $local_ver"
    echo "$ICON_FALLBACK  Block wait:    ${FALLBACK_TEST_BLOCK_WAIT}s"
    echo "$ICON_FALLBACK  Handback wait: ${FALLBACK_TEST_HANDBACK_WAIT}s"
    echo "$ICON_FALLBACK  Check interval: ${FALLBACK_CHECK_INTERVAL}s"
    echo "$ICON_FALLBACK  Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
    echo "$ICON_GEAR      Dry Run:       $DRY_RUN"

    if [[ -f "$FALLBACK_STATE_FILE" ]]; then
        CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
        echo "$ICON_FALLBACK  Current state: ${CURRENT_STATE:-unknown}"
    else
        echo "$ICON_FALLBACK  Current state: no state file"
    fi

    # Show Tier 1 containers for this host
    TIER1_VAR="FALLBACK_${REMOTE_ID}_TIER1"
    eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
    echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── PHASE TRACKING ────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
PHASES_PASS=()
PHASES_FAIL=()
TOTAL_START=$(date +%s)

phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE  Phase: $1 — PASSED ✅"; }
phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; }

# Get Tier 1 containers defined in remote's own conf
TIER1_VAR="FALLBACK_${REMOTE_ID}_TIER1"
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"

# ==============================================================================================
# ━━━ Phase 1 — Pre-flight ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_SHIELD FALLBACK TEST — $(date '+%Y-%m-%d %H:%M:%S')"
echo "  $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"

# Must run before the reachability check below — a stale DROP rule from a killed run makes
# the partner look unreachable, and the test would abort blaming the network for its own
# leftover.
_clear_stale_block

# Remote reachable
if ping_remote; then
    log "$REMOTE_SERVER_NAME is reachable"
else
    error "$REMOTE_SERVER_NAME is not reachable — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Internet reachable
if ping_internet; then
    log "Internet is reachable"
else
    error "No internet connectivity — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Version parity — test may produce misleading results on mismatch
if ! check_os_version_parity; then
    error "unRAID version mismatch — test aborted to prevent misleading results"
    phase_fail "Pre-flight"
    exit 1
fi

# Remote Docker daemon — must be responsive before test manipulates containers
if ! check_remote_docker_daemon; then
    error "Remote Docker daemon not responsive — cannot run test"
    phase_fail "Pre-flight"
    exit 1
fi

# Fallback state must be NORMAL before test
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
    CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
    if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
        error "Fallback state is $CURRENT_STATE — must be NORMAL before running test"
        phase_fail "Pre-flight"
        exit 1
    fi
    log "Fallback state is NORMAL"
else
    warn "No state file found — assuming NORMAL (first run)"
fi

# Tier 1 containers configured
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
    error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
    error "Check FALLBACK_${REMOTE_ID}_TIER1 in ${REMOTE_ID,,}.conf"
    phase_fail "Pre-flight"
    exit 1
fi
log "Tier 1 containers: ${TIER1_CONTAINERS[*]}"

phase_pass "Pre-flight"

# ==============================================================================================
# ━━━ Phase 2 — Block Remote Connectivity ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)"

if [[ "$DRY_RUN" == false ]]; then
    iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
    IPTABLES_RULE_ACTIVE=true

    # Verify block is working
    sleep 2
    if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
        log "Connectivity block confirmed — ping to remote fails as expected"
        phase_pass "Block Remote"
    else
        error "iptables rule did not block connectivity — ping still succeeds"
        phase_fail "Block Remote"
        exit 1
    fi
else
    warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule"
    phase_pass "Block Remote"
fi

# ==============================================================================================
# ━━━ Phase 3 — Fallback Detection ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FALLBACK Phase 3 — Fallback Detection ━━━"
warn "Waiting ${FALLBACK_TEST_BLOCK_WAIT}s for fallback.sh to detect outage..."
log "fallback.sh check interval: ${FALLBACK_CHECK_INTERVAL}s"

if [[ "$DRY_RUN" == false ]]; then
    sleep "$FALLBACK_TEST_BLOCK_WAIT"

    if [[ -f "$FALLBACK_STATE_FILE" ]]; then
        NEW_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
        if [[ "$NEW_STATE" == "FALLBACK" ]]; then
            echo "State changed to FALLBACK — outage detected correctly ✅"
            phase_pass "Fallback Detection"
        else
            error "State is $NEW_STATE — expected FALLBACK after ${FALLBACK_TEST_BLOCK_WAIT}s"
            warn "Is fallback.sh running? Check User Scripts plugin"
            phase_fail "Fallback Detection"
        fi
    else
        error "No state file found after wait — fallback.sh may not be running"
        phase_fail "Fallback Detection"
    fi
else
    warn "DRY RUN — would wait ${FALLBACK_TEST_BLOCK_WAIT}s then check for FALLBACK state"
    phase_pass "Fallback Detection"
fi

# ==============================================================================================
# ━━━ Phase 4 — Container Start Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━"
log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}"

if [[ "$DRY_RUN" == false ]]; then
    CONTAINERS_OK=true
    for container in "${TIER1_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
            "$container" 2>/dev/null)
        if [[ "$STATUS" == "true" ]]; then
            echo "$ICON_RUNNING $container is running locally ✅"
        else
            error "$ICON_NOT_RUNNING $container is NOT running locally"
            CONTAINERS_OK=false
        fi
    done

    if [[ "$CONTAINERS_OK" == true ]]; then
        phase_pass "Container Start"
    else
        phase_fail "Container Start"
    fi
else
    warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}"
    phase_pass "Container Start"
fi

# ==============================================================================================
# ━━━ Phase 5 — Restore Remote Connectivity ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again"

if [[ "$DRY_RUN" == false ]]; then
    iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
    IPTABLES_RULE_ACTIVE=false

    sleep 3
    if ping_remote; then
        echo "$REMOTE_SERVER_NAME is reachable again ✅"
        phase_pass "Restore Connectivity"
    else
        error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
        phase_fail "Restore Connectivity"
    fi
else
    warn "DRY RUN — would remove iptables rule"
    phase_pass "Restore Connectivity"
fi

# ==============================================================================================
# ━━━ Phase 6 — Handback ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_FALLBACK Phase 6 — Handback ━━━"
warn "Waiting ${FALLBACK_TEST_HANDBACK_WAIT}s for fallback.sh to complete handback..."
log "Requires $FALLBACK_HANDBACK_STRIKES consecutive checks at ${FALLBACK_CHECK_INTERVAL}s"
log "Minimum handback time: $(( FALLBACK_HANDBACK_STRIKES * FALLBACK_CHECK_INTERVAL ))s"

if [[ "$DRY_RUN" == false ]]; then
    sleep "$FALLBACK_TEST_HANDBACK_WAIT"

    if [[ -f "$FALLBACK_STATE_FILE" ]]; then
        FINAL_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
        if [[ "$FINAL_STATE" == "NORMAL" ]]; then
            echo "State returned to NORMAL — handback completed ✅"
            phase_pass "Handback"
        else
            error "State is $FINAL_STATE — expected NORMAL after ${FALLBACK_TEST_HANDBACK_WAIT}s"
            warn "Handback may still be in progress — check fallback.sh output"
            phase_fail "Handback"
        fi
    else
        error "No state file found"
        phase_fail "Handback"
    fi
else
    warn "DRY RUN — would wait ${FALLBACK_TEST_HANDBACK_WAIT}s then verify NORMAL state"
    phase_pass "Handback"
fi

# ==============================================================================================
# ━━━ Phase 7 — Container Handback Verification ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━"
log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME"

if [[ "$DRY_RUN" == false ]]; then
    HANDBACK_OK=true
    for container in "${TIER1_CONTAINERS[@]}"; do
        [[ -z "$container" ]] && continue
        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
            "$container" 2>/dev/null)
        if [[ "$STATUS" != "true" ]]; then
            echo "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
        else
            error "$ICON_RUNNING $container still running locally — handback may have failed"
            HANDBACK_OK=false
        fi
    done

    if [[ "$HANDBACK_OK" == true ]]; then
        phase_pass "Container Handback"
    else
        phase_fail "Container Handback"
    fi
else
    warn "DRY RUN — would verify Tier 1 containers stopped locally after handback"
    phase_pass "Container Handback"
fi

TOTAL_END=$(date +%s)

# ==============================================================================================
# ━━━ Test Report ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY FALLBACK TEST REPORT ━━━━━"
echo "$ICON_HOST    My ID:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_HOST    Remote:   $REMOTE_ID ($REMOTE_SERVER_NAME)"
echo "$ICON_TIME    Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
echo ""
echo "  Phase Results:"
for phase in "${PHASES_PASS[@]}"; do
    echo "    $ICON_SUCCESS $phase"
done
for phase in "${PHASES_FAIL[@]}"; do
    echo "    $ICON_ERROR $phase"
done
echo ""

PASS_COUNT=${#PHASES_PASS[@]}
FAIL_COUNT=${#PHASES_FAIL[@]}
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
    warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED"
    notify "Fallback test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \
        "Fallback Test" "normal"
else
    error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
    notify "Fallback test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \
        "Fallback Test" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
exit 0