628 lines
27 KiB
Bash
Executable File
628 lines
27 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= Fallback Test ==============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Controlled simulation of the fallback lifecycle. Validates the entire sequence
|
||
# without waiting for a real outage. Contains no fallback logic — exercises the
|
||
# real fallback.sh via an iptables DROP rule on the remote Tailscale IP.
|
||
#
|
||
# Run during a maintenance window. Users will experience a brief service
|
||
# interruption. Use --dry-run to walk through all phases without real changes.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Phase 1 — Pre-flight Both servers reachable, Docker daemons healthy,
|
||
# version parity, fallback.sh exists, state NORMAL
|
||
# Phase 2 — Block Remote iptables DROP rule added — remote appears unreachable
|
||
# Phase 3 — Fallback Detection Wait FALLBACK_TEST_BLOCK_WAIT for fallback.sh to
|
||
# detect the outage and enter FALLBACK state
|
||
# Phase 4 — Container Start Verify Tier 1 containers started locally
|
||
# Phase 5 — Restore iptables rule removed — remote reachable again
|
||
# Phase 6 — Handback Wait FALLBACK_TEST_HANDBACK_WAIT for fallback.sh to
|
||
# complete full handback and return to NORMAL
|
||
# Phase 7 — Container Handback Verify Tier 1 containers stopped locally
|
||
# Report — Full pass/fail per phase with timing
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Test Harness Only
|
||
# Contains zero fallback logic. All fallback is exercised through fallback.sh.
|
||
# Any change to fallback.sh is automatically reflected in the test result.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# iptables Safety Trap
|
||
# The DROP rule is removed via an EXIT trap that fires on normal completion, error
|
||
# exit, script crash, ctrl-c (SIGINT) and SIGTERM — verified, not assumed. Remote
|
||
# connectivity is restored regardless of test outcome.
|
||
#
|
||
# Stale Rule Sweep
|
||
# The trap above cannot cover SIGKILL or a power cut, which are the only ways a DROP
|
||
# rule survives the test. One stranded that way makes fallback.sh see the partner as
|
||
# permanently down and hold FALLBACK indefinitely, so pre-flight clears any leftover
|
||
# rule before doing anything else — including before the reachability check, which
|
||
# would otherwise fail and blame the network for the test's own residue.
|
||
#
|
||
# Root Enforcement
|
||
# iptables and container control require root.
|
||
#
|
||
# iptables Presence Check
|
||
# platform_require_cmd confirms iptables exists before the test begins — there is no
|
||
# point entering a connectivity simulation that cannot simulate anything.
|
||
#
|
||
# FALLBACK_ENABLED Gate
|
||
# Aborts if FALLBACK_ENABLED=false. Testing a disabled fallback system is
|
||
# misleading and potentially destructive.
|
||
#
|
||
# State Must Be NORMAL
|
||
# Pre-flight fails if state is not NORMAL. Running a test during an actual
|
||
# fallback event would interfere with the real event.
|
||
#
|
||
# Version Parity Check
|
||
# Pre-flight verifies unRAID version parity before any iptables rules are
|
||
# added. A mismatch makes the test result unreliable.
|
||
#
|
||
# Remote Docker Daemon Check
|
||
# Pre-flight confirms remote Docker daemon is responsive before Phase 2.
|
||
#
|
||
# Lock Acquisition
|
||
# acquire_lock() prevents concurrent test runs. Running two tests simultaneously
|
||
# would produce conflicting iptables rules and unreliable results.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() resolves MY_ID / REMOTE_ID from master.conf at startup.
|
||
# Exits if the host cannot be identified — prevents testing on an unknown machine.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# FALLBACK_TEST_BLOCK_WAIT
|
||
# Seconds to wait in Phase 3 for fallback.sh to detect the outage.
|
||
# Must be > FALLBACK_CHECK_INTERVAL + buffer. At 30s interval: use ≥60s.
|
||
# (shipped default: 150)
|
||
#
|
||
# FALLBACK_TEST_HANDBACK_WAIT
|
||
# Seconds to wait in Phase 6 for fallback.sh to complete handback.
|
||
# Must cover: FALLBACK_HANDBACK_STRIKES × FALLBACK_CHECK_INTERVAL + rsync
|
||
# duration + container start time. At 3 strikes × 30s + ~2min rsync +
|
||
# ~1min container start: use ≥240s. (shipped default: 360)
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# fallback_test.sh --dry-run
|
||
# Walk through all 7 phases with output but no iptables changes and no
|
||
# container starts/stops. ALWAYS run this before a live test.
|
||
#
|
||
# fallback_test.sh
|
||
# Full live test — real iptables DROP rule, real container lifecycle.
|
||
# Users will experience a brief service interruption. Run during a
|
||
# maintenance window.
|
||
#
|
||
# fallback_test.sh --status
|
||
# Show current fallback state and test timing configuration. No test run.
|
||
#
|
||
# fallback_test.sh --log
|
||
# Verbose output on every check in every phase.
|
||
#
|
||
# fallback_test.sh --stop
|
||
# Stop a running test. SIGTERM only — never SIGKILL, because only this script's EXIT
|
||
# trap removes the iptables DROP rule it installed. Also sweeps a rule stranded by an
|
||
# earlier SIGKILL or power cut.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||
#
|
||
# SIGTERM ONLY, and deliberately no SIGKILL escalation — the opposite of fallback.sh --stop.
|
||
# A running test holds an iptables DROP rule against the partner, and the only thing that removes
|
||
# it is this script's own EXIT trap. SIGKILL does not run traps, so force-killing a test strands
|
||
# the rule: the partner stays invisible, fallback.sh reads that as a permanent outage and holds
|
||
# FALLBACK indefinitely. A test that will not die is a worse outcome than a test still running,
|
||
# so this reports the stranded rule and the command to clear it rather than causing one.
|
||
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||
LOCKFILE="${LOCK_DIR}/fallback_test.lock"
|
||
if [[ ! -f "$LOCKFILE" ]]; then
|
||
log "No fallback_test.sh lock found — not running"
|
||
exit 0
|
||
fi
|
||
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||
target_pid="${lock_content%%:*}"
|
||
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||
warn "Stale lock — fallback_test.sh not running (PID ${target_pid:-unknown} gone) — clearing"
|
||
rm -f "$LOCKFILE"
|
||
# A stale lock is exactly the SIGKILL/power-cut case, so the rule may still be in place.
|
||
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||
warn "Stranded iptables DROP rule found for $REMOTE_SERVER — removing"
|
||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||
&& warn "Stranded rule removed — remote connectivity restored ✅" \
|
||
|| error "Could not remove stranded rule — run: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||
fi
|
||
exit 0
|
||
fi
|
||
warn "Stopping fallback_test.sh (PID $target_pid) — SIGTERM so its trap clears the iptables rule..."
|
||
kill -TERM "$target_pid" 2>/dev/null || true
|
||
waited=0
|
||
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 30 ]]; do
|
||
sleep 1
|
||
(( waited++ )) || true
|
||
done
|
||
if kill -0 "$target_pid" 2>/dev/null; then
|
||
error "fallback_test.sh (PID $target_pid) did not exit within 30s"
|
||
error "NOT force-killing — SIGKILL would strand the iptables DROP rule on $REMOTE_SERVER"
|
||
error "Wait, or clear manually: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||
exit 1
|
||
fi
|
||
warn "Stopped: fallback_test.sh (PID $target_pid) ✅"
|
||
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||
error "iptables DROP rule for $REMOTE_SERVER survived the stop — removing"
|
||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||
&& warn "Rule removed ✅" || error "Could not remove — run it by hand"
|
||
else
|
||
log "No iptables DROP rule remains for $REMOTE_SERVER ✅"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
||
DOCKER_TIMEOUT=15
|
||
|
||
# ==============================================================================================
|
||
# ── SAFETY TRAP — always remove iptables rule on exit ─────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Fires on normal exit, error exit, ctrl-c, and script crashes.
|
||
# Remote connectivity is ALWAYS restored regardless of test outcome.
|
||
|
||
IPTABLES_RULE_ACTIVE=false
|
||
|
||
cleanup() {
|
||
if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
|
||
echo ""
|
||
warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||
IPTABLES_RULE_ACTIVE=false
|
||
warn "iptables rule removed — remote connectivity restored"
|
||
else
|
||
warn "DRY RUN — would remove iptables rule"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
trap cleanup EXIT
|
||
|
||
# ── Stale rule sweep — the one case the trap above cannot cover ───────────────────────────────
|
||
# The EXIT trap fires on normal exit, error, ctrl-c and SIGTERM, but not on SIGKILL or a power
|
||
# cut. A DROP rule stranded that way makes fallback.sh see the partner as permanently down and
|
||
# sit in FALLBACK indefinitely — so clear any leftover from a previous run before starting.
|
||
_clear_stale_block() {
|
||
[[ -z "${REMOTE_SERVER:-}" ]] && return
|
||
local removed=0
|
||
while iptables -C OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null; do
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove stale iptables block on $REMOTE_SERVER"
|
||
return
|
||
fi
|
||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null || break
|
||
(( removed++ ))
|
||
done
|
||
if [[ "$removed" -gt 0 ]]; then
|
||
warn "$ICON_SHIELD Removed $removed stale iptables block(s) on $REMOTE_SERVER from a previous run"
|
||
notify "Fallback test cleared $removed stale iptables block(s) on $(hostname) — a previous test was killed before cleanup" \
|
||
"Fallback Test" "warning"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# FALLBACK_ENABLED gate — no point testing if fallback is disabled
|
||
# Fail-closed, matching fallback.sh — anything not exactly "true" counts as disabled.
|
||
if [[ "${FALLBACK_ENABLED:-false}" != "true" ]]; then
|
||
warn "FALLBACK_ENABLED=false — fallback test aborted"
|
||
warn "Enable fallback in master.conf before running this test"
|
||
exit 0
|
||
fi
|
||
|
||
acquire_lock # strict single instance — modifies iptables and containers
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
resolve_remote_ip
|
||
|
||
# Validate commands used by this script
|
||
platform_require_cmd \
|
||
"$(which iptables 2>/dev/null || echo /sbin/iptables)" \
|
||
"--version" "iptables" \
|
||
"iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; }
|
||
|
||
|
||
if [[ ! -f "$FALLBACK_SCRIPT" ]]; then
|
||
error "fallback.sh not found at $FALLBACK_SCRIPT"
|
||
exit 1
|
||
fi
|
||
log "fallback.sh found at $FALLBACK_SCRIPT"
|
||
log "$ICON_GEAR Config: remote=${REMOTE_SERVER_NAME} (${REMOTE_SERVER}) fallback-script=${FALLBACK_SCRIPT}"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
|
||
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||
echo "$ICON_GEAR OS ver: $local_ver"
|
||
echo "$ICON_FALLBACK Block wait: ${FALLBACK_TEST_BLOCK_WAIT}s"
|
||
echo "$ICON_FALLBACK Handback wait: ${FALLBACK_TEST_HANDBACK_WAIT}s"
|
||
echo "$ICON_FALLBACK Check interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||
echo "$ICON_FALLBACK Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
|
||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||
|
||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
echo "$ICON_FALLBACK Current state: ${CURRENT_STATE:-unknown}"
|
||
else
|
||
echo "$ICON_FALLBACK Current state: no state file"
|
||
fi
|
||
|
||
# Show Tier 1 containers for this host
|
||
TIER1_VAR="FALLBACK_${REMOTE_ID}_TIER1"
|
||
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
|
||
echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── PHASE TRACKING ────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
PHASES_PASS=()
|
||
PHASES_FAIL=()
|
||
TOTAL_START=$(date +%s)
|
||
|
||
phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE Phase: $1 — PASSED ✅"; }
|
||
phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; }
|
||
|
||
# Get Tier 1 containers defined in remote's own conf
|
||
TIER1_VAR="FALLBACK_${REMOTE_ID}_TIER1"
|
||
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 1 — Pre-flight ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " $ICON_SHIELD FALLBACK TEST — $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
|
||
|
||
# Must run before the reachability check below — a stale DROP rule from a killed run makes
|
||
# the partner look unreachable, and the test would abort blaming the network for its own
|
||
# leftover.
|
||
_clear_stale_block
|
||
|
||
# Remote reachable
|
||
if ping_remote; then
|
||
log "$REMOTE_SERVER_NAME is reachable"
|
||
else
|
||
error "$REMOTE_SERVER_NAME is not reachable — cannot run test"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
|
||
# Internet reachable
|
||
if ping_internet; then
|
||
log "Internet is reachable"
|
||
else
|
||
error "No internet connectivity — cannot run test"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
|
||
# Version parity — test may produce misleading results on mismatch
|
||
if ! check_os_version_parity; then
|
||
error "unRAID version mismatch — test aborted to prevent misleading results"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
|
||
# Remote Docker daemon — must be responsive before test manipulates containers
|
||
if ! check_remote_docker_daemon; then
|
||
error "Remote Docker daemon not responsive — cannot run test"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
|
||
# Fallback state must be NORMAL before test
|
||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
|
||
error "Fallback state is $CURRENT_STATE — must be NORMAL before running test"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
log "Fallback state is NORMAL"
|
||
else
|
||
warn "No state file found — assuming NORMAL (first run)"
|
||
fi
|
||
|
||
# fallback.sh must actually be RUNNING, not merely enabled
|
||
#
|
||
# Every phase after this one waits for the daemon to change state. FALLBACK_ENABLED=true says
|
||
# it is allowed to run; it does not say array_started.sh launched it, or that it is still alive.
|
||
# Without this the test passes pre-flight, drops a real iptables rule on the partner, waits
|
||
# FALLBACK_TEST_BLOCK_WAIT for a transition nothing is there to make, and fails Phase 3 blaming
|
||
# fallback detection. Only the EXIT trap gets connectivity back.
|
||
#
|
||
# In --dry-run nothing is blocked and nothing is waited on, so a dead daemon is worth saying but
|
||
# not worth aborting for — the walkthrough still shows the operator the shape of the run.
|
||
if pgrep -f "Fallback/fallback\.sh" >/dev/null 2>&1; then
|
||
log "fallback.sh daemon is running"
|
||
elif [[ "$DRY_RUN" == true ]]; then
|
||
warn "fallback.sh is NOT running — a real test would abort here"
|
||
else
|
||
error "fallback.sh is not running — nothing would detect the outage this test creates"
|
||
error "Start it with array_started.sh, or run with --dry-run to walk the phases"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
|
||
# Tier 1 containers configured
|
||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||
error "Check FALLBACK_${REMOTE_ID}_TIER1 in ${REMOTE_ID,,}.conf"
|
||
phase_fail "Pre-flight"
|
||
exit 1
|
||
fi
|
||
log "Tier 1 containers: ${TIER1_CONTAINERS[*]}"
|
||
|
||
phase_pass "Pre-flight"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 2 — Block Remote Connectivity ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
|
||
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
|
||
IPTABLES_RULE_ACTIVE=true
|
||
|
||
# Verify block is working
|
||
sleep 2
|
||
if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
|
||
log "Connectivity block confirmed — ping to remote fails as expected"
|
||
phase_pass "Block Remote"
|
||
else
|
||
error "iptables rule did not block connectivity — ping still succeeds"
|
||
phase_fail "Block Remote"
|
||
exit 1
|
||
fi
|
||
else
|
||
warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule"
|
||
phase_pass "Block Remote"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 3 — Fallback Detection ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Phase 3 — Fallback Detection ━━━"
|
||
warn "Waiting ${FALLBACK_TEST_BLOCK_WAIT}s for fallback.sh to detect outage..."
|
||
log "fallback.sh check interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
sleep "$FALLBACK_TEST_BLOCK_WAIT"
|
||
|
||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||
NEW_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
if [[ "$NEW_STATE" == "FALLBACK" ]]; then
|
||
echo "State changed to FALLBACK — outage detected correctly ✅"
|
||
phase_pass "Fallback Detection"
|
||
else
|
||
error "State is $NEW_STATE — expected FALLBACK after ${FALLBACK_TEST_BLOCK_WAIT}s"
|
||
warn "Is fallback.sh running? Check User Scripts plugin"
|
||
phase_fail "Fallback Detection"
|
||
fi
|
||
else
|
||
error "No state file found after wait — fallback.sh may not be running"
|
||
phase_fail "Fallback Detection"
|
||
fi
|
||
else
|
||
warn "DRY RUN — would wait ${FALLBACK_TEST_BLOCK_WAIT}s then check for FALLBACK state"
|
||
phase_pass "Fallback Detection"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 4 — Container Start Verification ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━"
|
||
log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
CONTAINERS_OK=true
|
||
for container in "${TIER1_CONTAINERS[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$STATUS" == "true" ]]; then
|
||
echo "$ICON_RUNNING $container is running locally ✅"
|
||
else
|
||
error "$ICON_NOT_RUNNING $container is NOT running locally"
|
||
CONTAINERS_OK=false
|
||
fi
|
||
done
|
||
|
||
if [[ "$CONTAINERS_OK" == true ]]; then
|
||
phase_pass "Container Start"
|
||
else
|
||
phase_fail "Container Start"
|
||
fi
|
||
else
|
||
warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}"
|
||
phase_pass "Container Start"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 5 — Restore Remote Connectivity ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
|
||
warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||
IPTABLES_RULE_ACTIVE=false
|
||
|
||
sleep 3
|
||
if ping_remote; then
|
||
echo "$REMOTE_SERVER_NAME is reachable again ✅"
|
||
phase_pass "Restore Connectivity"
|
||
else
|
||
error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
|
||
phase_fail "Restore Connectivity"
|
||
fi
|
||
else
|
||
warn "DRY RUN — would remove iptables rule"
|
||
phase_pass "Restore Connectivity"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 6 — Handback ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Phase 6 — Handback ━━━"
|
||
warn "Waiting ${FALLBACK_TEST_HANDBACK_WAIT}s for fallback.sh to complete handback..."
|
||
log "Requires $FALLBACK_HANDBACK_STRIKES consecutive checks at ${FALLBACK_CHECK_INTERVAL}s"
|
||
log "Minimum handback time: $(( FALLBACK_HANDBACK_STRIKES * FALLBACK_CHECK_INTERVAL ))s"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
sleep "$FALLBACK_TEST_HANDBACK_WAIT"
|
||
|
||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||
FINAL_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
if [[ "$FINAL_STATE" == "NORMAL" ]]; then
|
||
echo "State returned to NORMAL — handback completed ✅"
|
||
phase_pass "Handback"
|
||
else
|
||
error "State is $FINAL_STATE — expected NORMAL after ${FALLBACK_TEST_HANDBACK_WAIT}s"
|
||
warn "Handback may still be in progress — check fallback.sh output"
|
||
phase_fail "Handback"
|
||
fi
|
||
else
|
||
error "No state file found"
|
||
phase_fail "Handback"
|
||
fi
|
||
else
|
||
warn "DRY RUN — would wait ${FALLBACK_TEST_HANDBACK_WAIT}s then verify NORMAL state"
|
||
phase_pass "Handback"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Phase 7 — Container Handback Verification ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━"
|
||
log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
HANDBACK_OK=true
|
||
for container in "${TIER1_CONTAINERS[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$STATUS" != "true" ]]; then
|
||
echo "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
|
||
else
|
||
error "$ICON_RUNNING $container still running locally — handback may have failed"
|
||
HANDBACK_OK=false
|
||
fi
|
||
done
|
||
|
||
if [[ "$HANDBACK_OK" == true ]]; then
|
||
phase_pass "Container Handback"
|
||
else
|
||
phase_fail "Container Handback"
|
||
fi
|
||
else
|
||
warn "DRY RUN — would verify Tier 1 containers stopped locally after handback"
|
||
phase_pass "Container Handback"
|
||
fi
|
||
|
||
TOTAL_END=$(date +%s)
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Test Report ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY FALLBACK TEST REPORT ━━━━━"
|
||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
|
||
echo ""
|
||
echo " Phase Results:"
|
||
for phase in "${PHASES_PASS[@]}"; do
|
||
echo " $ICON_SUCCESS $phase"
|
||
done
|
||
for phase in "${PHASES_FAIL[@]}"; do
|
||
echo " $ICON_ERROR $phase"
|
||
done
|
||
echo ""
|
||
|
||
PASS_COUNT=${#PHASES_PASS[@]}
|
||
FAIL_COUNT=${#PHASES_FAIL[@]}
|
||
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
|
||
warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED"
|
||
notify "Fallback test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \
|
||
"Fallback Test" "normal"
|
||
else
|
||
error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
|
||
notify "Fallback test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \
|
||
"Fallback Test" "warning"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
|
||
exit 0 |