massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+199
-135
@@ -1,47 +1,71 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Failover Test ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Controlled simulation of the failover scenario — validates the entire failover lifecycle
|
||||
# ==============================================================================================
|
||||
# ================================= Failover Test ==============================================
|
||||
# ==============================================================================================
|
||||
# Controlled simulation of the failover lifecycle — validates the entire failover sequence
|
||||
# without waiting for a real outage.
|
||||
#
|
||||
# This script is a TEST HARNESS only — it does not contain failover logic.
|
||||
# All failover logic lives in failover.sh and is called directly from here.
|
||||
# Any changes to failover.sh are automatically reflected in this test.
|
||||
# ── WHAT THIS SCRIPT IS ───────────────────────────────────────────────────────────────────────
|
||||
# A test harness only — contains no failover logic.
|
||||
# All failover logic lives in failover.sh and is exercised by this test.
|
||||
# Any changes to failover.sh are automatically reflected here.
|
||||
#
|
||||
# Test sequence:
|
||||
# 1. Pre-flight — verify both servers reachable, failover.sh exists, state is NORMAL
|
||||
# 2. Block — add iptables rule dropping all traffic to remote IP
|
||||
# 3. Detect — run failover.sh one cycle — confirm FAILOVER state detected
|
||||
# 4. Start — verify failover containers started locally
|
||||
# 5. Restore — remove iptables rule, remote becomes reachable again
|
||||
# 6. Handback — wait for failover.sh to confirm handback strikes and hand back
|
||||
# 7. Verify — confirm containers returned to remote, local copies stopped
|
||||
# 8. Report — full pass/fail summary per phase
|
||||
# ── TEST SEQUENCE ─────────────────────────────────────────────────────────────────────────────
|
||||
# Phase 1 — Pre-flight verify both servers reachable, daemons healthy,
|
||||
# version parity, failover.sh exists, state is NORMAL
|
||||
# Phase 2 — Block Remote iptables rule drops all traffic to remote IP
|
||||
# Phase 3 — Failover Detection wait for failover.sh to detect outage and enter FAILOVER
|
||||
# Phase 4 — Container Start verify Tier 1 failover containers started locally
|
||||
# Phase 5 — Restore remove iptables rule, remote becomes reachable
|
||||
# Phase 6 — Handback wait for failover.sh to complete handback to NORMAL
|
||||
# Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback
|
||||
# Phase 8 — Report full pass/fail summary per phase
|
||||
#
|
||||
# Safety: iptables rule is removed via trap on ANY exit — crash, error, ctrl-c, or normal.
|
||||
# Remote connectivity is always restored regardless of test outcome.
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# FAILOVER_ENABLED gate — aborts if failover monitoring is disabled
|
||||
# iptables safety trap — rule ALWAYS removed on exit (crash, error, ctrl-c, normal)
|
||||
# remote connectivity always restored regardless of outcome
|
||||
# Version parity check — pre-flight verifies both servers on compatible unRAID versions
|
||||
# Remote Docker daemon — pre-flight verifies remote daemon is responsive
|
||||
# DOCKER_TIMEOUT — all docker calls protected against daemon hangs
|
||||
# MY_ID-based routing — tier containers selected via MY_ID not hostname comparison
|
||||
# Command validation — iptables and notify validated before use
|
||||
# Dry-run safe — full sequence walkthrough without touching iptables or containers
|
||||
#
|
||||
# ⚠️ This script starts and stops real containers on both servers.
|
||||
# ── WARNING ───────────────────────────────────────────────────────────────────────────────────
|
||||
# ⚠️ This script starts and stops REAL containers on both servers.
|
||||
# Run during a maintenance window — users will experience a brief service interruption.
|
||||
# Use --dry-run to walk through the sequence without touching containers or iptables.
|
||||
# Use --dry-run to walk through the sequence without any real changes.
|
||||
#
|
||||
# All configuration in Master.conf under Failover and Failover Test sections.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# FAILOVER_TEST_BLOCK_WAIT — seconds to wait for failover.sh to detect outage
|
||||
# FAILOVER_TEST_HANDBACK_WAIT — seconds to wait for failover.sh to complete handback
|
||||
# FAILOVER_CHECK_INTERVAL — check interval of the running failover.sh (informational)
|
||||
# FAILOVER_HANDBACK_STRIKES — strikes required before handback (informational)
|
||||
# FAILOVER_STATE_FILE — state file path to read current state
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# failover_test.sh — run full test sequence
|
||||
# failover_test.sh --dry-run — walk through all phases without changes
|
||||
# failover_test.sh --status — show current failover state and test config
|
||||
# failover_test.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh"
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SAFETY TRAP — always remove iptables rule on exit ─────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Fires on normal exit, error exit, ctrl-c, and script crashes.
|
||||
# Remote connectivity is ALWAYS restored regardless of test outcome.
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SAFETY TRAP — always remove iptables rule on exit
|
||||
# Fires on normal exit, error exit, ctrl-c, and script crashes
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
|
||||
cleanup() {
|
||||
@@ -51,7 +75,7 @@ cleanup() {
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
success "iptables rule removed — remote connectivity restored"
|
||||
warn "iptables rule removed — remote connectivity restored"
|
||||
else
|
||||
warn "DRY RUN — would remove iptables rule"
|
||||
fi
|
||||
@@ -60,9 +84,9 @@ cleanup() {
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
@@ -71,93 +95,127 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v iptables >/dev/null 2>&1; then
|
||||
error "iptables not found — required for connectivity simulation"
|
||||
exit 1
|
||||
# FAILOVER_ENABLED gate — no point testing if failover is disabled
|
||||
if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then
|
||||
warn "FAILOVER_ENABLED=false — failover test aborted"
|
||||
warn "Enable failover in master.conf before running this test"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
success "iptables available"
|
||||
acquire_lock # strict single instance — modifies iptables and containers
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Validate commands used by this script
|
||||
validate_unraid_cmd \
|
||||
"$(which iptables 2>/dev/null || echo /sbin/iptables)" \
|
||||
"--version" "iptables" \
|
||||
"iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; }
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
if [[ ! -f "$FAILOVER_SCRIPT" ]]; then
|
||||
error "failover.sh not found at $FAILOVER_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "failover.sh found"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
log "failover.sh found at $FAILOVER_SCRIPT"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
echo "$ICON_GEAR unRAID ver: $local_ver"
|
||||
echo "$ICON_FAILOVER Block wait: ${FAILOVER_TEST_BLOCK_WAIT}s"
|
||||
echo "$ICON_FAILOVER Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s"
|
||||
echo "$ICON_FAILOVER Check interval: ${FAILOVER_CHECK_INTERVAL}s"
|
||||
echo "$ICON_FAILOVER Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
|
||||
# Current failover state
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
echo "$ICON_FAILOVER Current state: ${CURRENT_STATE:-unknown}"
|
||||
else
|
||||
echo "$ICON_FAILOVER Current state: no state file"
|
||||
fi
|
||||
|
||||
# Show Tier 1 containers for this host
|
||||
TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1"
|
||||
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
|
||||
echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# PHASE TRACKING
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── PHASE TRACKING ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
PHASES_PASS=()
|
||||
PHASES_FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
phase_pass() { PHASES_PASS+=("$1"); success "$ICON_DONE Phase: $1 — PASSED"; }
|
||||
phase_fail() { PHASES_FAIL+=("$1"); error "$ICON_ERROR Phase: $1 — FAILED"; }
|
||||
phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE Phase: $1 — PASSED ✅"; }
|
||||
phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; }
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 1 — Pre-flight ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Get Tier 1 containers for this server's failover responsibility
|
||||
TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1"
|
||||
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 1 — Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo " $ICON_HOST Local: $LOCAL_SERVER_NAME"
|
||||
echo " $ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
|
||||
|
||||
# Check remote reachable
|
||||
info "Checking remote reachability..."
|
||||
# Remote reachable
|
||||
if ping_remote; then
|
||||
success "Remote $REMOTE_SERVER_NAME is reachable"
|
||||
log "$REMOTE_SERVER_NAME is reachable"
|
||||
else
|
||||
error "Remote $REMOTE_SERVER_NAME is not reachable — cannot run test"
|
||||
error "$REMOTE_SERVER_NAME is not reachable — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check internet reachable
|
||||
info "Checking internet connectivity..."
|
||||
# Internet reachable
|
||||
if ping_internet; then
|
||||
success "Internet is reachable"
|
||||
log "Internet is reachable"
|
||||
else
|
||||
error "No internet connectivity — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check current failover state is NORMAL
|
||||
# Version parity — test may produce misleading results on mismatch
|
||||
if ! check_unraid_version_parity; then
|
||||
error "unRAID version mismatch — test aborted to prevent misleading results"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remote Docker daemon — must be responsive before test manipulates containers
|
||||
if ! check_remote_docker_daemon; then
|
||||
error "Remote Docker daemon not responsive — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Failover state must be NORMAL before test
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
|
||||
@@ -165,28 +223,37 @@ if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
success "Failover state is NORMAL"
|
||||
log "Failover state is NORMAL"
|
||||
else
|
||||
warn "No state file found — assuming NORMAL (first run)"
|
||||
fi
|
||||
|
||||
# Tier 1 containers configured
|
||||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||||
error "Check FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1 in master_host*.conf"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
log "Tier 1 containers: ${TIER1_CONTAINERS[*]}"
|
||||
|
||||
phase_pass "Pre-flight"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 2 — Block Remote ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 2 — Block Remote Connectivity ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
|
||||
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER"
|
||||
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
|
||||
IPTABLES_RULE_ACTIVE=true
|
||||
success "iptables rule active — $REMOTE_SERVER_NAME appears unreachable"
|
||||
|
||||
# Verify block is working
|
||||
sleep 2
|
||||
if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
|
||||
success "Connectivity block confirmed — ping to remote fails as expected"
|
||||
log "Connectivity block confirmed — ping to remote fails as expected"
|
||||
phase_pass "Block Remote"
|
||||
else
|
||||
error "iptables rule did not block connectivity — ping still succeeds"
|
||||
@@ -194,30 +261,29 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would block $REMOTE_SERVER with iptables"
|
||||
warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule"
|
||||
phase_pass "Block Remote"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 3 — Failover Detection ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 3 — Failover Detection ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━"
|
||||
info "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
|
||||
info "failover.sh check interval is ${FAILOVER_CHECK_INTERVAL}s"
|
||||
warn "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..."
|
||||
log "failover.sh check interval: ${FAILOVER_CHECK_INTERVAL}s"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$FAILOVER_TEST_BLOCK_WAIT"
|
||||
|
||||
# Check state file updated to FAILOVER
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$NEW_STATE" == "FAILOVER" ]]; then
|
||||
success "State changed to FAILOVER — outage detected correctly"
|
||||
log "State changed to FAILOVER — outage detected correctly ✅"
|
||||
phase_pass "Failover Detection"
|
||||
else
|
||||
error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s"
|
||||
warn "failover.sh may not be running — check User Scripts plugin"
|
||||
warn "Is failover.sh running? Check User Scripts plugin"
|
||||
phase_fail "Failover Detection"
|
||||
fi
|
||||
else
|
||||
@@ -225,30 +291,25 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
phase_fail "Failover Detection"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s and check for FAILOVER state"
|
||||
warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s then check for FAILOVER state"
|
||||
phase_pass "Failover Detection"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 4 — Container Start Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 4 — Container Start Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Phase 4 — Failover Containers Started ━━━"
|
||||
|
||||
# Determine which containers should have started on this host
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
EXPECTED_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
|
||||
else
|
||||
EXPECTED_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
|
||||
fi
|
||||
echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━"
|
||||
log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
CONTAINERS_OK=true
|
||||
for container in "${EXPECTED_CONTAINERS[@]}"; do
|
||||
for container in "${TIER1_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
success "$ICON_RUNNING $container is running locally"
|
||||
log "$ICON_RUNNING $container is running locally ✅"
|
||||
else
|
||||
error "$ICON_NOT_RUNNING $container is NOT running locally"
|
||||
CONTAINERS_OK=false
|
||||
@@ -261,29 +322,27 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
phase_fail "Container Start"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would verify these containers started: ${EXPECTED_CONTAINERS[*]}"
|
||||
warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}"
|
||||
phase_pass "Container Start"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 5 — Restore Connectivity ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 5 — Restore Remote Connectivity ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
|
||||
info "Removing iptables block — remote becomes reachable again"
|
||||
warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
success "iptables rule removed"
|
||||
|
||||
# Verify connectivity restored
|
||||
sleep 3
|
||||
if ping_remote; then
|
||||
success "Remote $REMOTE_SERVER_NAME is reachable again"
|
||||
log "$REMOTE_SERVER_NAME is reachable again ✅"
|
||||
phase_pass "Restore Connectivity"
|
||||
else
|
||||
error "Remote still unreachable after removing iptables rule"
|
||||
error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
|
||||
phase_fail "Restore Connectivity"
|
||||
fi
|
||||
else
|
||||
@@ -291,14 +350,14 @@ else
|
||||
phase_pass "Restore Connectivity"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 6 — Handback ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 6 — Handback ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━"
|
||||
info "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to confirm handback..."
|
||||
info "Requires $FAILOVER_HANDBACK_STRIKES consecutive remote-up checks at ${FAILOVER_CHECK_INTERVAL}s intervals"
|
||||
info "Estimated minimum wait: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"
|
||||
warn "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to complete handback..."
|
||||
log "Requires $FAILOVER_HANDBACK_STRIKES consecutive checks at ${FAILOVER_CHECK_INTERVAL}s"
|
||||
log "Minimum handback time: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$FAILOVER_TEST_HANDBACK_WAIT"
|
||||
@@ -306,10 +365,11 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$FINAL_STATE" == "NORMAL" ]]; then
|
||||
success "State returned to NORMAL — handback completed"
|
||||
log "State returned to NORMAL — handback completed ✅"
|
||||
phase_pass "Handback"
|
||||
else
|
||||
error "State is $FINAL_STATE — expected NORMAL after handback wait"
|
||||
error "State is $FINAL_STATE — expected NORMAL after ${FAILOVER_TEST_HANDBACK_WAIT}s"
|
||||
warn "Handback may still be in progress — check failover.sh output"
|
||||
phase_fail "Handback"
|
||||
fi
|
||||
else
|
||||
@@ -317,23 +377,25 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
phase_fail "Handback"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s and verify NORMAL state"
|
||||
warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s then verify NORMAL state"
|
||||
phase_pass "Handback"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ PHASE 7 — Container Handback Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 7 — Container Handback Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Phase 7 — Failover Containers Stopped Locally ━━━"
|
||||
echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━"
|
||||
log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
HANDBACK_OK=true
|
||||
for container in "${EXPECTED_CONTAINERS[@]}"; do
|
||||
for container in "${TIER1_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$STATUS" != "true" ]]; then
|
||||
success "$ICON_NOT_RUNNING $container stopped locally — handed back"
|
||||
log "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
|
||||
else
|
||||
error "$ICON_RUNNING $container still running locally — handback may have failed"
|
||||
HANDBACK_OK=false
|
||||
@@ -346,20 +408,20 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
phase_fail "Container Handback"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would verify failover containers stopped locally after handback"
|
||||
warn "DRY RUN — would verify Tier 1 containers stopped locally after handback"
|
||||
phase_pass "Container Handback"
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Test Report ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Test Report ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━"
|
||||
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
|
||||
echo ""
|
||||
echo " Phase Results:"
|
||||
for phase in "${PHASES_PASS[@]}"; do
|
||||
@@ -375,13 +437,15 @@ FAIL_COUNT=${#PHASES_FAIL[@]}
|
||||
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL $TOTAL_PHASES PHASES PASSED"
|
||||
notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed successfully" "Failover Test" "normal"
|
||||
warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED"
|
||||
notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \
|
||||
"Failover Test" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
|
||||
notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" "Failover Test" "warning"
|
||||
error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
|
||||
notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \
|
||||
"Failover Test" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user