refactor: rename failover/HA → fallback across entire codebase
Removes all references to "failover" and "HA" (high availability) terminology from variable names, config keys, state values, rsync profile names, directory paths, and user-visible strings. Mapping: FAILOVER_* → FALLBACK_* FAILOVER_HOST*_RUNS_FOR → FALLBACK_HOST*_COVERS critical-failover → critical-fallback emby-failover → emby-fallback appdata-Failover/ → appdata-Fallback/ "FAILOVER" state value → "FALLBACK" failover_start key → fallback_start Failover/ directory → Fallback/ failover.sh → fallback.sh failover_state.db → fallback_state.db -Failover folder suffix → -Fallback State machine: NORMAL | FALLBACK | DARK (unchanged) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
4e22f5d1f7
commit
009820e981
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,870 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Failover ===================================================
|
||||
# ==============================================================================================
|
||||
# Mutual container failover between two unRAID servers.
|
||||
# Each server runs this script independently — no direct coordination between servers.
|
||||
# All decisions based solely on two pings: remote reachable + internet reachable.
|
||||
#
|
||||
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
|
||||
# Both servers run this script continuously as a background task via User Scripts.
|
||||
# Every FALLBACK_CHECK_INTERVAL seconds each server:
|
||||
# 1. Pings the remote server
|
||||
# 2. Pings the internet
|
||||
# 3. Determines its current state
|
||||
# 4. Takes the appropriate action
|
||||
#
|
||||
# No SSH signaling, no shared state files, no coordination — each server acts autonomously
|
||||
# based only on what it can see from its own network perspective.
|
||||
#
|
||||
# ── STATES ────────────────────────────────────────────────────────────────────────────────────
|
||||
# NORMAL — remote up, internet up
|
||||
# Own containers only. DDNS ON. Silent operation.
|
||||
#
|
||||
# FAILOVER — remote down, internet up
|
||||
# Start remote containers locally — tiered by outage duration.
|
||||
# Remote DDNS started immediately (Tier 1).
|
||||
# Own containers keep running — failover is additive.
|
||||
#
|
||||
# NO_INTERNET — internet down (remote may be up or down)
|
||||
# Stop own DDNS immediately — can't update DNS without internet.
|
||||
# Do not start remote containers — no internet = no point.
|
||||
# Wait for recovery.
|
||||
#
|
||||
# DARK — remote down AND internet down
|
||||
# Same actions as NO_INTERNET.
|
||||
# Cannot determine if remote is truly down or just unreachable.
|
||||
#
|
||||
# ── DDNS RULES — ABSOLUTE ─────────────────────────────────────────────────────────────────────
|
||||
# Each server owns its own DDNS — ON when that server has internet.
|
||||
# Script controls DDNS exclusively — network state NEVER auto-starts DDNS.
|
||||
# DDNS only starts after full handback sequence confirms containers are up.
|
||||
# One DDNS per domain active at all times — never two, never zero for long.
|
||||
# 1 minute TTL + 1 minute check interval = minimal user impact on failover.
|
||||
#
|
||||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# When remote returns after FAILOVER:
|
||||
# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
|
||||
# 2. Pre-flight checks — version parity, remote array, remote Docker daemon
|
||||
# 3. Stop remote DDNS first — prevents split brain DNS during rsync
|
||||
# 4. Stop remote containers — clean state before rsync
|
||||
# 5. Tiered rsync writeback — skip if outage under threshold
|
||||
# 6. Start remote containers — in dependency order with verification
|
||||
# 7. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up
|
||||
# 8. Return to NORMAL
|
||||
#
|
||||
# ── TIERED FAILOVER ───────────────────────────────────────────────────────────────────────────
|
||||
# Tier 1 — Immediate — vital services + Live TV — cannot wait
|
||||
# Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr)
|
||||
# Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr)
|
||||
# Tier 4 — HOST*_TIER4_DELAY — arrs + downloaders (default 24hr)
|
||||
# Tier delays configurable per host in master_host*.conf
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# FALLBACK_ENABLED gate — exits cleanly if disabled in master.conf
|
||||
# Version parity check — refuses handback if unRAID versions mismatch
|
||||
# Remote Docker daemon — checks remote daemon before issuing any remote commands
|
||||
# Timeout protection — all docker and SSH commands wrapped in timeouts
|
||||
# Container verification — verifies containers came up after start (failover critical)
|
||||
# MY_ID-based routing — all tier/DDNS/writeback arrays selected via MY_ID not hostname
|
||||
# Command validation — validates unRAID notify script before use
|
||||
# Silent by default — state transitions warn(), routine cycle checks log()
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_DDNS_CONTAINERS — DDNS containers this host manages
|
||||
# FAILOVER_HOST*_STOP_ON_NO_NET — containers stopped on internet loss
|
||||
# FAILOVER_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other
|
||||
# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes
|
||||
# HOST*_TIER1_WRITEBACK_DELAY — skip Tier 1 writeback if outage under this
|
||||
# FAILOVER_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# FALLBACK_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.)
|
||||
# FALLBACK_CHECK_INTERVAL — seconds between checks
|
||||
# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback
|
||||
# FALLBACK_STATE_FILE — /boot/config path — survives reboots
|
||||
# FALLBACK_RSYNC_ENABLED — gate for writeback rsync jobs
|
||||
# EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# fallback.sh — normal start (continuous loop)
|
||||
# fallback.sh --dry-run — preview state changes without acting on containers
|
||||
# fallback.sh --status — show current state and exit
|
||||
# fallback.sh --log — verbose cycle output
|
||||
#
|
||||
# ── TO STOP THIS SCRIPT ───────────────────────────────────────────────────────────────────────
|
||||
# Click Abort in unRAID User Scripts — do NOT kill directly, state file may corrupt.
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt)
|
||||
if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
|
||||
warn "FALLBACK_ENABLED=false — fallback monitoring disabled"
|
||||
warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
acquire_lock # strict single instance — notifies on collision
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Validate unRAID notify script — used throughout for state change notifications
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"
|
||||
|
||||
# Timeout for all docker and SSH docker commands
|
||||
DOCKER_TIMEOUT=15
|
||||
SSH_TIMEOUT=10
|
||||
CONTAINER_VERIFY_WAIT=5 # seconds after start before verifying container is up
|
||||
|
||||
# ==============================================================================================
|
||||
# ── STATE FILE HELPERS ────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# State file on /boot/config — survives reboots.
|
||||
# Format: key=value one per line.
|
||||
# Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started
|
||||
|
||||
state_get() {
|
||||
grep "^${1}=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2
|
||||
}
|
||||
|
||||
state_set() {
|
||||
local key="$1" value="$2"
|
||||
if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then
|
||||
sed -i "s|^${key}=.*|${key}=${value}|" "$FALLBACK_STATE_FILE"
|
||||
else
|
||||
echo "${key}=${value}" >> "$FALLBACK_STATE_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
state_init() {
|
||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||||
[[ -z "$(state_get fallback_start)" ]] && state_set fallback_start "0"
|
||||
[[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0"
|
||||
[[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false"
|
||||
[[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false"
|
||||
[[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CONTAINER HELPERS ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# All docker commands wrapped in DOCKER_TIMEOUT.
|
||||
# All SSH docker commands wrapped in SSH_TIMEOUT.
|
||||
# Verification after start — failover is critical, confirm containers came up.
|
||||
|
||||
# Start a container locally and verify it came up
|
||||
local_start() {
|
||||
local container="$1"
|
||||
[[ -z "$container" ]] && return
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$status" == "true" ]]; then
|
||||
log "$container already running locally"
|
||||
return 0
|
||||
fi
|
||||
log "$ICON_START Starting $container locally..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container locally"
|
||||
return 0
|
||||
fi
|
||||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||||
sleep "$CONTAINER_VERIFY_WAIT"
|
||||
local post_status
|
||||
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$post_status" == "true" ]]; then
|
||||
log "$ICON_STARTED $container started and running ✅"
|
||||
return 0
|
||||
else
|
||||
error "$container started but crashed immediately"
|
||||
notify "$container failed to stay running during fallback on $(hostname)" \
|
||||
"Fallback" "warning"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
error "Failed to start $container locally"
|
||||
notify "Failed to start $container locally during fallback on $(hostname)" \
|
||||
"Fallback" "warning"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop a container locally
|
||||
local_stop() {
|
||||
local container="$1"
|
||||
[[ -z "$container" ]] && return
|
||||
local status
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
log "$container already stopped locally"
|
||||
return 0
|
||||
fi
|
||||
log "$ICON_STOP Stopping $container locally..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop $container locally"
|
||||
return 0
|
||||
fi
|
||||
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
|
||||
log "$ICON_STOPPED $container stopped" || \
|
||||
error "Failed to stop $container locally"
|
||||
}
|
||||
|
||||
# Start a container on remote via SSH and verify it came up
|
||||
remote_start() {
|
||||
local container="$1"
|
||||
[[ -z "$container" ]] && return
|
||||
local status
|
||||
status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||||
$container 2>/dev/null" 2>/dev/null)
|
||||
if [[ "$status" == "true" ]]; then
|
||||
log "$container already running on $REMOTE_SERVER_NAME"
|
||||
return 0
|
||||
fi
|
||||
log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
|
||||
return 0
|
||||
fi
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then
|
||||
sleep "$CONTAINER_VERIFY_WAIT"
|
||||
local post_status
|
||||
post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \
|
||||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||||
$container 2>/dev/null" 2>/dev/null)
|
||||
if [[ "$post_status" == "true" ]]; then
|
||||
log "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅"
|
||||
return 0
|
||||
else
|
||||
error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
|
||||
notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \
|
||||
"Fallback" "warning"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
error "Failed to start $container on $REMOTE_SERVER_NAME"
|
||||
notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \
|
||||
"Fallback" "warning"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop a container on remote via SSH
|
||||
remote_stop() {
|
||||
local container="$1"
|
||||
[[ -z "$container" ]] && return
|
||||
local status
|
||||
status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||||
$container 2>/dev/null" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
log "$container already stopped on $REMOTE_SERVER_NAME"
|
||||
return 0
|
||||
fi
|
||||
log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
|
||||
return 0
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
|
||||
log "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
|
||||
error "Failed to stop $container on $REMOTE_SERVER_NAME"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DDNS HELPERS ──────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# DDNS managed exclusively by this script — never by network state alone.
|
||||
# Stopping DDNS on internet loss prevents split-brain DNS updates.
|
||||
|
||||
local_ddns_stop() {
|
||||
warn "$ICON_NET Stopping local DDNS — internet lost"
|
||||
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
||||
local_stop "$container"
|
||||
done
|
||||
}
|
||||
|
||||
local_ddns_start() {
|
||||
warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up"
|
||||
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
||||
local_start "$container"
|
||||
done
|
||||
}
|
||||
|
||||
remote_ddns_stop() {
|
||||
warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync"
|
||||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
remote_stop "$container"
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TIERED FAILOVER HELPERS ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# All routing uses MY_ID — not hostname string comparison.
|
||||
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
|
||||
# "This server covers the other" — MY_ID selects which tier arrays to use.
|
||||
|
||||
get_tier_containers() {
|
||||
local tier="$1"
|
||||
local remote_id="${REMOTE_ID}"
|
||||
local var_name="FALLBACK_${MY_ID}_COVERS_${remote_id}_TIER${tier}"
|
||||
eval "echo \"\${${var_name}[@]:-}\""
|
||||
}
|
||||
|
||||
get_tier2_delay() {
|
||||
local var_name="${REMOTE_ID}_TIER2_DELAY"
|
||||
echo "${!var_name:-240}"
|
||||
}
|
||||
|
||||
get_tier3_delay() {
|
||||
local var_name="${REMOTE_ID}_TIER3_DELAY"
|
||||
echo "${!var_name:-720}"
|
||||
}
|
||||
|
||||
get_tier4_delay() {
|
||||
local var_name="${REMOTE_ID}_TIER4_DELAY"
|
||||
echo "${!var_name:-1440}"
|
||||
}
|
||||
|
||||
get_tier1_writeback_delay() {
|
||||
local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY"
|
||||
echo "${!var_name:-60}"
|
||||
}
|
||||
|
||||
get_writeback_jobs_for_tier() {
|
||||
local tier="$1"
|
||||
local remote_id="${REMOTE_ID}"
|
||||
if [[ "$tier" -eq 4 ]]; then
|
||||
# Tier 4 — daily sync shares of the remote + any extra writeback paths
|
||||
local shares_var="${remote_id}_DAILY_SYNC_SHARES"
|
||||
local extra_var="FALLBACK_${remote_id}_WRITEBACK_TIER4"
|
||||
eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\""
|
||||
else
|
||||
local var_name="FALLBACK_${remote_id}_WRITEBACK_TIER${tier}"
|
||||
eval "echo \"\${${var_name}[@]:-}\""
|
||||
fi
|
||||
}
|
||||
|
||||
# Select DDNS arrays based on MY_ID
|
||||
set_ddns_arrays() {
|
||||
local local_ddns_var="${MY_ID}_DDNS_CONTAINERS"
|
||||
local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS"
|
||||
eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")"
|
||||
eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")"
|
||||
}
|
||||
|
||||
# Select internet-loss stop list based on MY_ID
|
||||
get_stop_on_no_net() {
|
||||
local var_name="FALLBACK_${MY_ID}_STOP_ON_NO_NET"
|
||||
eval "echo \"\${${var_name}[@]:-}\""
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
state_init
|
||||
set_ddns_arrays
|
||||
CURRENT_STATE=$(state_get state)
|
||||
FALLBACK_START_TS=$(state_get fallback_start)
|
||||
TIER2=$(state_get tier2_started)
|
||||
TIER3=$(state_get tier3_started)
|
||||
TIER4=$(state_get tier4_started)
|
||||
STRIKES=$(state_get handback_strikes)
|
||||
|
||||
local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY FALLBACK STATUS ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
echo "$ICON_GEAR unRAID ver: $local_ver"
|
||||
echo "$ICON_FALLBACK State: $CURRENT_STATE"
|
||||
echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]:-none}"
|
||||
echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}"
|
||||
echo "$ICON_TIME Interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||||
echo "$ICON_FALLBACK Strikes: $STRIKES / $FALLBACK_HANDBACK_STRIKES"
|
||||
|
||||
if [[ "$CURRENT_STATE" == "FALLBACK" && "$FALLBACK_START_TS" -gt 0 ]]; then
|
||||
ELAPSED=$(( ($(date +%s) - FALLBACK_START_TS) / 60 ))
|
||||
echo "$ICON_TIME Outage: ${ELAPSED}min"
|
||||
echo "$ICON_FALLBACK Tier 2: $TIER2 (delay: $(get_tier2_delay)min)"
|
||||
echo "$ICON_FALLBACK Tier 3: $TIER3 (delay: $(get_tier3_delay)min)"
|
||||
echo "$ICON_FALLBACK Tier 4: $TIER4 (delay: $(get_tier4_delay)min)"
|
||||
fi
|
||||
|
||||
echo "$ICON_GEAR Enabled: $FALLBACK_ENABLED"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Called when remote returns after FAILOVER state.
|
||||
# CRITICAL sequencing — do not reorder without understanding the consequences.
|
||||
# Each step must succeed before proceeding — aborts and retries next cycle on failure.
|
||||
|
||||
run_handback() {
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "$REMOTE_SERVER_NAME has returned — beginning handback"
|
||||
|
||||
# ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Version parity — refuse handback if unRAID versions mismatch
|
||||
if ! check_unraid_version_parity; then
|
||||
warn "Version parity check failed — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote array must be mounted before rsync
|
||||
if ! check_remote_array; then
|
||||
warn "Remote array not ready — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote Docker daemon must be responsive before issuing container commands
|
||||
if ! check_remote_docker_daemon; then
|
||||
warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Pre-flight checks passed ✅"
|
||||
|
||||
# ── Step 2: Stop remote DDNS FIRST ──────────────────────────────────────────────────────
|
||||
# CRITICAL — prevents split-brain DNS while data is being synced
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET DDNS Handoff ━━━"
|
||||
remote_ddns_stop
|
||||
sleep 5 # brief pause to ensure DDNS stop propagates before rsync
|
||||
|
||||
# ── Step 3: Stop remote containers ──────────────────────────────────────────────────────
|
||||
# Clean state before rsync — no dirty writes during transfer window
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Remote Containers ━━━"
|
||||
|
||||
local ALL_FALLBACK_CONTAINERS=()
|
||||
|
||||
# Collect all started tiers in reverse order (highest tier stops first)
|
||||
if [[ "$(state_get tier4_started)" == "true" ]]; then
|
||||
read -r -a t4 <<< "$(get_tier_containers 4)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t4[@]}")
|
||||
fi
|
||||
if [[ "$(state_get tier3_started)" == "true" ]]; then
|
||||
read -r -a t3 <<< "$(get_tier_containers 3)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t3[@]}")
|
||||
fi
|
||||
if [[ "$(state_get tier2_started)" == "true" ]]; then
|
||||
read -r -a t2 <<< "$(get_tier_containers 2)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t2[@]}")
|
||||
fi
|
||||
|
||||
# Tier 1 always started — stop last (DDNS already handled above)
|
||||
read -r -a t1 <<< "$(get_tier_containers 1)"
|
||||
for container in "${t1[@]}"; do
|
||||
local is_ddns=false
|
||||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||||
done
|
||||
[[ "$is_ddns" == false ]] && ALL_FALLBACK_CONTAINERS+=("$container")
|
||||
done
|
||||
|
||||
for container in "${ALL_FALLBACK_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local_stop "$container"
|
||||
done
|
||||
|
||||
log "All fallback containers stopped locally"
|
||||
|
||||
# ── Step 4: Rsync writeback ──────────────────────────────────────────────────────────────
|
||||
# Tiered writeback with skip window — short outages do not benefit from writeback.
|
||||
# After a short outage primary's clean nightly state is more reliable than dirty
|
||||
# accumulation — skip writeback entirely for outages under the threshold.
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Rsync Writeback ━━━"
|
||||
|
||||
local outage_minutes
|
||||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||||
local tier1_wb_delay
|
||||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||||
|
||||
warn "Outage duration: ${outage_minutes}min"
|
||||
|
||||
if ! check_rsync_enabled "FALLBACK"; then
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping all writeback jobs"
|
||||
warn "Handback will complete without syncing state back to primary"
|
||||
else
|
||||
run_writeback_tier() {
|
||||
local tier="$1" threshold="$2" label="$3"
|
||||
local jobs
|
||||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||||
[[ ${#jobs[@]} -eq 0 ]] && return
|
||||
|
||||
if [[ "$outage_minutes" -ge "$threshold" ]]; then
|
||||
warn "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min"
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ "$(basename "$job")" == "Emby" ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min — primary state is cleaner"
|
||||
fi
|
||||
}
|
||||
|
||||
local t2_delay t3_delay t4_delay
|
||||
t2_delay=$(get_tier2_delay)
|
||||
t3_delay=$(get_tier3_delay)
|
||||
t4_delay=$(get_tier4_delay)
|
||||
|
||||
run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack"
|
||||
run_writeback_tier 2 "$t2_delay" "productivity services"
|
||||
run_writeback_tier 3 "$t3_delay" "secondary services"
|
||||
run_writeback_tier 4 "$t4_delay" "media shares + edge cases"
|
||||
fi
|
||||
|
||||
log "Writeback complete"
|
||||
|
||||
# ── Step 5: Start remote containers ─────────────────────────────────────────────────────
|
||||
# Start in tier order with dependency awareness — databases before apps
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Start Remote Containers ━━━"
|
||||
|
||||
# Tier 1 — excluding DDNS (handled separately as final step)
|
||||
for container in "${t1[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_ddns=false
|
||||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||||
done
|
||||
[[ "$is_ddns" == false ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
sleep 10 # give databases time to initialise before apps
|
||||
|
||||
[[ "$(state_get tier2_started)" == "true" ]] && \
|
||||
for container in "${t2[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
[[ "$(state_get tier3_started)" == "true" ]] && \
|
||||
for container in "${t3[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
[[ "$(state_get tier4_started)" == "true" ]] && \
|
||||
for container in "${t4[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
log "Remote containers started"
|
||||
|
||||
# ── Step 6: Start remote DDNS LAST ──────────────────────────────────────────────────────
|
||||
# DNS cuts over ONLY after containers confirmed up — this is the final step
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET DNS Cutover ━━━"
|
||||
sleep 5 # brief pause to ensure containers are accepting connections
|
||||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
remote_start "$container"
|
||||
done
|
||||
warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
|
||||
|
||||
# ── Step 7: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
state_set state "NORMAL"
|
||||
state_set fallback_start "0"
|
||||
state_set handback_strikes "0"
|
||||
state_set tier2_started "false"
|
||||
state_set tier3_started "false"
|
||||
state_set tier4_started "false"
|
||||
|
||||
warn "$ICON_DONE Handback complete — returned to NORMAL"
|
||||
notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
|
||||
"Fallback" "normal"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Main State Machine ━━━
|
||||
# ==============================================================================================
|
||||
state_init
|
||||
set_ddns_arrays
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_FALLBACK FALLBACK — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
FALLBACK_RUNNING=true
|
||||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
||||
SIGTERM SIGINT
|
||||
|
||||
while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
NOW=$(date +%s)
|
||||
CURRENT_STATE=$(state_get state)
|
||||
|
||||
# ── Connectivity checks ──────────────────────────────────────────────────────────────────
|
||||
REMOTE_UP=false
|
||||
INTERNET_UP=false
|
||||
|
||||
ping_remote && REMOTE_UP=true
|
||||
ping_internet && INTERNET_UP=true
|
||||
|
||||
log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"
|
||||
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
# NORMAL STATE
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
||||
|
||||
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
||||
log "$ICON_SUCCESS NORMAL — all systems up"
|
||||
|
||||
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
||||
# Remote is down — enter FAILOVER
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"
|
||||
|
||||
state_set state "FALLBACK"
|
||||
state_set fallback_start "$NOW"
|
||||
state_set handback_strikes "0"
|
||||
state_set tier2_started "false"
|
||||
state_set tier3_started "false"
|
||||
state_set tier4_started "false"
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
||||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||||
for container in "${tier1[@]}"; do
|
||||
[[ -n "$container" ]] && local_start "$container"
|
||||
done
|
||||
|
||||
notify "FALLBACK started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \
|
||||
"Fallback" "warning"
|
||||
|
||||
elif [[ "$INTERNET_UP" == false ]]; then
|
||||
# Lost internet — enter NO_INTERNET
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "Internet connectivity lost — stopping local DDNS"
|
||||
|
||||
state_set state "NO_INTERNET"
|
||||
local_ddns_stop
|
||||
|
||||
# Stop containers configured to stop on internet loss
|
||||
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
|
||||
for container in "${stop_on_no_net[@]}"; do
|
||||
[[ -n "$container" ]] && local_stop "$container"
|
||||
done
|
||||
|
||||
notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \
|
||||
"Fallback" "warning"
|
||||
fi
|
||||
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
# FAILOVER STATE
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||||
|
||||
FALLBACK_START_TS=$(state_get fallback_start)
|
||||
ELAPSED_MIN=$(( (NOW - FALLBACK_START_TS) / 60 ))
|
||||
|
||||
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
||||
# Remote returned — increment handback strikes
|
||||
STRIKES=$(state_get handback_strikes)
|
||||
STRIKES=$(( STRIKES + 1 ))
|
||||
state_set handback_strikes "$STRIKES"
|
||||
|
||||
warn "$ICON_FALLBACK $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FALLBACK_HANDBACK_STRIKES"
|
||||
|
||||
if [[ "$STRIKES" -ge "$FALLBACK_HANDBACK_STRIKES" ]]; then
|
||||
run_handback
|
||||
fi
|
||||
|
||||
elif [[ "$INTERNET_UP" == false ]]; then
|
||||
# Lost internet during fallback — enter DARK
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "Lost internet during fallback — entering DARK state"
|
||||
state_set state "DARK"
|
||||
local_ddns_stop
|
||||
notify "DARK state on $(hostname) — lost internet during fallback" \
|
||||
"Fallback" "warning"
|
||||
|
||||
else
|
||||
# Still in failover — check tier escalation
|
||||
state_set handback_strikes "0"
|
||||
|
||||
TIER2_DELAY=$(get_tier2_delay)
|
||||
if [[ "$(state_get tier2_started)" == "false" && \
|
||||
"$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
|
||||
read -r -a tier2 <<< "$(get_tier_containers 2)"
|
||||
for container in "${tier2[@]}"; do
|
||||
[[ -n "$container" ]] && local_start "$container"
|
||||
done
|
||||
state_set tier2_started "true"
|
||||
notify "Fallback Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \
|
||||
"Fallback" "warning"
|
||||
fi
|
||||
|
||||
TIER3_DELAY=$(get_tier3_delay)
|
||||
if [[ "$(state_get tier3_started)" == "false" && \
|
||||
"$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
|
||||
read -r -a tier3 <<< "$(get_tier_containers 3)"
|
||||
for container in "${tier3[@]}"; do
|
||||
[[ -n "$container" ]] && local_start "$container"
|
||||
done
|
||||
state_set tier3_started "true"
|
||||
notify "Fallback Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \
|
||||
"Fallback" "warning"
|
||||
fi
|
||||
|
||||
TIER4_DELAY=$(get_tier4_delay)
|
||||
if [[ "$(state_get tier4_started)" == "false" && \
|
||||
"$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
|
||||
read -r -a tier4 <<< "$(get_tier_containers 4)"
|
||||
for container in "${tier4[@]}"; do
|
||||
[[ -n "$container" ]] && local_start "$container"
|
||||
done
|
||||
state_set tier4_started "true"
|
||||
notify "Fallback Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \
|
||||
"Fallback" "warning"
|
||||
fi
|
||||
|
||||
log "$ICON_FALLBACK FALLBACK — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
|
||||
fi
|
||||
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
# NO_INTERNET STATE
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then
|
||||
|
||||
if [[ "$INTERNET_UP" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ "$REMOTE_UP" == true ]]; then
|
||||
warn "Remote is up — returning to NORMAL"
|
||||
state_set state "NORMAL"
|
||||
local_ddns_start
|
||||
notify "Internet recovered on $(hostname) — returning to NORMAL" \
|
||||
"Fallback" "normal"
|
||||
else
|
||||
warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
|
||||
state_set state "FALLBACK"
|
||||
state_set fallback_start "$NOW"
|
||||
state_set handback_strikes "0"
|
||||
state_set tier2_started "false"
|
||||
state_set tier3_started "false"
|
||||
state_set tier4_started "false"
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
||||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||||
for container in "${tier1[@]}"; do
|
||||
[[ -n "$container" ]] && local_start "$container"
|
||||
done
|
||||
notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FALLBACK" \
|
||||
"Fallback" "warning"
|
||||
fi
|
||||
else
|
||||
log "$ICON_NET NO_INTERNET — waiting for connectivity"
|
||||
fi
|
||||
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
# DARK STATE
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
elif [[ "$CURRENT_STATE" == "DARK" ]]; then
|
||||
|
||||
if [[ "$INTERNET_UP" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ "$REMOTE_UP" == true ]]; then
|
||||
warn "Remote up, internet up — transitioning through FALLBACK for handback"
|
||||
# Was in failover before DARK — route through FALLBACK state for handback
|
||||
state_set state "FALLBACK"
|
||||
state_set handback_strikes "0"
|
||||
else
|
||||
warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
|
||||
state_set state "FALLBACK"
|
||||
state_set fallback_start "$NOW"
|
||||
state_set handback_strikes "0"
|
||||
state_set tier2_started "false"
|
||||
state_set tier3_started "false"
|
||||
state_set tier4_started "false"
|
||||
|
||||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||||
for container in "${tier1[@]}"; do
|
||||
[[ -n "$container" ]] && local_start "$container"
|
||||
done
|
||||
fi
|
||||
notify "Emerging from DARK state on $(hostname)" "Fallback" "warning"
|
||||
else
|
||||
log "$ICON_FALLBACK DARK — no internet, no remote — waiting"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Sleep until next check ───────────────────────────────────────────────────────────────
|
||||
log "Next check in ${FALLBACK_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
|
||||
sleep "$FALLBACK_CHECK_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
@@ -0,0 +1,453 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Failover Test ==============================================
|
||||
# ==============================================================================================
|
||||
# Controlled simulation of the failover lifecycle — validates the entire failover sequence
|
||||
# without waiting for a real outage.
|
||||
#
|
||||
# ── WHAT THIS SCRIPT IS ───────────────────────────────────────────────────────────────────────
|
||||
# A test harness only — contains no failover logic.
|
||||
# All failover logic lives in fallback.sh and is exercised by this test.
|
||||
# Any changes to fallback.sh are automatically reflected here.
|
||||
#
|
||||
# ── TEST SEQUENCE ─────────────────────────────────────────────────────────────────────────────
|
||||
# Phase 1 — Pre-flight verify both servers reachable, daemons healthy,
|
||||
# version parity, fallback.sh exists, state is NORMAL
|
||||
# Phase 2 — Block Remote iptables rule drops all traffic to remote IP
|
||||
# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FAILOVER
|
||||
# Phase 4 — Container Start verify Tier 1 failover containers started locally
|
||||
# Phase 5 — Restore remove iptables rule, remote becomes reachable
|
||||
# Phase 6 — Handback wait for fallback.sh to complete handback to NORMAL
|
||||
# Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback
|
||||
# Phase 8 — Report full pass/fail summary per phase
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# FALLBACK_ENABLED gate — aborts if fallback monitoring is disabled
|
||||
# iptables safety trap — rule ALWAYS removed on exit (crash, error, ctrl-c, normal)
|
||||
# remote connectivity always restored regardless of outcome
|
||||
# Version parity check — pre-flight verifies both servers on compatible unRAID versions
|
||||
# Remote Docker daemon — pre-flight verifies remote daemon is responsive
|
||||
# DOCKER_TIMEOUT — all docker calls protected against daemon hangs
|
||||
# MY_ID-based routing — tier containers selected via MY_ID not hostname comparison
|
||||
# Command validation — iptables and notify validated before use
|
||||
# Dry-run safe — full sequence walkthrough without touching iptables or containers
|
||||
#
|
||||
# ── WARNING ───────────────────────────────────────────────────────────────────────────────────
|
||||
# ⚠️ This script starts and stops REAL containers on both servers.
|
||||
# Run during a maintenance window — users will experience a brief service interruption.
|
||||
# Use --dry-run to walk through the sequence without any real changes.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# FALLBACK_TEST_BLOCK_WAIT — seconds to wait for fallback.sh to detect outage
|
||||
# FALLBACK_TEST_HANDBACK_WAIT — seconds to wait for fallback.sh to complete handback
|
||||
# FALLBACK_CHECK_INTERVAL — check interval of the running fallback.sh (informational)
|
||||
# FALLBACK_HANDBACK_STRIKES — strikes required before handback (informational)
|
||||
# FALLBACK_STATE_FILE — state file path to read current state
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# fallback_test.sh — run full test sequence
|
||||
# fallback_test.sh --dry-run — walk through all phases without changes
|
||||
# fallback_test.sh --status — show current fallback state and test config
|
||||
# fallback_test.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SAFETY TRAP — always remove iptables rule on exit ─────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Fires on normal exit, error exit, ctrl-c, and script crashes.
|
||||
# Remote connectivity is ALWAYS restored regardless of test outcome.
|
||||
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
|
||||
cleanup() {
|
||||
if [[ "$IPTABLES_RULE_ACTIVE" == true ]]; then
|
||||
echo ""
|
||||
warn "$ICON_SHIELD Cleanup — removing iptables block on $REMOTE_SERVER..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
warn "iptables rule removed — remote connectivity restored"
|
||||
else
|
||||
warn "DRY RUN — would remove iptables rule"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# FALLBACK_ENABLED gate — no point testing if failover is disabled
|
||||
if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
|
||||
warn "FALLBACK_ENABLED=false — fallback test aborted"
|
||||
warn "Enable fallback in master.conf before running this test"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
acquire_lock # strict single instance — modifies iptables and containers
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Validate commands used by this script
|
||||
validate_unraid_cmd \
|
||||
"$(which iptables 2>/dev/null || echo /sbin/iptables)" \
|
||||
"--version" "iptables" \
|
||||
"iptables" || { error "iptables not found — required for connectivity simulation"; exit 1; }
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
if [[ ! -f "$FALLBACK_SCRIPT" ]]; then
|
||||
error "fallback.sh not found at $FALLBACK_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
log "fallback.sh found at $FALLBACK_SCRIPT"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown")
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
echo "$ICON_GEAR unRAID ver: $local_ver"
|
||||
echo "$ICON_FALLBACK Block wait: ${FALLBACK_TEST_BLOCK_WAIT}s"
|
||||
echo "$ICON_FALLBACK Handback wait: ${FALLBACK_TEST_HANDBACK_WAIT}s"
|
||||
echo "$ICON_FALLBACK Check interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||||
echo "$ICON_FALLBACK Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
|
||||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
echo "$ICON_FALLBACK Current state: ${CURRENT_STATE:-unknown}"
|
||||
else
|
||||
echo "$ICON_FALLBACK Current state: no state file"
|
||||
fi
|
||||
|
||||
# Show Tier 1 containers for this host
|
||||
TIER1_VAR="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1"
|
||||
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
|
||||
echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PHASE TRACKING ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
PHASES_PASS=()
|
||||
PHASES_FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE Phase: $1 — PASSED ✅"; }
|
||||
phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; }
|
||||
|
||||
# Get Tier 1 containers for this server's failover responsibility
|
||||
TIER1_VAR="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1"
|
||||
eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 1 — Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_SHIELD FALLBACK TEST — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Phase 1 — Pre-flight ━━━"
|
||||
|
||||
# Remote reachable
|
||||
if ping_remote; then
|
||||
log "$REMOTE_SERVER_NAME is reachable"
|
||||
else
|
||||
error "$REMOTE_SERVER_NAME is not reachable — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Internet reachable
|
||||
if ping_internet; then
|
||||
log "Internet is reachable"
|
||||
else
|
||||
error "No internet connectivity — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Version parity — test may produce misleading results on mismatch
|
||||
if ! check_unraid_version_parity; then
|
||||
error "unRAID version mismatch — test aborted to prevent misleading results"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remote Docker daemon — must be responsive before test manipulates containers
|
||||
if ! check_remote_docker_daemon; then
|
||||
error "Remote Docker daemon not responsive — cannot run test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Fallback state must be NORMAL before test
|
||||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$CURRENT_STATE" != "NORMAL" ]]; then
|
||||
error "Fallback state is $CURRENT_STATE — must be NORMAL before running test"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
log "Fallback state is NORMAL"
|
||||
else
|
||||
warn "No state file found — assuming NORMAL (first run)"
|
||||
fi
|
||||
|
||||
# Tier 1 containers configured
|
||||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||||
error "Check FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1 in master_host*.conf"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
log "Tier 1 containers: ${TIER1_CONTAINERS[*]}"
|
||||
|
||||
phase_pass "Pre-flight"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 2 — Block Remote Connectivity ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PING Phase 2 — Block Remote Connectivity ━━━"
|
||||
warn "Adding iptables rule — dropping all traffic to $REMOTE_SERVER ($REMOTE_SERVER_NAME)"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -I OUTPUT -d "$REMOTE_SERVER" -j DROP
|
||||
IPTABLES_RULE_ACTIVE=true
|
||||
|
||||
# Verify block is working
|
||||
sleep 2
|
||||
if ! ping -c1 -W2 "$REMOTE_SERVER" &>/dev/null; then
|
||||
log "Connectivity block confirmed — ping to remote fails as expected"
|
||||
phase_pass "Block Remote"
|
||||
else
|
||||
error "iptables rule did not block connectivity — ping still succeeds"
|
||||
phase_fail "Block Remote"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would block $REMOTE_SERVER with iptables DROP rule"
|
||||
phase_pass "Block Remote"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 3 — Fallback Detection ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Phase 3 — Fallback Detection ━━━"
|
||||
warn "Waiting ${FALLBACK_TEST_BLOCK_WAIT}s for fallback.sh to detect outage..."
|
||||
log "fallback.sh check interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$FALLBACK_TEST_BLOCK_WAIT"
|
||||
|
||||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||
NEW_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$NEW_STATE" == "FALLBACK" ]]; then
|
||||
log "State changed to FALLBACK — outage detected correctly ✅"
|
||||
phase_pass "Fallback Detection"
|
||||
else
|
||||
error "State is $NEW_STATE — expected FALLBACK after ${FALLBACK_TEST_BLOCK_WAIT}s"
|
||||
warn "Is fallback.sh running? Check User Scripts plugin"
|
||||
phase_fail "Fallback Detection"
|
||||
fi
|
||||
else
|
||||
error "No state file found after wait — fallback.sh may not be running"
|
||||
phase_fail "Fallback Detection"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would wait ${FALLBACK_TEST_BLOCK_WAIT}s then check for FALLBACK state"
|
||||
phase_pass "Fallback Detection"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 4 — Container Start Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Phase 4 — Tier 1 Containers Started Locally ━━━"
|
||||
log "Checking Tier 1 containers: ${TIER1_CONTAINERS[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
CONTAINERS_OK=true
|
||||
for container in "${TIER1_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
log "$ICON_RUNNING $container is running locally ✅"
|
||||
else
|
||||
error "$ICON_NOT_RUNNING $container is NOT running locally"
|
||||
CONTAINERS_OK=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$CONTAINERS_OK" == true ]]; then
|
||||
phase_pass "Container Start"
|
||||
else
|
||||
phase_fail "Container Start"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would verify these Tier 1 containers started: ${TIER1_CONTAINERS[*]}"
|
||||
phase_pass "Container Start"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 5 — Restore Remote Connectivity ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_PING Phase 5 — Restore Remote Connectivity ━━━"
|
||||
warn "Removing iptables block — $REMOTE_SERVER_NAME becomes reachable again"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null
|
||||
IPTABLES_RULE_ACTIVE=false
|
||||
|
||||
sleep 3
|
||||
if ping_remote; then
|
||||
log "$REMOTE_SERVER_NAME is reachable again ✅"
|
||||
phase_pass "Restore Connectivity"
|
||||
else
|
||||
error "$REMOTE_SERVER_NAME still unreachable after removing iptables rule"
|
||||
phase_fail "Restore Connectivity"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would remove iptables rule"
|
||||
phase_pass "Restore Connectivity"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 6 — Handback ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Phase 6 — Handback ━━━"
|
||||
warn "Waiting ${FALLBACK_TEST_HANDBACK_WAIT}s for fallback.sh to complete handback..."
|
||||
log "Requires $FALLBACK_HANDBACK_STRIKES consecutive checks at ${FALLBACK_CHECK_INTERVAL}s"
|
||||
log "Minimum handback time: $(( FALLBACK_HANDBACK_STRIKES * FALLBACK_CHECK_INTERVAL ))s"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sleep "$FALLBACK_TEST_HANDBACK_WAIT"
|
||||
|
||||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||
FINAL_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$FINAL_STATE" == "NORMAL" ]]; then
|
||||
log "State returned to NORMAL — handback completed ✅"
|
||||
phase_pass "Handback"
|
||||
else
|
||||
error "State is $FINAL_STATE — expected NORMAL after ${FALLBACK_TEST_HANDBACK_WAIT}s"
|
||||
warn "Handback may still be in progress — check fallback.sh output"
|
||||
phase_fail "Handback"
|
||||
fi
|
||||
else
|
||||
error "No state file found"
|
||||
phase_fail "Handback"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would wait ${FALLBACK_TEST_HANDBACK_WAIT}s then verify NORMAL state"
|
||||
phase_pass "Handback"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Phase 7 — Container Handback Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Phase 7 — Tier 1 Containers Stopped Locally ━━━"
|
||||
log "Verifying Tier 1 containers returned to $REMOTE_SERVER_NAME"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
HANDBACK_OK=true
|
||||
for container in "${TIER1_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
if [[ "$STATUS" != "true" ]]; then
|
||||
log "$ICON_NOT_RUNNING $container stopped locally — handed back ✅"
|
||||
else
|
||||
error "$ICON_RUNNING $container still running locally — handback may have failed"
|
||||
HANDBACK_OK=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$HANDBACK_OK" == true ]]; then
|
||||
phase_pass "Container Handback"
|
||||
else
|
||||
phase_fail "Container Handback"
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would verify Tier 1 containers stopped locally after handback"
|
||||
phase_pass "Container Handback"
|
||||
fi
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Test Report ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY FALLBACK TEST REPORT ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))"
|
||||
echo ""
|
||||
echo " Phase Results:"
|
||||
for phase in "${PHASES_PASS[@]}"; do
|
||||
echo " $ICON_SUCCESS $phase"
|
||||
done
|
||||
for phase in "${PHASES_FAIL[@]}"; do
|
||||
echo " $ICON_ERROR $phase"
|
||||
done
|
||||
echo ""
|
||||
|
||||
PASS_COUNT=${#PHASES_PASS[@]}
|
||||
FAIL_COUNT=${#PHASES_FAIL[@]}
|
||||
TOTAL_PHASES=$(( PASS_COUNT + FAIL_COUNT ))
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$FAIL_COUNT" -eq 0 ]]; then
|
||||
warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED"
|
||||
notify "Fallback test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \
|
||||
"Fallback Test" "normal"
|
||||
else
|
||||
error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED"
|
||||
notify "Fallback test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \
|
||||
"Fallback Test" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$FAIL_COUNT" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user