821 lines
38 KiB
Bash
821 lines
38 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Failover ===================================================
|
|
# ==============================================================================================
|
|
# Mutual container failover between two unRAID servers.
|
|
# Each server runs this script independently — no direct coordination between servers.
|
|
# All decisions based solely on two pings: remote reachable + internet reachable.
|
|
#
|
|
# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
|
|
# Both servers run this script continuously as a background task via User Scripts.
|
|
# Every FAILOVER_CHECK_INTERVAL seconds each server:
|
|
# 1. Pings the remote server
|
|
# 2. Pings the internet
|
|
# 3. Determines its current state
|
|
# 4. Takes the appropriate action
|
|
#
|
|
# No SSH signaling, no shared state files, no coordination — each server acts autonomously
|
|
# based only on what it can see from its own network perspective.
|
|
#
|
|
# ── STATES ────────────────────────────────────────────────────────────────────────────────────
|
|
# NORMAL — remote up, internet up
|
|
# Own containers only. DDNS ON. Silent operation.
|
|
#
|
|
# FAILOVER — remote down, internet up
|
|
# Start remote containers locally — tiered by outage duration.
|
|
# Remote DDNS started immediately (Tier 1).
|
|
# Own containers keep running — failover is additive.
|
|
#
|
|
# NO_INTERNET — internet down (remote may be up or down)
|
|
# Stop own DDNS immediately — can't update DNS without internet.
|
|
# Do not start remote containers — no internet = no point.
|
|
# Wait for recovery.
|
|
#
|
|
# DARK — remote down AND internet down
|
|
# Same actions as NO_INTERNET.
|
|
# Cannot determine if remote is truly down or just unreachable.
|
|
#
|
|
# ── DDNS RULES — ABSOLUTE ─────────────────────────────────────────────────────────────────────
|
|
# Each server owns its own DDNS — ON when that server has internet.
|
|
# Script controls DDNS exclusively — network state NEVER auto-starts DDNS.
|
|
# DDNS only starts after full handback sequence confirms containers are up.
|
|
# One DDNS per domain active at all times — never two, never zero for long.
|
|
# 1 minute TTL + 1 minute check interval = minimal user impact on failover.
|
|
#
|
|
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
|
# When remote returns after FAILOVER:
|
|
# 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
|
|
# prevents handing back during a brief network blip
|
|
# 2. Pre-flight checks — remote array started, Docker healthy, rootfs not full
|
|
# 3. Stop remote DDNS first — prevents split brain DNS during rsync
|
|
# 4. Stop remote containers — clean state, no dirty writes during rsync
|
|
# 5. Tiered rsync writeback — skip if outage under threshold (short outages = cleaner to skip)
|
|
# Tier 1: skip if under HOST1_TIER1_WRITEBACK_DELAY (60min default)
|
|
# Tier 2: skip if under HOST1_TIER2_DELAY
|
|
# Tier 3: skip if under HOST1_TIER3_DELAY
|
|
# Tier 4: skip if under HOST*_TIER4_DELAY — opposing daily sync shares + edge cases
|
|
# 6. Start local containers — confirmed up before DNS cuts over
|
|
# 7. Start local DDNS — DNS cuts back ONLY after containers confirmed up
|
|
# 8. Return to NORMAL
|
|
#
|
|
# ── TIERED FAILOVER ───────────────────────────────────────────────────────────────────────────
|
|
# Tier 1 — Immediate — vital services + Live TV — can't wait
|
|
# Tier 2 — 2hr default — shared productivity services
|
|
# Tier 3 — 6hr default — secondary services
|
|
# Tier 4 — 18hr default — arrs + downloaders — minimal writeback on handback
|
|
# Delays configurable per host in Master.conf
|
|
#
|
|
# ── TO STOP THIS SCRIPT ───────────────────────────────────────────────────────────────────────
|
|
# Click Abort beside the script in unRAID's User Scripts plugin page.
|
|
# Do NOT kill the process directly — state file may be left in inconsistent state.
|
|
#
|
|
# All configuration in Master.conf under Failover section.
|
|
# Supports --dry-run (no container changes) and --status (show current state and exit).
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
acquire_lock # failover.sh — strict single instance, notifies on collision
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# STATE FILE HELPERS
|
|
# State file format — key=value, one per line, persists on /boot/
|
|
# Tracks: state, failover_start, handback_strikes, tier2_started, tier3_started, tier4_started
|
|
# ==============================================================================================
|
|
|
|
state_get() {
|
|
grep "^${1}=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2
|
|
}
|
|
|
|
state_set() {
|
|
local key="$1" value="$2"
|
|
if grep -q "^${key}=" "$FAILOVER_STATE_FILE" 2>/dev/null; then
|
|
sed -i "s|^${key}=.*|${key}=${value}|" "$FAILOVER_STATE_FILE"
|
|
else
|
|
echo "${key}=${value}" >> "$FAILOVER_STATE_FILE"
|
|
fi
|
|
}
|
|
|
|
state_init() {
|
|
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
|
|
[[ ! -f "$FAILOVER_STATE_FILE" ]] && touch "$FAILOVER_STATE_FILE"
|
|
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
|
[[ -z "$(state_get failover_start)" ]] && state_set failover_start "0"
|
|
[[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0"
|
|
[[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false"
|
|
[[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false"
|
|
[[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false"
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# CONTAINER HELPERS
|
|
# ==============================================================================================
|
|
|
|
# Start a container locally
|
|
local_start() {
|
|
local container="$1"
|
|
[[ -z "$container" ]] && return
|
|
local status
|
|
status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
if [[ "$status" == "true" ]]; then
|
|
log "$container already running locally"
|
|
return 0
|
|
fi
|
|
info "$ICON_START Starting $container locally..."
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
docker start "$container" >/dev/null 2>&1 && \
|
|
success "$ICON_STARTED $container started" || \
|
|
error "Failed to start $container locally"
|
|
else
|
|
warn "DRY RUN — would start $container locally"
|
|
fi
|
|
}
|
|
|
|
# Stop a container locally
|
|
local_stop() {
|
|
local container="$1"
|
|
[[ -z "$container" ]] && return
|
|
local status
|
|
status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
if [[ "$status" != "true" ]]; then
|
|
log "$container already stopped locally"
|
|
return 0
|
|
fi
|
|
info "$ICON_STOP Stopping $container locally..."
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
docker stop "$container" >/dev/null 2>&1 && \
|
|
success "$ICON_STOPPED $container stopped" || \
|
|
error "Failed to stop $container locally"
|
|
else
|
|
warn "DRY RUN — would stop $container locally"
|
|
fi
|
|
}
|
|
|
|
# Start a container on remote via SSH
|
|
remote_start() {
|
|
local container="$1"
|
|
[[ -z "$container" ]] && return
|
|
local status
|
|
status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
|
|
if [[ "$status" == "true" ]]; then
|
|
log "$container already running on $REMOTE_SERVER_NAME"
|
|
return 0
|
|
fi
|
|
info "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"docker start $container" >/dev/null 2>&1 && \
|
|
success "$ICON_STARTED $container started on $REMOTE_SERVER_NAME" || \
|
|
error "Failed to start $container on $REMOTE_SERVER_NAME"
|
|
else
|
|
warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
|
|
fi
|
|
}
|
|
|
|
# Stop a container on remote via SSH
|
|
remote_stop() {
|
|
local container="$1"
|
|
[[ -z "$container" ]] && return
|
|
local status
|
|
status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
|
|
if [[ "$status" != "true" ]]; then
|
|
log "$container already stopped on $REMOTE_SERVER_NAME"
|
|
return 0
|
|
fi
|
|
info "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"docker stop $container" >/dev/null 2>&1 && \
|
|
success "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
|
|
error "Failed to stop $container on $REMOTE_SERVER_NAME"
|
|
else
|
|
warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# DDNS HELPERS
|
|
# DDNS is managed exclusively by this script — never by network state
|
|
# ==============================================================================================
|
|
|
|
# Stop local DDNS — called on internet loss
|
|
local_ddns_stop() {
|
|
info "$ICON_NET Stopping local DDNS — internet lost"
|
|
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
|
local_stop "$container"
|
|
done
|
|
}
|
|
|
|
# Start local DDNS — called at end of handback ONLY after containers confirmed up
|
|
local_ddns_start() {
|
|
info "$ICON_NET Starting local DDNS — containers confirmed up"
|
|
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
|
local_start "$container"
|
|
done
|
|
}
|
|
|
|
# Stop remote DDNS — FIRST step of handback sequence, prevents split brain
|
|
remote_ddns_stop() {
|
|
info "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — handback starting"
|
|
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
|
remote_stop "$container"
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# TIERED FAILOVER HELPERS
|
|
# Containers selected based on which server is local and which arrays to use
|
|
# ==============================================================================================
|
|
|
|
# Get the correct tier arrays based on local host
|
|
# HOST2 runs for HOST1, HOST1 runs for HOST2
|
|
get_tier_containers() {
|
|
local tier="$1"
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
|
# HOST2 is running — covers HOST1
|
|
case "$tier" in
|
|
1) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE[@]}" ;;
|
|
2) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_2HR[@]}" ;;
|
|
3) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_6HR[@]}" ;;
|
|
4) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_18HR[@]}" ;;
|
|
esac
|
|
else
|
|
# HOST1 is running — covers HOST2
|
|
case "$tier" in
|
|
1) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE[@]}" ;;
|
|
2) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_2HR[@]}" ;;
|
|
3) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_6HR[@]}" ;;
|
|
4) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_18HR[@]}" ;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
get_tier2_delay() {
|
|
[[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \
|
|
echo "$HOST1_TIER2_DELAY" || echo "$HOST2_TIER2_DELAY"
|
|
}
|
|
|
|
get_tier3_delay() {
|
|
[[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \
|
|
echo "$HOST1_TIER3_DELAY" || echo "$HOST2_TIER3_DELAY"
|
|
}
|
|
|
|
get_tier4_delay() {
|
|
[[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \
|
|
echo "$HOST1_TIER4_DELAY" || echo "$HOST2_TIER4_DELAY"
|
|
}
|
|
|
|
get_tier1_writeback_delay() {
|
|
[[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \
|
|
echo "$HOST1_TIER1_WRITEBACK_DELAY" || echo "$HOST2_TIER1_WRITEBACK_DELAY"
|
|
}
|
|
|
|
get_writeback_jobs_for_tier() {
|
|
local tier="$1"
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
|
# HOST2 is covering HOST1 — write back HOST1's data
|
|
case "$tier" in
|
|
1) echo "${FAILOVER_HOST1_WRITEBACK_TIER1[@]}" ;;
|
|
2) echo "${FAILOVER_HOST1_WRITEBACK_TIER2[@]}" ;;
|
|
3) echo "${FAILOVER_HOST1_WRITEBACK_TIER3[@]}" ;;
|
|
4)
|
|
# Tier 4 — push HOST1's daily sync shares back (opposing orch list)
|
|
# These are HOST1's source-of-truth shares that HOST2's arrs managed during outage
|
|
# Plus any edge case paths defined in FAILOVER_HOST1_WRITEBACK_TIER4
|
|
echo "${HOST1_DAILY_SYNC_SHARES[@]}"
|
|
echo "${FAILOVER_HOST1_WRITEBACK_TIER4[@]}"
|
|
;;
|
|
esac
|
|
else
|
|
# HOST1 is covering HOST2 — write back HOST2's data
|
|
case "$tier" in
|
|
1) echo "${FAILOVER_HOST2_WRITEBACK_TIER1[@]}" ;;
|
|
2) echo "${FAILOVER_HOST2_WRITEBACK_TIER2[@]}" ;;
|
|
3) echo "${FAILOVER_HOST2_WRITEBACK_TIER3[@]}" ;;
|
|
4)
|
|
# Tier 4 — push HOST2's daily sync shares back (opposing orch list)
|
|
# These are HOST2's source-of-truth shares that HOST1's arrs managed during outage
|
|
# Plus any edge case paths defined in FAILOVER_HOST2_WRITEBACK_TIER4
|
|
echo "${HOST2_DAILY_SYNC_SHARES[@]}"
|
|
echo "${FAILOVER_HOST2_WRITEBACK_TIER4[@]}"
|
|
;;
|
|
esac
|
|
fi
|
|
}
|
|
|
|
# Select correct DDNS arrays based on local host
|
|
set_ddns_arrays() {
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
|
LOCAL_DDNS_CONTAINERS=("${HOST1_DDNS_CONTAINERS[@]}")
|
|
REMOTE_DDNS_CONTAINERS=("${HOST2_DDNS_CONTAINERS[@]}")
|
|
else
|
|
LOCAL_DDNS_CONTAINERS=("${HOST2_DDNS_CONTAINERS[@]}")
|
|
REMOTE_DDNS_CONTAINERS=("${HOST1_DDNS_CONTAINERS[@]}")
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# STATUS DISPLAY
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
state_init
|
|
set_ddns_arrays
|
|
CURRENT_STATE=$(state_get state)
|
|
FAILOVER_START=$(state_get failover_start)
|
|
TIER2=$(state_get tier2_started)
|
|
TIER3=$(state_get tier3_started)
|
|
TIER4=$(state_get tier4_started)
|
|
STRIKES=$(state_get handback_strikes)
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━"
|
|
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
|
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
|
echo "$ICON_FAILOVER State: $CURRENT_STATE"
|
|
echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]}"
|
|
echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]}"
|
|
echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s"
|
|
echo "$ICON_FAILOVER Strikes: $STRIKES / $FAILOVER_HANDBACK_STRIKES"
|
|
|
|
if [[ "$CURRENT_STATE" == "FAILOVER" && "$FAILOVER_START" -gt 0 ]]; then
|
|
ELAPSED=$(( ($(date +%s) - FAILOVER_START) / 60 ))
|
|
echo "$ICON_TIME Outage: ${ELAPSED}min"
|
|
echo "$ICON_FAILOVER Tier 2: $TIER2 (delay: $(get_tier2_delay)min)"
|
|
echo "$ICON_FAILOVER Tier 3: $TIER3 (delay: $(get_tier3_delay)min)"
|
|
echo "$ICON_FAILOVER Tier 4: $TIER4 (delay: $(get_tier4_delay)min)"
|
|
fi
|
|
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# HANDBACK SEQUENCE
|
|
# Called when remote returns after FAILOVER state
|
|
# Critical sequencing — do not reorder without understanding the consequences
|
|
# ==============================================================================================
|
|
run_handback() {
|
|
echo ""
|
|
echo "━━━ $ICON_FAILOVER Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
info "Remote $REMOTE_SERVER_NAME has returned — beginning handback"
|
|
|
|
# ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
|
|
if ! check_remote_array; then
|
|
warn "Remote array not ready — aborting handback, will retry next cycle"
|
|
state_set handback_strikes 0
|
|
return 1
|
|
fi
|
|
|
|
if ! check_remote_docker; then
|
|
warn "Remote Docker not ready — aborting handback, will retry next cycle"
|
|
state_set handback_strikes 0
|
|
return 1
|
|
fi
|
|
|
|
success "Pre-flight checks passed"
|
|
|
|
# ── Step 2: Stop remote DDNS FIRST ──────────────────────────────────────────────────────
|
|
# CRITICAL — prevents split brain DNS during rsync window
|
|
# Remote DDNS must be OFF before any data moves
|
|
echo ""
|
|
echo "━━━ $ICON_NET DDNS Handoff ━━━"
|
|
remote_ddns_stop
|
|
info "Remote DDNS stopped — DNS will not update during rsync"
|
|
sleep 5 # brief pause to ensure DDNS stop propagates
|
|
|
|
# ── Step 3: Stop remote containers ──────────────────────────────────────────────────────
|
|
# Clean state before rsync — no dirty writes during transfer
|
|
# Containers are only down during the rsync window — minimise this time
|
|
echo ""
|
|
echo "━━━ $ICON_STOP Stop Remote Containers ━━━"
|
|
|
|
# Stop all tiers that were started — in reverse order
|
|
local ALL_FAILOVER_CONTAINERS=()
|
|
|
|
if [[ "$(state_get tier4_started)" == "true" ]]; then
|
|
read -r -a t4 <<< "$(get_tier_containers 4)"
|
|
ALL_FAILOVER_CONTAINERS+=("${t4[@]}")
|
|
fi
|
|
if [[ "$(state_get tier3_started)" == "true" ]]; then
|
|
read -r -a t3 <<< "$(get_tier_containers 3)"
|
|
ALL_FAILOVER_CONTAINERS+=("${t3[@]}")
|
|
fi
|
|
if [[ "$(state_get tier2_started)" == "true" ]]; then
|
|
read -r -a t2 <<< "$(get_tier_containers 2)"
|
|
ALL_FAILOVER_CONTAINERS+=("${t2[@]}")
|
|
fi
|
|
|
|
# Tier 1 always started — stop last (DDNS already stopped above)
|
|
read -r -a t1 <<< "$(get_tier_containers 1)"
|
|
# Remove DDNS containers from tier 1 list — already handled
|
|
for container in "${t1[@]}"; do
|
|
local is_ddns=false
|
|
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
|
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
|
done
|
|
[[ "$is_ddns" == false ]] && ALL_FAILOVER_CONTAINERS+=("$container")
|
|
done
|
|
|
|
for container in "${ALL_FAILOVER_CONTAINERS[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
local_stop "$container"
|
|
done
|
|
|
|
success "All failover containers stopped locally"
|
|
|
|
# ── Step 4: Rsync writeback ──────────────────────────────────────────────────────────────
|
|
# Tiered writeback with skip window — short outages do not benefit from writeback.
|
|
# Emby syncs every 30min dirty (live container). Clean sync runs nightly at 2:30am.
|
|
# After a short outage HOST1's clean nightly state is more reliable than HOST2's
|
|
# dirty sync accumulation — skip writeback entirely for short outages.
|
|
#
|
|
# Tier 1 — skip if under HOST1_TIER1_WRITEBACK_DELAY (default 60min)
|
|
# Tier 2 — skip if under HOST1_TIER2_DELAY (reused — if Tier 2 never started, skip)
|
|
# Tier 3 — skip if under HOST1_TIER3_DELAY (reused — same logic)
|
|
# Tier 4 — always writeback — 18hr+ means meaningful delta accumulated
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Rsync Writeback ━━━"
|
|
|
|
local outage_minutes=$(( ($(date +%s) - $(state_get failover_start)) / 60 ))
|
|
local tier1_wb_delay
|
|
tier1_wb_delay=$(get_tier1_writeback_delay)
|
|
|
|
info "Outage duration: ${outage_minutes}min"
|
|
|
|
# Check rsync gate before running any writeback
|
|
if ! check_rsync_enabled "FAILOVER"; then
|
|
warn "FAILOVER_RSYNC_ENABLED=false — skipping all writeback jobs"
|
|
warn "Handback will complete without syncing state back to primary"
|
|
return 0
|
|
fi
|
|
|
|
run_writeback_tier() {
|
|
local tier="$1"
|
|
local threshold="$2"
|
|
local label="$3"
|
|
local jobs
|
|
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
|
[[ ${#jobs[@]} -eq 0 ]] && return
|
|
|
|
if [[ "$outage_minutes" -ge "$threshold" ]]; then
|
|
info "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min"
|
|
for job in "${jobs[@]}"; do
|
|
[[ -z "$job" ]] && continue
|
|
info "Syncing: $job"
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
# Emby writeback uses emby-failover profile — critical data only, Emby stays running
|
|
if [[ "$(basename "$job")" == "Emby" ]]; then
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-failover
|
|
else
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
|
fi
|
|
else
|
|
warn "DRY RUN — would rsync: $job"
|
|
fi
|
|
done
|
|
else
|
|
info "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min threshold"
|
|
info "Primary has cleaner state — no writeback needed"
|
|
fi
|
|
}
|
|
|
|
run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack"
|
|
run_writeback_tier 2 "$HOST1_TIER2_DELAY" "NextCloud + Immich"
|
|
run_writeback_tier 3 "$HOST1_TIER3_DELAY" "secondary services"
|
|
run_writeback_tier 4 "$(get_tier4_delay)" "media shares + edge cases"
|
|
|
|
success "Writeback complete"
|
|
|
|
# ── Step 5: Start remote containers ─────────────────────────────────────────────────────
|
|
# Start in dependency order — databases before apps
|
|
echo ""
|
|
echo "━━━ $ICON_START Start Remote Containers ━━━"
|
|
|
|
# Start tier 1 on remote (excluding DDNS — handled separately)
|
|
for container in "${t1[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
local is_ddns=false
|
|
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
|
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
|
done
|
|
[[ "$is_ddns" == false ]] && remote_start "$container"
|
|
done
|
|
|
|
# Brief pause — give databases time to initialise before apps
|
|
sleep 10
|
|
|
|
if [[ "$(state_get tier2_started)" == "true" ]]; then
|
|
for container in "${t2[@]}"; do
|
|
[[ -n "$container" ]] && remote_start "$container"
|
|
done
|
|
fi
|
|
|
|
if [[ "$(state_get tier3_started)" == "true" ]]; then
|
|
for container in "${t3[@]}"; do
|
|
[[ -n "$container" ]] && remote_start "$container"
|
|
done
|
|
fi
|
|
|
|
if [[ "$(state_get tier4_started)" == "true" ]]; then
|
|
for container in "${t4[@]}"; do
|
|
[[ -n "$container" ]] && remote_start "$container"
|
|
done
|
|
fi
|
|
|
|
success "Remote containers started"
|
|
|
|
# ── Step 6: Start remote DDNS LAST ──────────────────────────────────────────────────────
|
|
# DNS cuts over ONLY after containers confirmed up
|
|
# This is the final step — after this users hit remote
|
|
echo ""
|
|
echo "━━━ $ICON_NET DNS Cutover ━━━"
|
|
sleep 5 # brief pause to ensure containers are accepting connections
|
|
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
|
remote_start "$container"
|
|
done
|
|
success "Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
|
|
|
|
# ── Step 7: Return to NORMAL ─────────────────────────────────────────────────────────────
|
|
echo ""
|
|
state_set state "NORMAL"
|
|
state_set failover_start "0"
|
|
state_set handback_strikes "0"
|
|
state_set tier2_started "false"
|
|
state_set tier3_started "false"
|
|
state_set tier4_started "false"
|
|
|
|
success "$ICON_DONE Handback complete — returned to NORMAL"
|
|
notify "Failover handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" "Failover" "normal"
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# MAIN STATE MACHINE
|
|
# ==============================================================================================
|
|
state_init
|
|
set_ddns_arrays
|
|
|
|
echo ""
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " $ICON_FAILOVER FAILOVER — $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo " $ICON_HOST $LOCAL_SERVER_NAME → monitoring $REMOTE_SERVER_NAME"
|
|
echo " $ICON_NET Remote: $REMOTE_SERVER"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
while true; do
|
|
NOW=$(date +%s)
|
|
CURRENT_STATE=$(state_get state)
|
|
|
|
# ── Connectivity checks ──────────────────────────────────────────────────────────────────
|
|
REMOTE_UP=false
|
|
INTERNET_UP=false
|
|
|
|
ping_remote && REMOTE_UP=true
|
|
ping_internet && INTERNET_UP=true
|
|
|
|
log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"
|
|
|
|
# ── State machine ────────────────────────────────────────────────────────────────────────
|
|
|
|
# ════════════════════════════════════════════════════════════════
|
|
# NORMAL STATE
|
|
# ════════════════════════════════════════════════════════════════
|
|
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
|
|
|
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
|
# All good — silent operation
|
|
log "$ICON_SUCCESS NORMAL — all systems up"
|
|
|
|
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
|
# Remote is down — enter FAILOVER
|
|
echo ""
|
|
echo "━━━ $ICON_FAILOVER Entering FAILOVER — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
warn "$REMOTE_SERVER_NAME is unreachable — internet is up — starting failover"
|
|
|
|
state_set state "FAILOVER"
|
|
state_set failover_start "$NOW"
|
|
state_set handback_strikes "0"
|
|
state_set tier2_started "false"
|
|
state_set tier3_started "false"
|
|
state_set tier4_started "false"
|
|
|
|
# Start Tier 1 immediately — DDNS first, then vital services
|
|
echo ""
|
|
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
|
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
|
for container in "${tier1[@]}"; do
|
|
[[ -n "$container" ]] && local_start "$container"
|
|
done
|
|
|
|
notify "FAILOVER started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 containers started" "Failover" "warning"
|
|
|
|
elif [[ "$INTERNET_UP" == false ]]; then
|
|
# Lost internet — enter NO_INTERNET
|
|
echo ""
|
|
echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
warn "Internet connectivity lost — stopping local DDNS"
|
|
|
|
state_set state "NO_INTERNET"
|
|
local_ddns_stop
|
|
|
|
# Stop any containers configured to stop without internet
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
|
for container in "${FAILOVER_HOST1_STOP_ON_NO_NET[@]}"; do
|
|
[[ -n "$container" ]] && local_stop "$container"
|
|
done
|
|
else
|
|
for container in "${FAILOVER_HOST2_STOP_ON_NO_NET[@]}"; do
|
|
[[ -n "$container" ]] && local_stop "$container"
|
|
done
|
|
fi
|
|
|
|
notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" "Failover" "warning"
|
|
fi
|
|
|
|
# ════════════════════════════════════════════════════════════════
|
|
# FAILOVER STATE
|
|
# ════════════════════════════════════════════════════════════════
|
|
elif [[ "$CURRENT_STATE" == "FAILOVER" ]]; then
|
|
|
|
FAILOVER_START=$(state_get failover_start)
|
|
ELAPSED_MIN=$(( (NOW - FAILOVER_START) / 60 ))
|
|
|
|
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
|
# Remote returned — increment handback strikes
|
|
STRIKES=$(state_get handback_strikes)
|
|
STRIKES=$(( STRIKES + 1 ))
|
|
state_set handback_strikes "$STRIKES"
|
|
|
|
info "$ICON_FAILOVER $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FAILOVER_HANDBACK_STRIKES"
|
|
|
|
if [[ "$STRIKES" -ge "$FAILOVER_HANDBACK_STRIKES" ]]; then
|
|
run_handback
|
|
fi
|
|
|
|
elif [[ "$INTERNET_UP" == false ]]; then
|
|
# Lost internet during failover — enter DARK
|
|
echo ""
|
|
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
warn "Lost internet during failover — entering DARK state"
|
|
state_set state "DARK"
|
|
local_ddns_stop
|
|
notify "DARK state on $(hostname) — lost internet during failover" "Failover" "warning"
|
|
|
|
else
|
|
# Still in failover — check tier escalation
|
|
state_set handback_strikes "0" # reset strikes — remote still down
|
|
|
|
# Tier 2
|
|
TIER2_DELAY=$(get_tier2_delay)
|
|
if [[ "$(state_get tier2_started)" == "false" && \
|
|
"$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
|
|
read -r -a tier2 <<< "$(get_tier_containers 2)"
|
|
for container in "${tier2[@]}"; do
|
|
[[ -n "$container" ]] && local_start "$container"
|
|
done
|
|
state_set tier2_started "true"
|
|
notify "Failover Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" "Failover" "warning"
|
|
fi
|
|
|
|
# Tier 3
|
|
TIER3_DELAY=$(get_tier3_delay)
|
|
if [[ "$(state_get tier3_started)" == "false" && \
|
|
"$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
|
|
read -r -a tier3 <<< "$(get_tier_containers 3)"
|
|
for container in "${tier3[@]}"; do
|
|
[[ -n "$container" ]] && local_start "$container"
|
|
done
|
|
state_set tier3_started "true"
|
|
notify "Failover Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" "Failover" "warning"
|
|
fi
|
|
|
|
# Tier 4
|
|
TIER4_DELAY=$(get_tier4_delay)
|
|
if [[ "$(state_get tier4_started)" == "false" && \
|
|
"$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
|
|
read -r -a tier4 <<< "$(get_tier_containers 4)"
|
|
for container in "${tier4[@]}"; do
|
|
[[ -n "$container" ]] && local_start "$container"
|
|
done
|
|
state_set tier4_started "true"
|
|
notify "Failover Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" "Failover" "warning"
|
|
fi
|
|
|
|
log "$ICON_FAILOVER FAILOVER active — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
|
|
fi
|
|
|
|
# ════════════════════════════════════════════════════════════════
|
|
# NO_INTERNET STATE
|
|
# ════════════════════════════════════════════════════════════════
|
|
elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then
|
|
|
|
if [[ "$INTERNET_UP" == true ]]; then
|
|
# Internet recovered
|
|
echo ""
|
|
echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if [[ "$REMOTE_UP" == true ]]; then
|
|
# Remote up, internet up — return to NORMAL
|
|
info "Remote is up — returning to NORMAL"
|
|
state_set state "NORMAL"
|
|
# DDNS does NOT auto-start here — it only starts via handback sequence
|
|
# or explicit NORMAL state management
|
|
# If local DDNS should be running in NORMAL — start it now
|
|
local_ddns_start
|
|
notify "Internet recovered on $(hostname) — returning to NORMAL" "Failover" "normal"
|
|
else
|
|
# Internet back but remote still down — enter FAILOVER
|
|
info "Remote still down — entering FAILOVER"
|
|
state_set state "FAILOVER"
|
|
state_set failover_start "$NOW"
|
|
state_set handback_strikes "0"
|
|
state_set tier2_started "false"
|
|
state_set tier3_started "false"
|
|
state_set tier4_started "false"
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
|
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
|
for container in "${tier1[@]}"; do
|
|
[[ -n "$container" ]] && local_start "$container"
|
|
done
|
|
notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FAILOVER" "Failover" "warning"
|
|
fi
|
|
else
|
|
log "$ICON_NET NO_INTERNET — waiting for connectivity"
|
|
fi
|
|
|
|
# ════════════════════════════════════════════════════════════════
|
|
# DARK STATE
|
|
# ════════════════════════════════════════════════════════════════
|
|
elif [[ "$CURRENT_STATE" == "DARK" ]]; then
|
|
|
|
if [[ "$INTERNET_UP" == true ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if [[ "$REMOTE_UP" == true ]]; then
|
|
info "Remote up, internet up — returning to NORMAL"
|
|
# Was in failover before DARK — need to handback
|
|
state_set state "FAILOVER"
|
|
state_set handback_strikes "0"
|
|
# Will trigger handback on next cycle via FAILOVER + remote up logic
|
|
else
|
|
info "Internet back but remote still down — entering FAILOVER"
|
|
state_set state "FAILOVER"
|
|
state_set failover_start "$NOW"
|
|
state_set handback_strikes "0"
|
|
state_set tier2_started "false"
|
|
state_set tier3_started "false"
|
|
state_set tier4_started "false"
|
|
|
|
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
|
for container in "${tier1[@]}"; do
|
|
[[ -n "$container" ]] && local_start "$container"
|
|
done
|
|
fi
|
|
notify "Emerging from DARK state on $(hostname)" "Failover" "warning"
|
|
else
|
|
log "$ICON_FAILOVER DARK — no internet, no remote — waiting"
|
|
fi
|
|
fi
|
|
|
|
# ── Sleep until next check ───────────────────────────────────────────────────────────────
|
|
log "Next check in ${FAILOVER_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
|
|
sleep "$FAILOVER_CHECK_INTERVAL"
|
|
|
|
done |