512 lines
19 KiB
Bash
512 lines
19 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Failover Script --------------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Mutual container failover between two unRAID servers.
|
|
# Runs continuously on BOTH servers — each operates fully autonomously.
|
|
# No coordination between servers — decisions based solely on ping results.
|
|
#
|
|
# States:
|
|
# NORMAL — remote up, internet up — own containers only
|
|
# FAILOVER — remote down, internet up — own + remote's containers (additive)
|
|
# NO_INTERNET — internet down — stop public-facing containers
|
|
# DARK — remote down + internet down — same as NO_INTERNET
|
|
#
|
|
# Handback (remote returns after FAILOVER):
|
|
# Strike confirmation → pre-flight checks → rsync → start on remote → stop locally
|
|
#
|
|
# All configuration in Master.conf under Failover section.
|
|
# Run via User Scripts plugin as a background task — runs continuously until stopped.
|
|
# Supports --dry-run to walk through logic without taking any action.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
# Select correct arrays based on which server we are
|
|
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
|
FAILOVER_START_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}")
|
|
FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST1_STOP_ON_NO_NET[@]}")
|
|
FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST1_RSYNC_JOBS[@]}")
|
|
else
|
|
FAILOVER_START_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}")
|
|
FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST2_STOP_ON_NO_NET[@]}")
|
|
FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST2_RSYNC_JOBS[@]}")
|
|
fi
|
|
|
|
info "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}"
|
|
info "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}"
|
|
info "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}"
|
|
|
|
# Ensure state file directory exists
|
|
touch "$FAILOVER_STATE_FILE" 2>/dev/null || {
|
|
error "Cannot create state file: $FAILOVER_STATE_FILE"
|
|
exit 1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Status ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━"
|
|
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
|
echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
|
echo "$ICON_PING External IP: $EXTERNAL_IP"
|
|
echo "$ICON_TIME Check interval: ${FAILOVER_CHECK_INTERVAL}s"
|
|
echo "$ICON_RETRY Handback strikes: $FAILOVER_HANDBACK_STRIKES"
|
|
echo "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}"
|
|
echo "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}"
|
|
echo "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}"
|
|
echo "$ICON_GEAR State file: $FAILOVER_STATE_FILE"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
|
|
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
|
echo ""
|
|
echo "━━━ Persisted State ━━━"
|
|
cat "$FAILOVER_STATE_FILE"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be started or stopped"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# STATE FILE HELPERS
|
|
# State persists to /boot/ so it survives reboots.
|
|
# On restart the script re-evaluates from scratch using live ping results.
|
|
# State file is reference only — pings are always the source of truth.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
get_state() {
|
|
grep -E "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2
|
|
}
|
|
|
|
set_state() {
|
|
local new_state="$1"
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
cat > "$FAILOVER_STATE_FILE" <<EOF
|
|
state=$new_state
|
|
last_change=$timestamp
|
|
host=$LOCAL_SERVER_NAME
|
|
EOF
|
|
log "State set to $new_state"
|
|
}
|
|
|
|
get_handback_strikes() {
|
|
grep -E "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2
|
|
}
|
|
|
|
set_handback_strikes() {
|
|
local count="$1"
|
|
if grep -q "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null; then
|
|
sed -i "s/^handback_strikes=.*/handback_strikes=$count/" "$FAILOVER_STATE_FILE"
|
|
else
|
|
echo "handback_strikes=$count" >> "$FAILOVER_STATE_FILE"
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# CONTAINER HELPERS — LOCAL operations for failover
|
|
# These operate on the LOCAL server unlike stop_containers/start_containers in common.sh
|
|
# which operate on the REMOTE server via SSH.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
# Start a container locally — skip if already running
|
|
start_local_container() {
|
|
local container="$1"
|
|
|
|
local status
|
|
status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$status" == "true" ]]; then
|
|
log "$container already running locally — skipping"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$status" == "unknown" ]]; then
|
|
warn "$container not found on this host — skipping"
|
|
return 1
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would start $container locally"
|
|
return 0
|
|
fi
|
|
|
|
echo "$ICON_START Starting $container locally..."
|
|
if docker start "$container" >/dev/null 2>&1; then
|
|
echo "$ICON_STARTED $container started"
|
|
return 0
|
|
else
|
|
error "Failed to start $container locally"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Stop a container locally — skip if already stopped
|
|
stop_local_container() {
|
|
local container="$1"
|
|
|
|
local status
|
|
status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$status" == "false" ]]; then
|
|
log "$container already stopped locally — skipping"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$status" == "unknown" ]]; then
|
|
log "$container not found on this host — skipping"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would stop $container locally"
|
|
return 0
|
|
fi
|
|
|
|
echo "$ICON_STOP Stopping $container locally..."
|
|
if docker stop "$container" >/dev/null 2>&1; then
|
|
echo "$ICON_STOPPED $container stopped"
|
|
return 0
|
|
else
|
|
error "Failed to stop $container locally"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Start a container on the remote server via SSH
|
|
start_remote_container() {
|
|
local container="$1"
|
|
|
|
local status
|
|
status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"docker inspect -f '{{.State.Running}}' $container 2>/dev/null || echo unknown" 2>/dev/null)
|
|
|
|
if [[ "$status" == "true" ]]; then
|
|
log "$container already running on remote — skipping"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
|
|
return 0
|
|
fi
|
|
|
|
echo "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
|
|
if ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
|
"docker start $container" >/dev/null 2>&1; then
|
|
echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME"
|
|
return 0
|
|
else
|
|
error "Failed to start $container on $REMOTE_SERVER_NAME"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# STATE ACTIONS
|
|
# Each state has a corresponding action function.
|
|
# These are idempotent — safe to call on every loop iteration.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
# NORMAL — remote is up, internet is up
|
|
# Ensure failover containers are stopped locally (cleanup after returning from FAILOVER)
|
|
action_normal() {
|
|
local prev_state="$1"
|
|
|
|
if [[ "$prev_state" == "FAILOVER" ]]; then
|
|
info "$ICON_FAILOVER Returning from FAILOVER — cleaning up local failover containers"
|
|
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
stop_local_container "$c"
|
|
done
|
|
notify "Failover ended on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME is back online" "Failover" "normal"
|
|
fi
|
|
|
|
# Ensure no-net containers are running (they may have been stopped)
|
|
if [[ "$prev_state" == "NO_INTERNET" || "$prev_state" == "DARK" ]]; then
|
|
info "$ICON_START Internet restored — starting previously stopped containers"
|
|
for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
start_local_container "$c"
|
|
done
|
|
notify "Internet restored on $LOCAL_SERVER_NAME — normal containers restarted" "Failover" "normal"
|
|
fi
|
|
}
|
|
|
|
# FAILOVER — remote is down, internet is up
|
|
# Start remote's containers locally (additive — own containers keep running)
|
|
action_failover() {
|
|
local prev_state="$1"
|
|
|
|
if [[ "$prev_state" != "FAILOVER" ]]; then
|
|
# Just entered failover — notify and start containers
|
|
info "$ICON_FAILOVER Entering FAILOVER — $REMOTE_SERVER_NAME is unreachable"
|
|
|
|
# Check local array before starting containers
|
|
if ! check_local_array; then
|
|
error "Local array not ready — cannot start failover containers safely"
|
|
notify "Failover triggered but local array not ready on $LOCAL_SERVER_NAME" "Failover" "warning"
|
|
return 1
|
|
fi
|
|
|
|
# Check local docker daemon
|
|
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
|
error "Local Docker daemon not responding — cannot start failover containers"
|
|
notify "Failover triggered but Docker not ready on $LOCAL_SERVER_NAME" "Failover" "warning"
|
|
return 1
|
|
fi
|
|
|
|
notify "Failover ACTIVE on $LOCAL_SERVER_NAME — starting ${REMOTE_SERVER_NAME} containers: ${FAILOVER_START_CONTAINERS[*]}" "Failover" "warning"
|
|
fi
|
|
|
|
# Start each failover container — idempotent, skips if already running
|
|
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
start_local_container "$c"
|
|
done
|
|
}
|
|
|
|
# NO_INTERNET / DARK — this server has no internet
|
|
# Stop public-facing containers — no point serving if offline
|
|
action_no_internet() {
|
|
local prev_state="$1"
|
|
|
|
if [[ "$prev_state" != "NO_INTERNET" && "$prev_state" != "DARK" ]]; then
|
|
warn "$ICON_WARN Internet lost on $LOCAL_SERVER_NAME — stopping public containers"
|
|
notify "Internet lost on $LOCAL_SERVER_NAME — stopping public-facing containers" "Failover" "warning"
|
|
fi
|
|
|
|
for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
stop_local_container "$c"
|
|
done
|
|
|
|
# If we were in FAILOVER, also stop the failover containers
|
|
if [[ "$prev_state" == "FAILOVER" ]]; then
|
|
info "Was in FAILOVER — stopping failover containers too"
|
|
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
stop_local_container "$c"
|
|
done
|
|
fi
|
|
}
|
|
|
|
# HANDBACK — remote has returned, ready to hand containers back
|
|
# Confirmed by FAILOVER_HANDBACK_STRIKES consecutive remote-up checks
|
|
action_handback() {
|
|
info "$ICON_FAILOVER Initiating handback to $REMOTE_SERVER_NAME..."
|
|
notify "Handback starting on $LOCAL_SERVER_NAME — syncing data to $REMOTE_SERVER_NAME" "Failover" "normal"
|
|
|
|
# Pre-flight checks before rsync
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Handback Pre-flight ━━━"
|
|
|
|
if ! check_remote_array; then
|
|
error "Remote array not ready — deferring handback"
|
|
notify "Handback deferred on $LOCAL_SERVER_NAME — remote array not ready" "Failover" "warning"
|
|
set_handback_strikes 0
|
|
return 1
|
|
fi
|
|
success "Remote array ready"
|
|
|
|
if ! check_remote_docker; then
|
|
error "Remote Docker not ready — deferring handback"
|
|
notify "Handback deferred on $LOCAL_SERVER_NAME — remote Docker not ready" "Failover" "warning"
|
|
set_handback_strikes 0
|
|
return 1
|
|
fi
|
|
success "Remote Docker ready"
|
|
|
|
REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
|
"df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null)
|
|
if [[ -n "$REMOTE_USAGE" ]] && [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then
|
|
error "Remote rootfs ${REMOTE_USAGE}% full — deferring handback"
|
|
notify "Handback deferred — remote rootfs ${REMOTE_USAGE}% full on $REMOTE_SERVER_NAME" "Failover" "warning"
|
|
set_handback_strikes 0
|
|
return 1
|
|
fi
|
|
success "Remote rootfs healthy"
|
|
|
|
# Rsync data back via rsync.sh — uses existing profile system
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Handback Rsync ━━━"
|
|
local rsync_failed=false
|
|
|
|
for path in "${FAILOVER_RSYNC_JOBS[@]}"; do
|
|
[[ -z "$path" ]] && continue
|
|
info "Syncing $path → $REMOTE_SERVER_NAME"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would rsync $path"
|
|
else
|
|
if bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$path" --no-log; then
|
|
success "Rsync complete: $path"
|
|
else
|
|
error "Rsync failed: $path"
|
|
rsync_failed=true
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [[ "$rsync_failed" == true ]]; then
|
|
error "One or more rsync jobs failed — deferring container handback"
|
|
notify "Handback rsync failed on $LOCAL_SERVER_NAME — containers not handed back yet" "Failover" "warning"
|
|
set_handback_strikes 0
|
|
return 1
|
|
fi
|
|
|
|
# Start containers on remote
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Start on Remote ━━━"
|
|
local start_failed=false
|
|
|
|
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
if ! start_remote_container "$c"; then
|
|
start_failed=true
|
|
fi
|
|
done
|
|
|
|
if [[ "$start_failed" == true ]]; then
|
|
error "One or more containers failed to start on remote — not stopping local copies"
|
|
notify "Handback partial failure on $LOCAL_SERVER_NAME — some containers failed to start on $REMOTE_SERVER_NAME" "Failover" "warning"
|
|
set_handback_strikes 0
|
|
return 1
|
|
fi
|
|
|
|
# Stop local failover containers — only after remote confirmed started
|
|
echo ""
|
|
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Local Failover ━━━"
|
|
for c in "${FAILOVER_START_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
stop_local_container "$c"
|
|
done
|
|
|
|
notify "Handback complete on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME has containers back" "Failover" "normal"
|
|
set_handback_strikes 0
|
|
return 0
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_FAILOVER Main Loop ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_FAILOVER Failover Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST Local: $LOCAL_SERVER_NAME"
|
|
echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
|
echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s"
|
|
echo ""
|
|
|
|
notify "Failover script started on $LOCAL_SERVER_NAME — monitoring $REMOTE_SERVER_NAME" "Failover" "normal"
|
|
|
|
LOOP_COUNT=0
|
|
|
|
while true; do
|
|
LOOP_COUNT=$((LOOP_COUNT + 1))
|
|
CURRENT_STATE=$(get_state)
|
|
[[ -z "$CURRENT_STATE" ]] && CURRENT_STATE="UNKNOWN"
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_PING Check #${LOOP_COUNT} — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
log "Previous state: $CURRENT_STATE"
|
|
|
|
# ── Two ping checks — the only inputs to state decisions ──
|
|
REMOTE_UP=false
|
|
INTERNET_UP=false
|
|
|
|
ping_remote && REMOTE_UP=true
|
|
ping_internet && INTERNET_UP=true
|
|
|
|
log "Remote reachable: $REMOTE_UP | Internet reachable: $INTERNET_UP"
|
|
|
|
# ── Determine new state ──
|
|
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
|
NEW_STATE="NORMAL"
|
|
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
|
NEW_STATE="FAILOVER"
|
|
elif [[ "$INTERNET_UP" == false ]]; then
|
|
NEW_STATE="NO_INTERNET"
|
|
else
|
|
NEW_STATE="DARK"
|
|
fi
|
|
|
|
# ── Handle handback strike system ──
|
|
# When returning from FAILOVER to NORMAL, require consecutive confirmations
|
|
if [[ "$CURRENT_STATE" == "FAILOVER" && "$NEW_STATE" == "NORMAL" ]]; then
|
|
STRIKES=$(get_handback_strikes)
|
|
[[ -z "$STRIKES" ]] && STRIKES=0
|
|
STRIKES=$((STRIKES + 1))
|
|
set_handback_strikes "$STRIKES"
|
|
|
|
if [[ "$STRIKES" -lt "$FAILOVER_HANDBACK_STRIKES" ]]; then
|
|
info "$ICON_RETRY Remote appears back — strike $STRIKES/$FAILOVER_HANDBACK_STRIKES — waiting for confirmation"
|
|
NEW_STATE="FAILOVER" # Stay in FAILOVER until strikes confirmed
|
|
else
|
|
info "$ICON_RETRY Remote confirmed stable ($STRIKES/$FAILOVER_HANDBACK_STRIKES) — initiating handback"
|
|
NEW_STATE="HANDBACK"
|
|
fi
|
|
else
|
|
# Reset handback strikes if we're not in the confirmation window
|
|
set_handback_strikes 0
|
|
fi
|
|
|
|
# ── Log state ──
|
|
if [[ "$NEW_STATE" != "$CURRENT_STATE" ]]; then
|
|
echo "$ICON_FAILOVER State: $CURRENT_STATE → $NEW_STATE"
|
|
else
|
|
info "State: $NEW_STATE (unchanged)"
|
|
fi
|
|
|
|
# ── Execute state action ──
|
|
case "$NEW_STATE" in
|
|
NORMAL)
|
|
action_normal "$CURRENT_STATE"
|
|
set_state "NORMAL"
|
|
;;
|
|
FAILOVER)
|
|
action_failover "$CURRENT_STATE"
|
|
set_state "FAILOVER"
|
|
;;
|
|
NO_INTERNET|DARK)
|
|
action_no_internet "$CURRENT_STATE"
|
|
set_state "$NEW_STATE"
|
|
;;
|
|
HANDBACK)
|
|
if action_handback; then
|
|
set_state "NORMAL"
|
|
action_normal "FAILOVER"
|
|
else
|
|
# Handback failed — stay in FAILOVER, retry next cycle
|
|
set_state "FAILOVER"
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
echo "$ICON_TIME Next check in ${FAILOVER_CHECK_INTERVAL}s"
|
|
sleep "$FAILOVER_CHECK_INTERVAL"
|
|
done |