Gmer4Lfe
01e4f97361
Retries up to PLAY_SYNC_HANDBACK_RETRIES times (default 5, 60s apart) before giving up — one successful run catches all state regardless of outage length, so users land on current watch state after DNS flips.
1137 lines
52 KiB
Bash
Executable File
1137 lines
52 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= Fallback ===================================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Mutual container fallback between two unRAID servers. Runs continuously as a
|
||
# background process — started at array start by array_started.sh. Each server
|
||
# runs this script independently with no coordination between servers.
|
||
#
|
||
# All decisions are based solely on two pings per cycle: remote Tailscale IP
|
||
# reachable + internet reachable. No SSH signaling, no shared state, no election
|
||
# algorithm. Each server acts entirely from its own network perspective.
|
||
#
|
||
# Never requires human intervention during normal fallback and handback.
|
||
# Stop only via `fallback.sh --stop` — do NOT kill directly (state file may corrupt).
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Each cycle (every FALLBACK_CHECK_INTERVAL seconds):
|
||
# 1. Ping remote Tailscale IP — is the other server reachable?
|
||
# 2. Ping internet (EXTERNAL_IP) — do I have internet?
|
||
# 3. Determine state from the two results
|
||
# 4. Take the action for that state
|
||
#
|
||
# States:
|
||
# NORMAL — remote up, internet up — own containers, own DDNS on, silent
|
||
# FALLBACK — remote down, internet up — start tier containers, DDNS over
|
||
# NO_INTERNET — internet down — stop own DDNS immediately, wait
|
||
# DARK — remote down AND internet down — same actions as NO_INTERNET
|
||
#
|
||
# FALLBACK escalation (tiered by outage duration):
|
||
# Tier 1 — immediate — vital services + Live TV (cannot wait)
|
||
# Tier 2 — after HOST*_TIER2_DELAY (default 4hr) — shared productivity
|
||
# Tier 3 — after HOST*_TIER3_DELAY (default 12hr) — secondary services
|
||
# Tier 4 — after HOST*_TIER4_DELAY (default 24hr) — arrs + downloaders
|
||
#
|
||
# Handback (when remote returns after FALLBACK):
|
||
# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
|
||
# 2. Pre-flight checks — version parity, remote array, remote Docker daemon
|
||
# 3. Staged reverse: Tier 4→3→2 — Emby stays on covering server throughout
|
||
# Each tier: stop local → rsync writeback → start on remote
|
||
# 4. DDNS handoff — stop remote DDNS before Tier 1 goes down
|
||
# 5. Tier 1 handback — stop local vital services, rsync writeback, start remote
|
||
# 6. Start remote DDNS — DNS cuts back ONLY after containers confirmed running
|
||
# 7. Return to NORMAL
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Pure Ping-Only Detection
|
||
# No SSH signaling, no election algorithm, no shared state between servers.
|
||
# Each server makes all decisions from its own vantage point. A server that
|
||
# cannot reach the remote and has internet acts — no coordination needed.
|
||
#
|
||
# DDNS Absolute Control
|
||
# This script is the sole authority over DDNS containers. Network state
|
||
# returning is not permission to start DDNS — only the completion of the
|
||
# full handback sequence grants that. Eliminates the split-brain DNS window
|
||
# where two servers update the same domain with different IPs simultaneously.
|
||
#
|
||
# Staged Reverse-Tier Handback
|
||
# Emby stays on the covering server serving users while Tiers 4→3→2 hand
|
||
# back. The user impact window (Emby down) is only the Tier 1 rsync
|
||
# duration — typically minutes. Users are served continuously until the
|
||
# final step.
|
||
#
|
||
# Strike Confirmation
|
||
# Handback requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks.
|
||
# Brief network recovery during an ongoing outage would otherwise trigger a
|
||
# failed handback: containers stop on the covering server, remote goes down
|
||
# again mid-rsync. Strikes prevent this.
|
||
#
|
||
# REMOTE_ID-Based Coverage Config
|
||
# Tier arrays, writeback paths, and delays are all defined in the covered
|
||
# host's own conf and read via REMOTE_ID — not MY_ID. Each host owns its
|
||
# complete recovery profile. The same script handles both directions.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Enforcement
|
||
# Docker and SSH operations require root.
|
||
#
|
||
# FALLBACK_ENABLED Gate
|
||
# Exits cleanly when disabled — safe to run on servers being rebuilt without
|
||
# triggering spurious fallback actions.
|
||
#
|
||
# Version Parity Check
|
||
# Refuses handback if remote unRAID version doesn't match. A mismatch may
|
||
# mean the remote is not fully ready or on an incompatible version.
|
||
#
|
||
# Remote Docker Daemon Check
|
||
# SSH-checks the remote Docker daemon is responding before any remote commands.
|
||
# Reachable on Tailscale does not mean Docker is ready.
|
||
#
|
||
# Container Verification
|
||
# After every local_start(), verifies the container came up and stayed running.
|
||
# Catches crashes-on-start before declaring fallback successful.
|
||
#
|
||
# Timeout Protection
|
||
# All docker commands (DOCKER_TIMEOUT=15s) and SSH commands (SSH_TIMEOUT=10s)
|
||
# wrapped in timeouts. A hung daemon does not block the state machine.
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock() prevents two instances running simultaneously. Both would
|
||
# modify containers and DDNS independently — conflict is certain.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() resolves MY_ID / REMOTE_ID from master.conf at startup.
|
||
# Exits if the host cannot be identified — prevents running on an unknown machine.
|
||
#
|
||
# Partnership Gate
|
||
# Fallback is only permitted when the local partnership DB reports an active
|
||
# partnership. If partnership goes inactive, a grace timer starts. After
|
||
# FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes the loop suspends: no new FALLBACK
|
||
# is entered, and any live fallback containers are stopped cleanly. Resumes
|
||
# automatically when partnership becomes active again.
|
||
#
|
||
# Silent by Default
|
||
# State transitions: warn() — always visible. Healthy routine cycles: log()
|
||
# — suppressed unless --log. Produces no output on clean cycles.
|
||
#
|
||
# ==============================================================================================
|
||
# STATE FILES
|
||
# ==============================================================================================
|
||
#
|
||
# FALLBACK_STATE_FILE — $STATE_DIR/fallback_state.db (survives reboots)
|
||
# Keys: state, fallback_start, handback_strikes, tier2_started,
|
||
# tier3_started, tier4_started, partnership_suspended, partner_lost_at.
|
||
# Survives reboots — $STATE_DIR is on the boot device (internal) or appdata
|
||
# (flash). Either way the array is up before this script runs, so the file
|
||
# is always accessible. If the server was in FALLBACK when it rebooted,
|
||
# it resumes FALLBACK on restart.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_DDNS_CONTAINERS
|
||
# DDNS containers this host manages — stopped on internet loss, started
|
||
# as the last step of handback
|
||
#
|
||
# FALLBACK_HOST*_STOP_ON_NO_NET
|
||
# Containers stopped when this host loses internet
|
||
#
|
||
# FALLBACK_HOST*_TIER1–4
|
||
# Containers started on this host when the partner is down — defined in the
|
||
# partner's own conf (what the partner wants covered). Read via REMOTE_ID.
|
||
# Variable pattern: FALLBACK_${REMOTE_ID}_TIER${N}
|
||
#
|
||
# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY
|
||
# Minutes after FALLBACK entry before activating each tier.
|
||
# References the remote host's ID: HOST1_TIER2_DELAY for HOST1 outage.
|
||
# (defaults: 240 / 720 / 1440)
|
||
#
|
||
# HOST*_TIER1_WRITEBACK_DELAY
|
||
# Minimum outage minutes before Tier 1 writeback runs. Skip for short
|
||
# outages where primary state is cleaner. (default: 60)
|
||
#
|
||
# FALLBACK_HOST*_WRITEBACK_TIER1–4
|
||
# Paths rsynced back to remote on handback, per tier.
|
||
# Tier 4 auto-uses HOST*_DAILY_SYNC_SHARES — no separate list needed.
|
||
# Variable pattern: FALLBACK_${REMOTE_ID}_WRITEBACK_TIER${N}
|
||
#
|
||
# master.conf
|
||
#
|
||
# FALLBACK_ENABLED
|
||
# false = exit cleanly (e.g. remote server being rebuilt). (default: false)
|
||
#
|
||
# FALLBACK_PARTNERSHIP_SUSPEND_AFTER
|
||
# Minutes without an active partnership before fallback suspends itself.
|
||
# Grace period: fallback continues normally during this window.
|
||
# 0 = suspend immediately when partnership goes inactive. (default: 120)
|
||
#
|
||
# FALLBACK_CHECK_INTERVAL
|
||
# Seconds between connectivity checks. (default: 30)
|
||
#
|
||
# FALLBACK_HANDBACK_STRIKES
|
||
# Consecutive remote-up checks required before handback begins. (default: 3)
|
||
#
|
||
# FALLBACK_STATE_FILE
|
||
# State file path — $STATE_DIR/fallback_state.db — survives reboots.
|
||
#
|
||
# FALLBACK_RSYNC_ENABLED
|
||
# Gate for writeback rsync jobs during handback. (default: true)
|
||
#
|
||
# EXTERNAL_IP
|
||
# IP pinged for internet check. (default: 8.8.8.8)
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# fallback.sh
|
||
# Normal start — continuous loop. Started by array_started.sh at array start.
|
||
#
|
||
# fallback.sh --stop
|
||
# Gracefully stop the running instance (SIGTERM → wait 10s → SIGKILL).
|
||
# Used by array_stopping.sh during array shutdown.
|
||
#
|
||
# fallback.sh --dry-run
|
||
# Preview state changes without starting or stopping containers or DDNS.
|
||
#
|
||
# fallback.sh --status
|
||
# Show current state, identity, DDNS containers, outage duration if in
|
||
# FALLBACK, tier flags, strike counter. Then exit.
|
||
#
|
||
# fallback.sh --log
|
||
# Verbose output on every decision in every cycle.
|
||
#
|
||
# To stop: use `fallback.sh --stop`. Do NOT kill -9 — state file may corrupt if mid-write.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||
LOCKFILE="${LOCK_DIR}/fallback.lock"
|
||
if [[ ! -f "$LOCKFILE" ]]; then
|
||
log "No fallback.sh lock found — not running"
|
||
exit 0
|
||
fi
|
||
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||
target_pid="${lock_content%%:*}"
|
||
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||
warn "Stale lock — fallback.sh not running (PID $target_pid gone) — clearing"
|
||
rm -f "$LOCKFILE"
|
||
exit 0
|
||
fi
|
||
warn "Stopping fallback.sh (PID $target_pid)..."
|
||
kill -TERM "$target_pid" 2>/dev/null || true
|
||
waited=0
|
||
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 10 ]]; do
|
||
sleep 1
|
||
(( waited++ ))
|
||
done
|
||
if kill -0 "$target_pid" 2>/dev/null; then
|
||
warn "SIGTERM ignored — sending SIGKILL to fallback.sh (PID $target_pid)"
|
||
kill -KILL "$target_pid" 2>/dev/null || true
|
||
sleep 2
|
||
if kill -0 "$target_pid" 2>/dev/null; then
|
||
error "Failed to kill fallback.sh (PID $target_pid)"
|
||
exit 1
|
||
fi
|
||
warn "Force-stopped: fallback.sh (PID $target_pid) ✅"
|
||
else
|
||
warn "Stopped: fallback.sh (PID $target_pid) ✅"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt)
|
||
if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
|
||
warn "FALLBACK_ENABLED=false — fallback monitoring disabled"
|
||
warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready"
|
||
exit 0
|
||
fi
|
||
|
||
acquire_lock # strict single instance — notifies on collision
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
require_partnership
|
||
resolve_remote_ip
|
||
|
||
# Validate unRAID notify script — used throughout for state change notifications
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"
|
||
|
||
# Timeout for all docker and SSH docker commands
|
||
DOCKER_TIMEOUT=15
|
||
SSH_TIMEOUT=10
|
||
CONTAINER_VERIFY_WAIT=5 # seconds after start before verifying container is up
|
||
|
||
log "$ICON_GEAR Config: check-interval=${FALLBACK_CHECK_INTERVAL}s handback-strikes=${FALLBACK_HANDBACK_STRIKES} rsync-enabled=${FALLBACK_RSYNC_ENABLED:-true}"
|
||
log "$ICON_GEAR Timeouts: docker=${DOCKER_TIMEOUT}s ssh=${SSH_TIMEOUT}s verify-wait=${CONTAINER_VERIFY_WAIT}s"
|
||
|
||
# ==============================================================================================
|
||
# ── STATE FILE HELPERS ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# State file at $FALLBACK_STATE_FILE ($STATE_DIR/fallback_state.db) — survives reboots.
|
||
# Format: key=value one per line.
|
||
# Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started
|
||
|
||
state_get() {
|
||
grep "^${1}=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2
|
||
}
|
||
|
||
state_set() {
|
||
local key="$1" value="$2"
|
||
if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then
|
||
# Escape sed replacement metacharacters: \ first, then & and |
|
||
local safe="${value//\\/\\\\}"; safe="${safe//&/\\&}"; safe="${safe//|/\\|}"
|
||
sed -i "s|^${key}=.*|${key}=${safe}|" "$FALLBACK_STATE_FILE"
|
||
else
|
||
echo "${key}=${value}" >> "$FALLBACK_STATE_FILE"
|
||
fi
|
||
}
|
||
|
||
state_init() {
|
||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||
[[ -z "$(state_get fallback_start)" ]] && state_set fallback_start "0"
|
||
[[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0"
|
||
[[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false"
|
||
[[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false"
|
||
[[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false"
|
||
[[ -z "$(state_get partnership_suspended)" ]] && state_set partnership_suspended "false"
|
||
[[ -z "$(state_get partner_lost_at)" ]] && state_set partner_lost_at "0"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PARTNERSHIP GATE HELPERS ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Returns 0 (true) if the local partnership DB reports an ACTIVE partnership.
|
||
# Reads /boot/config/partnership_<hostname>.db — no subprocess, no SSH.
|
||
check_partnership_active() {
|
||
local state_file="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||
local state
|
||
state=$(grep "^state=" "$state_file" 2>/dev/null | cut -d= -f2)
|
||
[[ "$state" == "ACTIVE" ]]
|
||
}
|
||
|
||
# Stop all tiers that were started during a fallback and clear their state flags.
|
||
# Called when the partnership gate suspends an in-progress fallback.
|
||
_abort_fallback_containers() {
|
||
for tier in 4 3 2 1; do
|
||
[[ "$(state_get "tier${tier}_started")" != "true" ]] && continue
|
||
warn "Aborting Tier $tier containers — partnership suspended"
|
||
local containers
|
||
read -r -a containers <<< "$(get_tier_containers "$tier")"
|
||
for container in "${containers[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
state_set "tier${tier}_started" "false"
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER HELPERS ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# All docker commands wrapped in DOCKER_TIMEOUT.
|
||
# All SSH docker commands wrapped in SSH_TIMEOUT.
|
||
# Verification after start — fallback is critical, confirm containers came up.
|
||
|
||
# Start a container locally and verify it came up
|
||
local_start() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$status" == "true" ]]; then
|
||
log "$container already running locally"
|
||
return 0
|
||
fi
|
||
log "$ICON_START Starting $container locally..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container locally"
|
||
return 0
|
||
fi
|
||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||
sleep "$CONTAINER_VERIFY_WAIT"
|
||
local post_status
|
||
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$post_status" == "true" ]]; then
|
||
echo "$ICON_STARTED $container started and running ✅"
|
||
return 0
|
||
else
|
||
error "$container started but crashed immediately"
|
||
notify "$container failed to stay running during fallback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
else
|
||
error "Failed to start $container locally"
|
||
notify "Failed to start $container locally during fallback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# Stop a container locally
|
||
local_stop() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$status" != "true" ]]; then
|
||
log "$container already stopped locally"
|
||
return 0
|
||
fi
|
||
log "$ICON_STOP Stopping $container locally..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop $container locally"
|
||
return 0
|
||
fi
|
||
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
|
||
echo "$ICON_STOPPED $container stopped" || \
|
||
error "Failed to stop $container locally"
|
||
}
|
||
|
||
# Start a container on remote via SSH and verify it came up
|
||
remote_start() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||
$container 2>/dev/null" 2>/dev/null)
|
||
if [[ "$status" == "true" ]]; then
|
||
log "$container already running on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then
|
||
sleep "$CONTAINER_VERIFY_WAIT"
|
||
local post_status
|
||
post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||
$container 2>/dev/null" 2>/dev/null)
|
||
if [[ "$post_status" == "true" ]]; then
|
||
echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅"
|
||
return 0
|
||
else
|
||
error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
|
||
notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
else
|
||
error "Failed to start $container on $REMOTE_SERVER_NAME"
|
||
notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# Stop a container on remote via SSH
|
||
remote_stop() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||
$container 2>/dev/null" 2>/dev/null)
|
||
if [[ "$status" != "true" ]]; then
|
||
log "$container already stopped on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
|
||
echo "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
|
||
error "Failed to stop $container on $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DDNS HELPERS ──────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# DDNS managed exclusively by this script — never by network state alone.
|
||
# Stopping DDNS on internet loss prevents split-brain DNS updates.
|
||
|
||
local_ddns_stop() {
|
||
warn "$ICON_NET Stopping local DDNS — internet lost"
|
||
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
||
local_stop "$container"
|
||
done
|
||
}
|
||
|
||
local_ddns_start() {
|
||
warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up"
|
||
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
||
local_start "$container"
|
||
done
|
||
}
|
||
|
||
remote_ddns_stop() {
|
||
warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync"
|
||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
remote_stop "$container"
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── TIERED FALLBACK HELPERS ───────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# All routing uses MY_ID — not hostname string comparison.
|
||
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
|
||
# "This server covers the other" — MY_ID selects which tier arrays to use.
|
||
|
||
get_tier_containers() {
|
||
local tier="$1"
|
||
local var_name="FALLBACK_${REMOTE_ID}_TIER${tier}"
|
||
eval "echo \"\${${var_name}[@]:-}\""
|
||
}
|
||
|
||
get_tier2_delay() {
|
||
local var_name="${REMOTE_ID}_TIER2_DELAY"
|
||
echo "${!var_name:-240}"
|
||
}
|
||
|
||
get_tier3_delay() {
|
||
local var_name="${REMOTE_ID}_TIER3_DELAY"
|
||
echo "${!var_name:-720}"
|
||
}
|
||
|
||
get_tier4_delay() {
|
||
local var_name="${REMOTE_ID}_TIER4_DELAY"
|
||
echo "${!var_name:-1440}"
|
||
}
|
||
|
||
get_tier1_writeback_delay() {
|
||
local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY"
|
||
echo "${!var_name:-60}"
|
||
}
|
||
|
||
get_writeback_jobs_for_tier() {
|
||
local tier="$1"
|
||
local remote_id="${REMOTE_ID}"
|
||
if [[ "$tier" -eq 4 ]]; then
|
||
# Tier 4 — daily sync shares of the remote + any extra writeback paths
|
||
local shares_var="${remote_id}_DAILY_SYNC_SHARES"
|
||
local extra_var="FALLBACK_${remote_id}_WRITEBACK_TIER4"
|
||
eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\""
|
||
else
|
||
local var_name="FALLBACK_${remote_id}_WRITEBACK_TIER${tier}"
|
||
eval "echo \"\${${var_name}[@]:-}\""
|
||
fi
|
||
}
|
||
|
||
# Select DDNS arrays based on MY_ID
|
||
set_ddns_arrays() {
|
||
local local_ddns_var="${MY_ID}_DDNS_CONTAINERS"
|
||
local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS"
|
||
eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")"
|
||
eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")"
|
||
}
|
||
|
||
# Select internet-loss stop list based on MY_ID
|
||
get_stop_on_no_net() {
|
||
local var_name="FALLBACK_${MY_ID}_STOP_ON_NO_NET"
|
||
eval "echo \"\${${var_name}[@]:-}\""
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
state_init
|
||
set_ddns_arrays
|
||
CURRENT_STATE=$(state_get state)
|
||
FALLBACK_START_TS=$(state_get fallback_start)
|
||
TIER2=$(state_get tier2_started)
|
||
TIER3=$(state_get tier3_started)
|
||
TIER4=$(state_get tier4_started)
|
||
STRIKES=$(state_get handback_strikes)
|
||
|
||
local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
|
||
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY FALLBACK STATUS ━━━━━"
|
||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||
echo "$ICON_GEAR OS ver: $local_ver"
|
||
echo "$ICON_FALLBACK State: $CURRENT_STATE"
|
||
echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]:-none}"
|
||
echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}"
|
||
echo "$ICON_TIME Interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||
echo "$ICON_FALLBACK Strikes: $STRIKES / $FALLBACK_HANDBACK_STRIKES"
|
||
|
||
if [[ "$CURRENT_STATE" == "FALLBACK" && "$FALLBACK_START_TS" -gt 0 ]]; then
|
||
ELAPSED=$(( ($(date +%s) - FALLBACK_START_TS) / 60 ))
|
||
echo "$ICON_TIME Outage: ${ELAPSED}min"
|
||
echo "$ICON_FALLBACK Tier 2: $TIER2 (delay: $(get_tier2_delay)min)"
|
||
echo "$ICON_FALLBACK Tier 3: $TIER3 (delay: $(get_tier3_delay)min)"
|
||
echo "$ICON_FALLBACK Tier 4: $TIER4 (delay: $(get_tier4_delay)min)"
|
||
fi
|
||
|
||
echo "$ICON_GEAR Enabled: $FALLBACK_ENABLED"
|
||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||
|
||
SUSPENDED=$(state_get partnership_suspended)
|
||
PLOST=$(state_get partner_lost_at)
|
||
if [[ "$SUSPENDED" == "true" ]]; then
|
||
echo "$ICON_GEAR Partnership: SUSPENDED (no active partnership)"
|
||
elif [[ -n "$PLOST" && "$PLOST" != "0" ]]; then
|
||
PLOST_MIN=$(( ($(date +%s) - PLOST) / 60 ))
|
||
echo "$ICON_GEAR Partnership: GRACE (${PLOST_MIN}min / ${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min until suspend)"
|
||
else
|
||
echo "$ICON_GEAR Partnership: ACTIVE"
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Called when remote returns after FALLBACK state.
|
||
# CRITICAL sequencing — do not reorder without understanding the consequences.
|
||
# Each step must succeed before proceeding — aborts and retries next cycle on failure.
|
||
|
||
run_handback() {
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "$REMOTE_SERVER_NAME has returned — beginning staged handback"
|
||
|
||
# ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||
|
||
if ! check_os_version_parity; then
|
||
warn "Version parity check failed — aborting handback, will retry next cycle"
|
||
state_set handback_strikes 0
|
||
return 1
|
||
fi
|
||
|
||
if ! check_remote_array; then
|
||
warn "Remote array not ready — aborting handback, will retry next cycle"
|
||
state_set handback_strikes 0
|
||
return 1
|
||
fi
|
||
|
||
if ! check_remote_docker_daemon; then
|
||
warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
|
||
state_set handback_strikes 0
|
||
return 1
|
||
fi
|
||
|
||
echo " Pre-flight checks passed ✅"
|
||
|
||
local outage_minutes
|
||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||
local tier1_wb_delay t2_delay t3_delay t4_delay
|
||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||
t2_delay=$(get_tier2_delay)
|
||
t3_delay=$(get_tier3_delay)
|
||
t4_delay=$(get_tier4_delay)
|
||
|
||
warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1"
|
||
|
||
# ── Staged handback helper ────────────────────────────────────────────────────────────────
|
||
# Each tier: stop local containers → rsync writeback → start on remote.
|
||
# Skips tiers that were never activated. Emby (Tier 1) stays on fallback
|
||
# serving users until all higher tiers complete.
|
||
handback_tier() {
|
||
local tier="$1" threshold="$2" label="$3"
|
||
|
||
[[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━"
|
||
|
||
local containers
|
||
read -r -a containers <<< "$(get_tier_containers "$tier")"
|
||
|
||
echo " Stopping local Tier $tier containers..."
|
||
for container in "${containers[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
|
||
if ! check_rsync_enabled "FALLBACK"; then
|
||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback"
|
||
else
|
||
local jobs
|
||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||
if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then
|
||
echo " Rsync writeback — outage ${outage_minutes}min >= ${threshold}min"
|
||
for job in "${jobs[@]}"; do
|
||
[[ -z "$job" ]] && continue
|
||
log "Syncing: $job"
|
||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||
|| warn "DRY RUN — would rsync: $job"
|
||
done
|
||
else
|
||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||
fi
|
||
fi
|
||
|
||
echo " Starting Tier $tier on $REMOTE_SERVER_NAME..."
|
||
for container in "${containers[@]}"; do
|
||
[[ -n "$container" ]] && remote_start "$container"
|
||
done
|
||
|
||
state_set "tier${tier}_started" "false"
|
||
warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ──────────────────────────────────
|
||
handback_tier 4 "$t4_delay" "media shares + edge cases"
|
||
handback_tier 3 "$t3_delay" "secondary services"
|
||
handback_tier 2 "$t2_delay" "productivity services"
|
||
|
||
# ── Step 5: DDNS handoff — immediately before Tier 1 goes down ──────────────────────────
|
||
# Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks.
|
||
# Prevents split-brain DNS during Tier 1 rsync window.
|
||
echo ""
|
||
echo "━━━ $ICON_NET DDNS Handoff ━━━"
|
||
remote_ddns_stop
|
||
sleep 5
|
||
|
||
# ── Step 6: Tier 1 handback — vital services + Emby ─────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━"
|
||
|
||
read -r -a t1 <<< "$(get_tier_containers 1)"
|
||
|
||
echo " Stopping local Tier 1 containers..."
|
||
for container in "${t1[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local is_ddns=false
|
||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||
done
|
||
[[ "$is_ddns" == false ]] && local_stop "$container"
|
||
done
|
||
|
||
if ! check_rsync_enabled "FALLBACK"; then
|
||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback"
|
||
else
|
||
local t1_jobs
|
||
read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)"
|
||
if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then
|
||
echo " Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min"
|
||
for job in "${t1_jobs[@]}"; do
|
||
[[ -z "$job" ]] && continue
|
||
log "Syncing: $job"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||
else
|
||
warn "DRY RUN — would rsync: $job"
|
||
fi
|
||
done
|
||
else
|
||
log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min"
|
||
fi
|
||
fi
|
||
|
||
echo " Starting Tier 1 on $REMOTE_SERVER_NAME..."
|
||
sleep 10 # give databases time to initialise before apps
|
||
for container in "${t1[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local is_ddns=false
|
||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||
done
|
||
[[ "$is_ddns" == false ]] && remote_start "$container"
|
||
done
|
||
|
||
# ── Step 7: Play State Sync — DNS held until sync succeeds or retries exhausted ──────────
|
||
# Retried so users land on current watch state — after a 3hr outage one successful
|
||
# run catches everything regardless of how many cron cycles were missed.
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Play State Sync ━━━"
|
||
local _sync_ok=false
|
||
local _retry_max="${PLAY_SYNC_HANDBACK_RETRIES:-5}"
|
||
local _retry_delay="${PLAY_SYNC_HANDBACK_RETRY_DELAY:-60}"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
for _attempt in $(seq 1 "$_retry_max"); do
|
||
if bash "$SCRIPT_DIR/../Media/play_state_sync.sh" --wait; then
|
||
_sync_ok=true
|
||
log "Play state synced ✅ (attempt ${_attempt}/${_retry_max})"
|
||
break
|
||
fi
|
||
if [[ "$_attempt" -lt "$_retry_max" ]]; then
|
||
warn "Play state sync failed (attempt ${_attempt}/${_retry_max}) — retrying in ${_retry_delay}s"
|
||
sleep "$_retry_delay"
|
||
fi
|
||
done
|
||
[[ "$_sync_ok" == false ]] && \
|
||
warn "Play state sync failed after ${_retry_max} attempts — proceeding to DNS cutover"
|
||
else
|
||
warn "DRY RUN — would retry play_state_sync --wait up to ${_retry_max} times before DNS cutover"
|
||
fi
|
||
|
||
# ── Step 8: DNS Cutover — final step ────────────────────────────────────────────────────
|
||
# DNS cuts over ONLY after Tier 1 containers confirmed up
|
||
echo ""
|
||
echo "━━━ $ICON_NET DNS Cutover ━━━"
|
||
sleep 5
|
||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
remote_start "$container"
|
||
done
|
||
warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
|
||
|
||
# ── Step 9: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||
echo ""
|
||
state_set state "NORMAL"
|
||
state_set fallback_start "0"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
# Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback
|
||
local_ddns_start
|
||
|
||
warn "$ICON_DONE Handback complete — returned to NORMAL"
|
||
notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
|
||
"Fallback" "normal"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Main State Machine ━━━
|
||
# ==============================================================================================
|
||
state_init
|
||
set_ddns_arrays
|
||
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " $ICON_FALLBACK FALLBACK — $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||
echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
FALLBACK_RUNNING=true
|
||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
||
SIGTERM SIGINT
|
||
|
||
while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||
NOW=$(date +%s)
|
||
CURRENT_STATE=$(state_get state)
|
||
|
||
# ── Connectivity checks ──────────────────────────────────────────────────────────────────
|
||
REMOTE_UP=false
|
||
INTERNET_UP=false
|
||
|
||
ping_remote && REMOTE_UP=true
|
||
ping_internet && INTERNET_UP=true
|
||
|
||
log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"
|
||
|
||
# ── Partnership gate ──────────────────────────────────────────────────────────────────────
|
||
# ── Partnership Gate ──────────────────────────────────────────────────────────────────────
|
||
# A fallback without an active partner is meaningless — who are we covering for?
|
||
# The gate suspends the loop after a grace period and aborts any live fallback
|
||
# containers. Clears automatically when partnership becomes active again.
|
||
if check_partnership_active; then
|
||
# Partnership healthy — clear any accumulated grace state
|
||
if [[ "$(state_get partnership_suspended)" == "true" ]]; then
|
||
warn "Active partnership restored — re-enabling fallback monitoring"
|
||
state_set partnership_suspended "false"
|
||
state_set partner_lost_at "0"
|
||
notify "Fallback re-enabled on $(hostname) — active partnership restored" \
|
||
"Fallback" "normal"
|
||
elif [[ "$(state_get partner_lost_at)" != "0" ]]; then
|
||
log "Partnership recovered within grace window — resetting timer"
|
||
state_set partner_lost_at "0"
|
||
fi
|
||
else
|
||
PARTNER_LOST_AT=$(state_get partner_lost_at)
|
||
if [[ -z "$PARTNER_LOST_AT" || "$PARTNER_LOST_AT" == "0" ]]; then
|
||
warn "No active partnership detected — grace timer started (${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min before suspend)"
|
||
state_set partner_lost_at "$NOW"
|
||
PARTNER_LOST_AT=$NOW
|
||
fi
|
||
|
||
PARTNER_LOST_MIN=$(( (NOW - PARTNER_LOST_AT) / 60 ))
|
||
SUSPEND_AFTER=${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}
|
||
|
||
if [[ "$(state_get partnership_suspended)" == "true" || "$PARTNER_LOST_MIN" -ge "$SUSPEND_AFTER" ]]; then
|
||
# Suspend — abort any live fallback containers, then skip this cycle entirely
|
||
if [[ "$(state_get partnership_suspended)" != "true" ]]; then
|
||
warn "No active partnership for ${PARTNER_LOST_MIN}min — suspending fallback"
|
||
state_set partnership_suspended "true"
|
||
notify "Fallback suspended on $(hostname) — no active partnership for ${PARTNER_LOST_MIN}min" \
|
||
"Fallback" "warning"
|
||
fi
|
||
if [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||
warn "Aborting in-progress fallback — partnership suspended"
|
||
_abort_fallback_containers
|
||
state_set state "NORMAL"
|
||
state_set fallback_start "0"
|
||
state_set handback_strikes "0"
|
||
notify "Fallback aborted on $(hostname) — no active partnership, containers stopped" \
|
||
"Fallback" "warning"
|
||
fi
|
||
log "Partnership suspended — skipping fallback cycle"
|
||
sleep "$FALLBACK_CHECK_INTERVAL" & wait $!
|
||
continue
|
||
else
|
||
log "No active partnership — ${PARTNER_LOST_MIN}min elapsed / ${SUSPEND_AFTER}min until suspend"
|
||
fi
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# NORMAL STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
||
|
||
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
||
log "$ICON_SUCCESS NORMAL — all systems up"
|
||
|
||
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
||
# Remote is down — enter FALLBACK immediately
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"
|
||
|
||
state_set state "FALLBACK"
|
||
state_set fallback_start "$NOW"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||
for container in "${tier1[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
|
||
notify "FALLBACK started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \
|
||
"Fallback" "warning"
|
||
|
||
elif [[ "$INTERNET_UP" == false ]]; then
|
||
# Lost internet — enter NO_INTERNET
|
||
echo ""
|
||
echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "Internet connectivity lost — stopping local DDNS"
|
||
|
||
state_set state "NO_INTERNET"
|
||
local_ddns_stop
|
||
|
||
# Stop containers configured to stop on internet loss
|
||
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
|
||
for container in "${stop_on_no_net[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
|
||
notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# FALLBACK STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||
|
||
FALLBACK_START_TS=$(state_get fallback_start)
|
||
ELAPSED_MIN=$(( (NOW - FALLBACK_START_TS) / 60 ))
|
||
|
||
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
||
# Remote returned — increment handback strikes
|
||
STRIKES=$(state_get handback_strikes)
|
||
STRIKES=$(( STRIKES + 1 ))
|
||
state_set handback_strikes "$STRIKES"
|
||
|
||
warn "$ICON_FALLBACK $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FALLBACK_HANDBACK_STRIKES"
|
||
|
||
if [[ "$STRIKES" -ge "$FALLBACK_HANDBACK_STRIKES" ]]; then
|
||
run_handback
|
||
fi
|
||
|
||
elif [[ "$INTERNET_UP" == false ]]; then
|
||
# Lost internet during fallback — enter DARK
|
||
echo ""
|
||
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "Lost internet during fallback — entering DARK state"
|
||
state_set state "DARK"
|
||
local_ddns_stop
|
||
notify "DARK state on $(hostname) — lost internet during fallback" \
|
||
"Fallback" "warning"
|
||
|
||
else
|
||
# Still in FALLBACK state — check tier escalation
|
||
state_set handback_strikes "0"
|
||
|
||
TIER2_DELAY=$(get_tier2_delay)
|
||
if [[ "$(state_get tier2_started)" == "false" && \
|
||
"$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
|
||
read -r -a tier2 <<< "$(get_tier_containers 2)"
|
||
for container in "${tier2[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
state_set tier2_started "true"
|
||
notify "Fallback Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
TIER3_DELAY=$(get_tier3_delay)
|
||
if [[ "$(state_get tier3_started)" == "false" && \
|
||
"$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
|
||
read -r -a tier3 <<< "$(get_tier_containers 3)"
|
||
for container in "${tier3[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
state_set tier3_started "true"
|
||
notify "Fallback Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
TIER4_DELAY=$(get_tier4_delay)
|
||
if [[ "$(state_get tier4_started)" == "false" && \
|
||
"$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
|
||
read -r -a tier4 <<< "$(get_tier_containers 4)"
|
||
for container in "${tier4[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
state_set tier4_started "true"
|
||
notify "Fallback Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
log "$ICON_FALLBACK FALLBACK — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# NO_INTERNET STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then
|
||
|
||
if [[ "$INTERNET_UP" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
if [[ "$REMOTE_UP" == true ]]; then
|
||
warn "Remote is up — returning to NORMAL"
|
||
state_set state "NORMAL"
|
||
local_ddns_start
|
||
notify "Internet recovered on $(hostname) — returning to NORMAL" \
|
||
"Fallback" "normal"
|
||
else
|
||
warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
|
||
state_set state "FALLBACK"
|
||
state_set fallback_start "$NOW"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||
for container in "${tier1[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FALLBACK" \
|
||
"Fallback" "warning"
|
||
fi
|
||
else
|
||
log "$ICON_NET NO_INTERNET — waiting for connectivity"
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# DARK STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
elif [[ "$CURRENT_STATE" == "DARK" ]]; then
|
||
|
||
if [[ "$INTERNET_UP" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
if [[ "$REMOTE_UP" == true ]]; then
|
||
warn "Remote up, internet up — transitioning through FALLBACK for handback"
|
||
# Was in FALLBACK state before DARK — route through FALLBACK state for handback
|
||
state_set state "FALLBACK"
|
||
state_set handback_strikes "0"
|
||
else
|
||
warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
|
||
state_set state "FALLBACK"
|
||
state_set fallback_start "$NOW"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||
for container in "${tier1[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
fi
|
||
notify "Emerging from DARK state on $(hostname)" "Fallback" "warning"
|
||
else
|
||
log "$ICON_FALLBACK DARK — no internet, no remote — waiting"
|
||
fi
|
||
fi
|
||
|
||
# ── Sleep until next check ───────────────────────────────────────────────────────────────
|
||
log "Next check in ${FALLBACK_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
|
||
sleep "$FALLBACK_CHECK_INTERVAL" &
|
||
wait $!
|
||
|
||
done |