1212 lines
56 KiB
Bash
Executable File
1212 lines
56 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= Fallback ===================================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Mutual container fallback between two unRAID servers. Runs continuously as a
|
||
# background process — started at array start by array_started.sh. Each server
|
||
# runs this script independently with no coordination between servers.
|
||
#
|
||
# All decisions are based solely on two pings per cycle: remote Tailscale IP
|
||
# reachable + internet reachable. No SSH signaling, no shared state, no election
|
||
# algorithm. Each server acts entirely from its own network perspective.
|
||
#
|
||
# Never requires human intervention during normal fallback and handback.
|
||
# Stop only via `fallback.sh --stop` — do NOT kill directly (state file may corrupt).
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Each cycle (every FALLBACK_CHECK_INTERVAL seconds):
|
||
# 1. Ping remote Tailscale IP — is the other server reachable?
|
||
# 2. Ping internet (EXTERNAL_IP) — do I have internet?
|
||
# 3. Determine state from the two results
|
||
# 4. Take the action for that state
|
||
#
|
||
# States:
|
||
# NORMAL — remote up, internet up — own containers, own DDNS on, silent
|
||
# FALLBACK — remote down, internet up — start tier containers, DDNS over
|
||
# NO_INTERNET — internet down — stop own DDNS immediately, wait
|
||
# DARK — remote down AND internet down — same actions as NO_INTERNET
|
||
#
|
||
# FALLBACK escalation (tiered by outage duration):
|
||
# Tier 1 — immediate — vital services + Live TV (cannot wait)
|
||
# Tier 2 — after HOST*_TIER2_DELAY (default 4hr) — shared productivity
|
||
# Tier 3 — after HOST*_TIER3_DELAY (default 12hr) — secondary services
|
||
# Tier 4 — after HOST*_TIER4_DELAY (default 24hr) — arrs + downloaders
|
||
#
|
||
# Handback (when remote returns after FALLBACK):
|
||
# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
|
||
# 2. Pre-flight checks — version parity, remote array, remote Docker daemon
|
||
# 3. Staged reverse: Tier 4→3→2 — Emby stays on covering server throughout
|
||
# Each tier: stop local → rsync writeback → start on remote
|
||
# 4. DDNS handoff — stop remote DDNS before Tier 1 goes down
|
||
# 5. Tier 1 handback — stop local vital services, rsync writeback, start remote
|
||
# 6. Start remote DDNS — DNS cuts back ONLY after containers confirmed running
|
||
# 7. Return to NORMAL
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Pure Ping-Only Detection
|
||
# No SSH signaling, no election algorithm, no shared state between servers.
|
||
# Each server makes all decisions from its own vantage point. A server that
|
||
# cannot reach the remote and has internet acts — no coordination needed.
|
||
#
|
||
# DDNS Absolute Control
|
||
# This script is the sole authority over DDNS containers. Network state
|
||
# returning is not permission to start DDNS — only the completion of the
|
||
# full handback sequence grants that. Eliminates the split-brain DNS window
|
||
# where two servers update the same domain with different IPs simultaneously.
|
||
#
|
||
# Staged Reverse-Tier Handback
|
||
# Emby stays on the covering server serving users while Tiers 4→3→2 hand
|
||
# back. The user impact window (Emby down) is only the Tier 1 rsync
|
||
# duration — typically minutes. Users are served continuously until the
|
||
# final step.
|
||
#
|
||
# Strike Confirmation
|
||
# Handback requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks.
|
||
# Brief network recovery during an ongoing outage would otherwise trigger a
|
||
# failed handback: containers stop on the covering server, remote goes down
|
||
# again mid-rsync. Strikes prevent this.
|
||
#
|
||
# REMOTE_ID-Based Coverage Config
|
||
# Tier arrays, writeback paths, and delays are all defined in the covered
|
||
# host's own conf and read via REMOTE_ID — not MY_ID. Each host owns its
|
||
# complete recovery profile. The same script handles both directions.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Enforcement
|
||
# Docker and SSH operations require root.
|
||
#
|
||
# FALLBACK_ENABLED Gate
|
||
# Exits cleanly when disabled — safe to run on servers being rebuilt without
|
||
# triggering spurious fallback actions. Fail-closed: anything that is not exactly
|
||
# "true" counts as disabled, so a malformed toggle cannot grant this script DDNS
|
||
# authority and cross-server container control by accident.
|
||
#
|
||
# Docker Presence Check
|
||
# Verifies the docker binary exists before the state machine starts.
|
||
#
|
||
# Remote IP Resolution
|
||
# resolve_remote_ip() must resolve the partner before any SSH operation, so a
|
||
# remote command can never be issued against an unresolved or stale address.
|
||
#
|
||
# Asymmetric Failover / Handback
|
||
# Entering FALLBACK is immediate — a down partner means users are already affected.
|
||
# Returning requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks. The
|
||
# asymmetry is deliberate: protecting fast costs a few minutes of redundant
|
||
# coverage, handing back fast on a flapping partner costs a second outage.
|
||
#
|
||
# DDNS Excluded From Tier Loops
|
||
# The Tier 1 stop and start loops explicitly skip any container that is also in
|
||
# REMOTE_DDNS_CONTAINERS. DDNS is sequenced by the handoff and cutover steps alone,
|
||
# so ordinary tier processing can never move DNS at the wrong moment.
|
||
#
|
||
# Writeback Delay Gate
|
||
# Tier writeback rsync only runs once the outage has exceeded that tier's writeback
|
||
# delay. A brief blip does not trigger a full data writeback, which would cost more
|
||
# than the outage it is compensating for.
|
||
#
|
||
# FALLBACK_RSYNC_ENABLED Gate
|
||
# Writeback is skipped entirely when disabled, and the skip is announced rather than
|
||
# silent — containers still hand back, but nobody is left assuming data moved.
|
||
#
|
||
# Play State Sync Before Cutover
|
||
# Handback retries play_state_sync up to PLAY_SYNC_HANDBACK_RETRIES times before DNS
|
||
# cuts over, so users land on current watch state. Exhausting retries warns and
|
||
# proceeds — stale resume positions are not worth holding DNS on a downed service.
|
||
#
|
||
# Partnership Suspend Abort
|
||
# If partnership goes inactive mid-fallback, _abort_fallback_containers() stops the
|
||
# fallback containers and returns to NORMAL rather than leaving this host serving a
|
||
# partner it is no longer paired with.
|
||
#
|
||
# Container Verify Wait
|
||
# CONTAINER_VERIFY_WAIT seconds elapse after each start before the running check, so
|
||
# a container that starts and immediately crashes is caught rather than counted up.
|
||
#
|
||
# Version Parity Check
|
||
# Refuses handback if remote unRAID version doesn't match. A mismatch may
|
||
# mean the remote is not fully ready or on an incompatible version.
|
||
#
|
||
# Remote Docker Daemon Check
|
||
# SSH-checks the remote Docker daemon is responding before any remote commands.
|
||
# Reachable on Tailscale does not mean Docker is ready.
|
||
#
|
||
# Container Verification
|
||
# After every local_start(), verifies the container came up and stayed running.
|
||
# Catches crashes-on-start before declaring fallback successful.
|
||
#
|
||
# Timeout Protection
|
||
# All docker commands (DOCKER_TIMEOUT=15s) and SSH commands (SSH_TIMEOUT=10s)
|
||
# wrapped in timeouts. A hung daemon does not block the state machine.
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock() prevents two instances running simultaneously. Both would
|
||
# modify containers and DDNS independently — conflict is certain.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() resolves MY_ID / REMOTE_ID from master.conf at startup.
|
||
# Exits if the host cannot be identified — prevents running on an unknown machine.
|
||
#
|
||
# Partnership Gate
|
||
# Fallback is only permitted when the local partnership DB reports an active
|
||
# partnership. If partnership goes inactive, a grace timer starts. After
|
||
# FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes the loop suspends: no new FALLBACK
|
||
# is entered, and any live fallback containers are stopped cleanly. Resumes
|
||
# automatically when partnership becomes active again.
|
||
#
|
||
# Silent by Default
|
||
# State transitions: warn() — always visible. Healthy routine cycles: log()
|
||
# — suppressed unless --log. Produces no output on clean cycles.
|
||
#
|
||
# ==============================================================================================
|
||
# STATE FILES
|
||
# ==============================================================================================
|
||
#
|
||
# FALLBACK_STATE_FILE — $STATE_DIR/fallback_state.db (survives reboots)
|
||
# Keys: state, fallback_start, handback_strikes, tier2_started,
|
||
# tier3_started, tier4_started, partnership_suspended, partner_lost_at.
|
||
# Survives reboots — $STATE_DIR is on the boot device (internal) or appdata
|
||
# (flash). Either way the array is up before this script runs, so the file
|
||
# is always accessible. If the server was in FALLBACK when it rebooted,
|
||
# it resumes FALLBACK on restart.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_DDNS_CONTAINERS
|
||
# DDNS containers this host manages — stopped on internet loss, started
|
||
# as the last step of handback
|
||
#
|
||
# FALLBACK_HOST*_STOP_ON_NO_NET
|
||
# Containers stopped when this host loses internet
|
||
#
|
||
# FALLBACK_HOST*_TIER1–4
|
||
# Containers started on this host when the partner is down — defined in the
|
||
# partner's own conf (what the partner wants covered). Read via REMOTE_ID.
|
||
# Variable pattern: FALLBACK_${REMOTE_ID}_TIER${N}
|
||
#
|
||
# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY
|
||
# Minutes after FALLBACK entry before activating each tier.
|
||
# References the remote host's ID: HOST1_TIER2_DELAY for HOST1 outage.
|
||
# (defaults: 240 / 720 / 1440)
|
||
#
|
||
# HOST*_TIER1_WRITEBACK_DELAY
|
||
# Minimum outage minutes before Tier 1 writeback runs. Skip for short
|
||
# outages where primary state is cleaner. (default: 60)
|
||
#
|
||
# FALLBACK_HOST*_WRITEBACK_TIER1–4
|
||
# Paths rsynced back to remote on handback, per tier.
|
||
# Tier 4 auto-uses HOST*_DAILY_SYNC_SHARES — no separate list needed.
|
||
# Variable pattern: FALLBACK_${REMOTE_ID}_WRITEBACK_TIER${N}
|
||
#
|
||
# master.conf
|
||
#
|
||
# FALLBACK_ENABLED
|
||
# false = exit cleanly (e.g. remote server being rebuilt). (default: false)
|
||
#
|
||
# FALLBACK_PARTNERSHIP_SUSPEND_AFTER
|
||
# Minutes without an active partnership before fallback suspends itself.
|
||
# Grace period: fallback continues normally during this window.
|
||
# 0 = suspend immediately when partnership goes inactive. (default: 120)
|
||
#
|
||
# FALLBACK_CHECK_INTERVAL
|
||
# Seconds between connectivity checks. (default: 30)
|
||
#
|
||
# FALLBACK_HANDBACK_STRIKES
|
||
# Consecutive remote-up checks required before handback begins. (default: 3)
|
||
#
|
||
# FALLBACK_STATE_FILE
|
||
# State file path — $STATE_DIR/fallback_state.db — survives reboots.
|
||
#
|
||
# FALLBACK_RSYNC_ENABLED
|
||
# Gate for writeback rsync jobs during handback. (default: true)
|
||
#
|
||
# EXTERNAL_IP
|
||
# IP pinged for internet check. (default: 8.8.8.8)
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# fallback.sh
|
||
# Normal start — continuous loop. Started by array_started.sh at array start.
|
||
#
|
||
# fallback.sh --stop
|
||
# Gracefully stop the running instance (SIGTERM → wait 10s → SIGKILL).
|
||
# Used by array_stopping.sh during array shutdown.
|
||
#
|
||
# fallback.sh --dry-run
|
||
# Preview state changes without starting or stopping containers or DDNS.
|
||
#
|
||
# fallback.sh --status
|
||
# Show current state, identity, DDNS containers, outage duration if in
|
||
# FALLBACK, tier flags, strike counter. Then exit.
|
||
#
|
||
# fallback.sh --log
|
||
# Verbose output on every decision in every cycle.
|
||
#
|
||
# To stop: use `fallback.sh --stop`. Do NOT kill -9 — state file may corrupt if mid-write.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||
LOCKFILE="${LOCK_DIR}/fallback.lock"
|
||
if [[ ! -f "$LOCKFILE" ]]; then
|
||
log "No fallback.sh lock found — not running"
|
||
exit 0
|
||
fi
|
||
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||
target_pid="${lock_content%%:*}"
|
||
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||
warn "Stale lock — fallback.sh not running (PID $target_pid gone) — clearing"
|
||
rm -f "$LOCKFILE"
|
||
exit 0
|
||
fi
|
||
warn "Stopping fallback.sh (PID $target_pid)..."
|
||
kill -TERM "$target_pid" 2>/dev/null || true
|
||
waited=0
|
||
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 10 ]]; do
|
||
sleep 1
|
||
(( waited++ ))
|
||
done
|
||
if kill -0 "$target_pid" 2>/dev/null; then
|
||
warn "SIGTERM ignored — sending SIGKILL to fallback.sh (PID $target_pid)"
|
||
kill -KILL "$target_pid" 2>/dev/null || true
|
||
sleep 2
|
||
if kill -0 "$target_pid" 2>/dev/null; then
|
||
error "Failed to kill fallback.sh (PID $target_pid)"
|
||
exit 1
|
||
fi
|
||
warn "Force-stopped: fallback.sh (PID $target_pid) ✅"
|
||
else
|
||
warn "Stopped: fallback.sh (PID $target_pid) ✅"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# FALLBACK_ENABLED gate — exits cleanly when disabled.
|
||
# Fail-closed: anything that isn't exactly "true" disables fallback. Matching only the
|
||
# literal "false" would let a typo ("no", "0", "FALSE") hand this script DDNS authority
|
||
# and cross-server container control on a toggle nobody meant to set.
|
||
if [[ "${FALLBACK_ENABLED:-false}" != "true" ]]; then
|
||
warn "FALLBACK_ENABLED=false — fallback monitoring disabled"
|
||
warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready"
|
||
exit 0
|
||
fi
|
||
|
||
acquire_lock # strict single instance — notifies on collision
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
require_partnership
|
||
resolve_remote_ip
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"
|
||
|
||
# Timeout for all docker and SSH docker commands
|
||
DOCKER_TIMEOUT=15
|
||
SSH_TIMEOUT=10
|
||
CONTAINER_VERIFY_WAIT=5 # seconds after start before verifying container is up
|
||
|
||
log "$ICON_GEAR Config: check-interval=${FALLBACK_CHECK_INTERVAL}s handback-strikes=${FALLBACK_HANDBACK_STRIKES} rsync-enabled=${FALLBACK_RSYNC_ENABLED:-true}"
|
||
log "$ICON_GEAR Timeouts: docker=${DOCKER_TIMEOUT}s ssh=${SSH_TIMEOUT}s verify-wait=${CONTAINER_VERIFY_WAIT}s"
|
||
|
||
# ==============================================================================================
|
||
# ── STATE FILE HELPERS ────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# State file at $FALLBACK_STATE_FILE ($STATE_DIR/fallback_state.db) — survives reboots.
|
||
# Format: key=value one per line.
|
||
# Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started
|
||
|
||
state_get() {
|
||
grep "^${1}=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2
|
||
}
|
||
|
||
state_set() {
|
||
local key="$1" value="$2"
|
||
if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then
|
||
# Escape sed replacement metacharacters: \ first, then & and |
|
||
local safe="${value//\\/\\\\}"; safe="${safe//&/\\&}"; safe="${safe//|/\\|}"
|
||
sed -i "s|^${key}=.*|${key}=${safe}|" "$FALLBACK_STATE_FILE"
|
||
else
|
||
echo "${key}=${value}" >> "$FALLBACK_STATE_FILE"
|
||
fi
|
||
}
|
||
|
||
state_init() {
|
||
# A dry run must not leave the host believing it failed over. state_set() writes
|
||
# unconditionally, and this file survives reboots and is what the real daemon — and the
|
||
# Monitor and Fallback cards — read to decide what is happening. A --dry-run walk through
|
||
# FAILOVER would have written state=FALLBACK, the tier flags and the strike counter into it
|
||
# for real, and nothing would have put them back.
|
||
#
|
||
# Copied rather than merely redirected, so the preview still starts from the live state and
|
||
# can advance through tiers exactly as a real run would. The copy lands in the RAM cache and
|
||
# dies with the reboot.
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
local live="$FALLBACK_STATE_FILE"
|
||
FALLBACK_STATE_FILE="${VV_CACHE_ROOT:-/tmp/varaverk}/fallback_state.dryrun.$$"
|
||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||
if [[ -f "$live" ]]; then cp -f "$live" "$FALLBACK_STATE_FILE"; else : > "$FALLBACK_STATE_FILE"; fi
|
||
warn "DRY RUN — state writes redirected to $FALLBACK_STATE_FILE (live state untouched)"
|
||
fi
|
||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||
[[ -z "$(state_get fallback_start)" ]] && state_set fallback_start "0"
|
||
[[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0"
|
||
[[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false"
|
||
[[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false"
|
||
[[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false"
|
||
[[ -z "$(state_get partnership_suspended)" ]] && state_set partnership_suspended "false"
|
||
[[ -z "$(state_get partner_lost_at)" ]] && state_set partner_lost_at "0"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── PARTNERSHIP GATE HELPERS ──────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
# Returns 0 (true) if the local partnership DB reports an ACTIVE partnership.
|
||
# Reads /boot/config/partnership_<hostname>.db — no subprocess, no SSH.
|
||
check_partnership_active() {
|
||
local state_file="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||
local state
|
||
state=$(grep "^state=" "$state_file" 2>/dev/null | cut -d= -f2)
|
||
[[ "$state" == "ACTIVE" ]]
|
||
}
|
||
|
||
# Stop all tiers that were started during a fallback and clear their state flags.
|
||
# Called when the partnership gate suspends an in-progress fallback.
|
||
_abort_fallback_containers() {
|
||
for tier in 4 3 2 1; do
|
||
[[ "$(state_get "tier${tier}_started")" != "true" ]] && continue
|
||
warn "Aborting Tier $tier containers — partnership suspended"
|
||
local containers
|
||
read -r -a containers <<< "$(get_tier_containers "$tier")"
|
||
for container in "${containers[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
state_set "tier${tier}_started" "false"
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── CONTAINER HELPERS ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# All docker commands wrapped in DOCKER_TIMEOUT.
|
||
# All SSH docker commands wrapped in SSH_TIMEOUT.
|
||
# Verification after start — fallback is critical, confirm containers came up.
|
||
|
||
# Start a container locally and verify it came up
|
||
local_start() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$status" == "true" ]]; then
|
||
log "$container already running locally"
|
||
return 0
|
||
fi
|
||
log "$ICON_START Starting $container locally..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container locally"
|
||
return 0
|
||
fi
|
||
if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
|
||
sleep "$CONTAINER_VERIFY_WAIT"
|
||
local post_status
|
||
post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$post_status" == "true" ]]; then
|
||
echo "$ICON_STARTED $container started and running ✅"
|
||
return 0
|
||
else
|
||
error "$container started but crashed immediately"
|
||
notify "$container failed to stay running during fallback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
else
|
||
error "Failed to start $container locally"
|
||
notify "Failed to start $container locally during fallback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# Stop a container locally
|
||
local_stop() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||
"$container" 2>/dev/null)
|
||
if [[ "$status" != "true" ]]; then
|
||
log "$container already stopped locally"
|
||
return 0
|
||
fi
|
||
log "$ICON_STOP Stopping $container locally..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop $container locally"
|
||
return 0
|
||
fi
|
||
timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
|
||
echo "$ICON_STOPPED $container stopped" || \
|
||
error "Failed to stop $container locally"
|
||
}
|
||
|
||
# Start a container on remote via SSH and verify it came up
|
||
remote_start() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||
$container 2>/dev/null" 2>/dev/null)
|
||
if [[ "$status" == "true" ]]; then
|
||
log "$container already running on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then
|
||
sleep "$CONTAINER_VERIFY_WAIT"
|
||
local post_status
|
||
post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||
$container 2>/dev/null" 2>/dev/null)
|
||
if [[ "$post_status" == "true" ]]; then
|
||
echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅"
|
||
return 0
|
||
else
|
||
error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
|
||
notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
else
|
||
error "Failed to start $container on $REMOTE_SERVER_NAME"
|
||
notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \
|
||
"Fallback" "warning"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# Stop a container on remote via SSH
|
||
remote_stop() {
|
||
local container="$1"
|
||
[[ -z "$container" ]] && return
|
||
local status
|
||
status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
|
||
$container 2>/dev/null" 2>/dev/null)
|
||
if [[ "$status" != "true" ]]; then
|
||
log "$container already stopped on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
|
||
return 0
|
||
fi
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
root@"$REMOTE_SERVER" \
|
||
"timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
|
||
echo "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
|
||
error "Failed to stop $container on $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── DDNS HELPERS ──────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# DDNS managed exclusively by this script — never by network state alone.
|
||
# Stopping DDNS on internet loss prevents split-brain DNS updates.
|
||
|
||
local_ddns_stop() {
|
||
warn "$ICON_NET Stopping local DDNS — internet lost"
|
||
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
||
local_stop "$container"
|
||
done
|
||
}
|
||
|
||
local_ddns_start() {
|
||
warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up"
|
||
for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
|
||
local_start "$container"
|
||
done
|
||
}
|
||
|
||
remote_ddns_stop() {
|
||
warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync"
|
||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
remote_stop "$container"
|
||
done
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── TIERED FALLBACK HELPERS ───────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# All routing uses MY_ID — not hostname string comparison.
|
||
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
|
||
# "This server covers the other" — MY_ID selects which tier arrays to use.
|
||
|
||
get_tier_containers() {
|
||
local tier="$1"
|
||
local var_name="FALLBACK_${REMOTE_ID}_TIER${tier}"
|
||
eval "echo \"\${${var_name}[@]:-}\""
|
||
}
|
||
|
||
get_tier2_delay() {
|
||
local var_name="${REMOTE_ID}_TIER2_DELAY"
|
||
echo "${!var_name:-240}"
|
||
}
|
||
|
||
get_tier3_delay() {
|
||
local var_name="${REMOTE_ID}_TIER3_DELAY"
|
||
echo "${!var_name:-720}"
|
||
}
|
||
|
||
get_tier4_delay() {
|
||
local var_name="${REMOTE_ID}_TIER4_DELAY"
|
||
echo "${!var_name:-1440}"
|
||
}
|
||
|
||
get_tier1_writeback_delay() {
|
||
local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY"
|
||
echo "${!var_name:-60}"
|
||
}
|
||
|
||
get_writeback_jobs_for_tier() {
|
||
local tier="$1"
|
||
local remote_id="${REMOTE_ID}"
|
||
if [[ "$tier" -eq 4 ]]; then
|
||
# Tier 4 — daily sync shares of the remote + any extra writeback paths
|
||
local shares_var="${remote_id}_DAILY_SYNC_SHARES"
|
||
local extra_var="FALLBACK_${remote_id}_WRITEBACK_TIER4"
|
||
eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\""
|
||
else
|
||
local var_name="FALLBACK_${remote_id}_WRITEBACK_TIER${tier}"
|
||
eval "echo \"\${${var_name}[@]:-}\""
|
||
fi
|
||
}
|
||
|
||
# Select DDNS arrays based on MY_ID
|
||
set_ddns_arrays() {
|
||
local local_ddns_var="${MY_ID}_DDNS_CONTAINERS"
|
||
local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS"
|
||
eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")"
|
||
eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")"
|
||
}
|
||
|
||
# Select internet-loss stop list based on MY_ID
|
||
get_stop_on_no_net() {
|
||
local var_name="FALLBACK_${MY_ID}_STOP_ON_NO_NET"
|
||
eval "echo \"\${${var_name}[@]:-}\""
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
state_init
|
||
set_ddns_arrays
|
||
CURRENT_STATE=$(state_get state)
|
||
FALLBACK_START_TS=$(state_get fallback_start)
|
||
TIER2=$(state_get tier2_started)
|
||
TIER3=$(state_get tier3_started)
|
||
TIER4=$(state_get tier4_started)
|
||
STRIKES=$(state_get handback_strikes)
|
||
|
||
local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
|
||
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY FALLBACK STATUS ━━━━━"
|
||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||
echo "$ICON_GEAR OS ver: $local_ver"
|
||
echo "$ICON_FALLBACK State: $CURRENT_STATE"
|
||
echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]:-none}"
|
||
echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}"
|
||
echo "$ICON_TIME Interval: ${FALLBACK_CHECK_INTERVAL}s"
|
||
echo "$ICON_FALLBACK Strikes: $STRIKES / $FALLBACK_HANDBACK_STRIKES"
|
||
|
||
if [[ "$CURRENT_STATE" == "FALLBACK" && "$FALLBACK_START_TS" -gt 0 ]]; then
|
||
ELAPSED=$(( ($(date +%s) - FALLBACK_START_TS) / 60 ))
|
||
echo "$ICON_TIME Outage: ${ELAPSED}min"
|
||
echo "$ICON_FALLBACK Tier 2: $TIER2 (delay: $(get_tier2_delay)min)"
|
||
echo "$ICON_FALLBACK Tier 3: $TIER3 (delay: $(get_tier3_delay)min)"
|
||
echo "$ICON_FALLBACK Tier 4: $TIER4 (delay: $(get_tier4_delay)min)"
|
||
fi
|
||
|
||
echo "$ICON_GEAR Enabled: $FALLBACK_ENABLED"
|
||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||
|
||
SUSPENDED=$(state_get partnership_suspended)
|
||
PLOST=$(state_get partner_lost_at)
|
||
if [[ "$SUSPENDED" == "true" ]]; then
|
||
echo "$ICON_GEAR Partnership: SUSPENDED (no active partnership)"
|
||
elif [[ -n "$PLOST" && "$PLOST" != "0" ]]; then
|
||
PLOST_MIN=$(( ($(date +%s) - PLOST) / 60 ))
|
||
echo "$ICON_GEAR Partnership: GRACE (${PLOST_MIN}min / ${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min until suspend)"
|
||
else
|
||
echo "$ICON_GEAR Partnership: ACTIVE"
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
# Called when remote returns after FALLBACK state.
|
||
# CRITICAL sequencing — do not reorder without understanding the consequences.
|
||
# Each step must succeed before proceeding — aborts and retries next cycle on failure.
|
||
|
||
run_handback() {
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "$REMOTE_SERVER_NAME has returned — beginning staged handback"
|
||
|
||
# ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||
|
||
if ! check_os_version_parity; then
|
||
warn "Version parity check failed — aborting handback, will retry next cycle"
|
||
state_set handback_strikes 0
|
||
return 1
|
||
fi
|
||
|
||
if ! check_remote_array; then
|
||
warn "Remote array not ready — aborting handback, will retry next cycle"
|
||
state_set handback_strikes 0
|
||
return 1
|
||
fi
|
||
|
||
if ! check_remote_docker_daemon; then
|
||
warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
|
||
state_set handback_strikes 0
|
||
return 1
|
||
fi
|
||
|
||
echo " Pre-flight checks passed ✅"
|
||
|
||
local outage_minutes
|
||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||
local tier1_wb_delay t2_delay t3_delay t4_delay
|
||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||
t2_delay=$(get_tier2_delay)
|
||
t3_delay=$(get_tier3_delay)
|
||
t4_delay=$(get_tier4_delay)
|
||
|
||
warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1"
|
||
|
||
# ── Staged handback helper ────────────────────────────────────────────────────────────────
|
||
# Each tier: stop local containers → rsync writeback → start on remote.
|
||
# Skips tiers that were never activated. Emby (Tier 1) stays on fallback
|
||
# serving users until all higher tiers complete.
|
||
handback_tier() {
|
||
local tier="$1" threshold="$2" label="$3"
|
||
|
||
[[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━"
|
||
|
||
local containers
|
||
read -r -a containers <<< "$(get_tier_containers "$tier")"
|
||
|
||
echo " Stopping local Tier $tier containers..."
|
||
for container in "${containers[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
|
||
if ! check_rsync_enabled "FALLBACK"; then
|
||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback"
|
||
else
|
||
local jobs
|
||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||
if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then
|
||
echo " Rsync writeback — outage ${outage_minutes}min >= ${threshold}min"
|
||
for job in "${jobs[@]}"; do
|
||
[[ -z "$job" ]] && continue
|
||
log "Syncing: $job"
|
||
# if/else, not A && B || C. In the shorthand a REAL run whose rsync exits
|
||
# non-zero falls through to the || branch and logs "DRY RUN — would rsync",
|
||
# so a failed Tier writeback reported itself as a preview and the real
|
||
# failure went unsaid. The Tier 1 block below always had this right.
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||
|| error "Tier $tier writeback FAILED: $job"
|
||
else
|
||
warn "DRY RUN — would rsync: $job"
|
||
fi
|
||
done
|
||
else
|
||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||
fi
|
||
fi
|
||
|
||
echo " Starting Tier $tier on $REMOTE_SERVER_NAME..."
|
||
for container in "${containers[@]}"; do
|
||
[[ -n "$container" ]] && remote_start "$container"
|
||
done
|
||
|
||
state_set "tier${tier}_started" "false"
|
||
warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME"
|
||
}
|
||
|
||
# ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ──────────────────────────────────
|
||
handback_tier 4 "$t4_delay" "media shares + edge cases"
|
||
handback_tier 3 "$t3_delay" "secondary services"
|
||
handback_tier 2 "$t2_delay" "productivity services"
|
||
|
||
# ── Step 5: DDNS handoff — immediately before Tier 1 goes down ──────────────────────────
|
||
# Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks.
|
||
# Prevents split-brain DNS during Tier 1 rsync window.
|
||
echo ""
|
||
echo "━━━ $ICON_NET DDNS Handoff ━━━"
|
||
remote_ddns_stop
|
||
sleep 5
|
||
|
||
# ── Step 6: Tier 1 handback — vital services + Emby ─────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━"
|
||
|
||
read -r -a t1 <<< "$(get_tier_containers 1)"
|
||
|
||
echo " Stopping local Tier 1 containers..."
|
||
for container in "${t1[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local is_ddns=false
|
||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||
done
|
||
[[ "$is_ddns" == false ]] && local_stop "$container"
|
||
done
|
||
|
||
if ! check_rsync_enabled "FALLBACK"; then
|
||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback"
|
||
else
|
||
local t1_jobs
|
||
read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)"
|
||
if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then
|
||
echo " Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min"
|
||
for job in "${t1_jobs[@]}"; do
|
||
[[ -z "$job" ]] && continue
|
||
log "Syncing: $job"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||
else
|
||
warn "DRY RUN — would rsync: $job"
|
||
fi
|
||
done
|
||
else
|
||
log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min"
|
||
fi
|
||
fi
|
||
|
||
echo " Starting Tier 1 on $REMOTE_SERVER_NAME..."
|
||
sleep 10 # give databases time to initialise before apps
|
||
for container in "${t1[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
local is_ddns=false
|
||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||
done
|
||
[[ "$is_ddns" == false ]] && remote_start "$container"
|
||
done
|
||
|
||
# ── Step 7: Play State Sync — DNS held until sync succeeds or retries exhausted ──────────
|
||
# Retried so users land on current watch state — after a 3hr outage one successful
|
||
# run catches everything regardless of how many cron cycles were missed.
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Play State Sync ━━━"
|
||
local _sync_ok=false
|
||
local _retry_max="${PLAY_SYNC_HANDBACK_RETRIES:-5}"
|
||
local _retry_delay="${PLAY_SYNC_HANDBACK_RETRY_DELAY:-60}"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
for _attempt in $(seq 1 "$_retry_max"); do
|
||
if bash "$SCRIPT_DIR/../Media/play_state_sync.sh" --wait; then
|
||
_sync_ok=true
|
||
log "Play state synced ✅ (attempt ${_attempt}/${_retry_max})"
|
||
break
|
||
fi
|
||
if [[ "$_attempt" -lt "$_retry_max" ]]; then
|
||
warn "Play state sync failed (attempt ${_attempt}/${_retry_max}) — retrying in ${_retry_delay}s"
|
||
sleep "$_retry_delay"
|
||
fi
|
||
done
|
||
[[ "$_sync_ok" == false ]] && \
|
||
warn "Play state sync failed after ${_retry_max} attempts — proceeding to DNS cutover"
|
||
else
|
||
warn "DRY RUN — would retry play_state_sync --wait up to ${_retry_max} times before DNS cutover"
|
||
fi
|
||
|
||
# ── Step 8: DNS Cutover — final step ────────────────────────────────────────────────────
|
||
# DNS cuts over ONLY after Tier 1 containers confirmed up
|
||
echo ""
|
||
echo "━━━ $ICON_NET DNS Cutover ━━━"
|
||
sleep 5
|
||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||
remote_start "$container"
|
||
done
|
||
warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
|
||
|
||
# ── Step 9: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||
echo ""
|
||
state_set state "NORMAL"
|
||
state_set fallback_start "0"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
# Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback
|
||
local_ddns_start
|
||
|
||
warn "$ICON_DONE Handback complete — returned to NORMAL"
|
||
notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
|
||
"Fallback" "normal"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Main State Machine ━━━
|
||
# ==============================================================================================
|
||
state_init
|
||
set_ddns_arrays
|
||
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " $ICON_FALLBACK FALLBACK — $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||
echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
FALLBACK_RUNNING=true
|
||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
||
SIGTERM SIGINT
|
||
|
||
while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||
NOW=$(date +%s)
|
||
CURRENT_STATE=$(state_get state)
|
||
|
||
# ── Connectivity checks ──────────────────────────────────────────────────────────────────
|
||
REMOTE_UP=false
|
||
INTERNET_UP=false
|
||
|
||
ping_remote && REMOTE_UP=true
|
||
ping_internet && INTERNET_UP=true
|
||
|
||
log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"
|
||
|
||
# ── Partnership gate ──────────────────────────────────────────────────────────────────────
|
||
# ── Partnership Gate ──────────────────────────────────────────────────────────────────────
|
||
# A fallback without an active partner is meaningless — who are we covering for?
|
||
# The gate suspends the loop after a grace period and aborts any live fallback
|
||
# containers. Clears automatically when partnership becomes active again.
|
||
if check_partnership_active; then
|
||
# Partnership healthy — clear any accumulated grace state
|
||
if [[ "$(state_get partnership_suspended)" == "true" ]]; then
|
||
warn "Active partnership restored — re-enabling fallback monitoring"
|
||
state_set partnership_suspended "false"
|
||
state_set partner_lost_at "0"
|
||
notify "Fallback re-enabled on $(hostname) — active partnership restored" \
|
||
"Fallback" "normal"
|
||
elif [[ "$(state_get partner_lost_at)" != "0" ]]; then
|
||
log "Partnership recovered within grace window — resetting timer"
|
||
state_set partner_lost_at "0"
|
||
fi
|
||
else
|
||
PARTNER_LOST_AT=$(state_get partner_lost_at)
|
||
if [[ -z "$PARTNER_LOST_AT" || "$PARTNER_LOST_AT" == "0" ]]; then
|
||
warn "No active partnership detected — grace timer started (${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min before suspend)"
|
||
state_set partner_lost_at "$NOW"
|
||
PARTNER_LOST_AT=$NOW
|
||
fi
|
||
|
||
PARTNER_LOST_MIN=$(( (NOW - PARTNER_LOST_AT) / 60 ))
|
||
SUSPEND_AFTER=${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}
|
||
|
||
if [[ "$(state_get partnership_suspended)" == "true" || "$PARTNER_LOST_MIN" -ge "$SUSPEND_AFTER" ]]; then
|
||
# Suspend — abort any live fallback containers, then skip this cycle entirely
|
||
if [[ "$(state_get partnership_suspended)" != "true" ]]; then
|
||
warn "No active partnership for ${PARTNER_LOST_MIN}min — suspending fallback"
|
||
state_set partnership_suspended "true"
|
||
notify "Fallback suspended on $(hostname) — no active partnership for ${PARTNER_LOST_MIN}min" \
|
||
"Fallback" "warning"
|
||
fi
|
||
if [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||
warn "Aborting in-progress fallback — partnership suspended"
|
||
_abort_fallback_containers
|
||
state_set state "NORMAL"
|
||
state_set fallback_start "0"
|
||
state_set handback_strikes "0"
|
||
notify "Fallback aborted on $(hostname) — no active partnership, containers stopped" \
|
||
"Fallback" "warning"
|
||
fi
|
||
log "Partnership suspended — skipping fallback cycle"
|
||
sleep "$FALLBACK_CHECK_INTERVAL" & wait $!
|
||
continue
|
||
else
|
||
log "No active partnership — ${PARTNER_LOST_MIN}min elapsed / ${SUSPEND_AFTER}min until suspend"
|
||
fi
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# NORMAL STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
||
|
||
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
||
log "$ICON_SUCCESS NORMAL — all systems up"
|
||
|
||
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
||
# Remote is down — enter FALLBACK immediately
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"
|
||
|
||
state_set state "FALLBACK"
|
||
state_set fallback_start "$NOW"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||
for container in "${tier1[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
|
||
notify "FALLBACK started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \
|
||
"Fallback" "warning"
|
||
|
||
elif [[ "$INTERNET_UP" == false ]]; then
|
||
# Lost internet — enter NO_INTERNET
|
||
echo ""
|
||
echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "Internet connectivity lost — stopping local DDNS"
|
||
|
||
state_set state "NO_INTERNET"
|
||
local_ddns_stop
|
||
|
||
# Stop containers configured to stop on internet loss
|
||
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
|
||
for container in "${stop_on_no_net[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
|
||
notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# FALLBACK STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||
|
||
FALLBACK_START_TS=$(state_get fallback_start)
|
||
ELAPSED_MIN=$(( (NOW - FALLBACK_START_TS) / 60 ))
|
||
|
||
if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
|
||
# Remote returned — increment handback strikes
|
||
STRIKES=$(state_get handback_strikes)
|
||
STRIKES=$(( STRIKES + 1 ))
|
||
state_set handback_strikes "$STRIKES"
|
||
|
||
warn "$ICON_FALLBACK $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FALLBACK_HANDBACK_STRIKES"
|
||
|
||
if [[ "$STRIKES" -ge "$FALLBACK_HANDBACK_STRIKES" ]]; then
|
||
run_handback
|
||
fi
|
||
|
||
elif [[ "$INTERNET_UP" == false ]]; then
|
||
# Lost internet during fallback — enter DARK — same actions as NO_INTERNET
|
||
echo ""
|
||
echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
warn "Lost internet during fallback — entering DARK state"
|
||
state_set state "DARK"
|
||
local_ddns_stop
|
||
|
||
# Stop containers configured to stop on internet loss
|
||
read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
|
||
for container in "${stop_on_no_net[@]}"; do
|
||
[[ -n "$container" ]] && local_stop "$container"
|
||
done
|
||
|
||
notify "DARK state on $(hostname) — lost internet during fallback" \
|
||
"Fallback" "warning"
|
||
|
||
else
|
||
# Still in FALLBACK state — check tier escalation
|
||
state_set handback_strikes "0"
|
||
|
||
TIER2_DELAY=$(get_tier2_delay)
|
||
if [[ "$(state_get tier2_started)" == "false" && \
|
||
"$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
|
||
read -r -a tier2 <<< "$(get_tier_containers 2)"
|
||
for container in "${tier2[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
state_set tier2_started "true"
|
||
notify "Fallback Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
TIER3_DELAY=$(get_tier3_delay)
|
||
if [[ "$(state_get tier3_started)" == "false" && \
|
||
"$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
|
||
read -r -a tier3 <<< "$(get_tier_containers 3)"
|
||
for container in "${tier3[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
state_set tier3_started "true"
|
||
notify "Fallback Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
TIER4_DELAY=$(get_tier4_delay)
|
||
if [[ "$(state_get tier4_started)" == "false" && \
|
||
"$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
|
||
read -r -a tier4 <<< "$(get_tier_containers 4)"
|
||
for container in "${tier4[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
state_set tier4_started "true"
|
||
notify "Fallback Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \
|
||
"Fallback" "warning"
|
||
fi
|
||
|
||
log "$ICON_FALLBACK FALLBACK — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# NO_INTERNET STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then
|
||
|
||
if [[ "$INTERNET_UP" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
if [[ "$REMOTE_UP" == true ]]; then
|
||
warn "Remote is up — returning to NORMAL"
|
||
state_set state "NORMAL"
|
||
local_ddns_start
|
||
notify "Internet recovered on $(hostname) — returning to NORMAL" \
|
||
"Fallback" "normal"
|
||
else
|
||
warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
|
||
state_set state "FALLBACK"
|
||
state_set fallback_start "$NOW"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
|
||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||
for container in "${tier1[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FALLBACK" \
|
||
"Fallback" "warning"
|
||
fi
|
||
else
|
||
log "$ICON_NET NO_INTERNET — waiting for connectivity"
|
||
fi
|
||
|
||
# ════════════════════════════════════════════════════════════════
|
||
# DARK STATE
|
||
# ════════════════════════════════════════════════════════════════
|
||
elif [[ "$CURRENT_STATE" == "DARK" ]]; then
|
||
|
||
if [[ "$INTERNET_UP" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
if [[ "$REMOTE_UP" == true ]]; then
|
||
warn "Remote up, internet up — transitioning through FALLBACK for handback"
|
||
# Was in FALLBACK state before DARK — route through FALLBACK state for handback
|
||
state_set state "FALLBACK"
|
||
state_set handback_strikes "0"
|
||
else
|
||
warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
|
||
state_set state "FALLBACK"
|
||
state_set fallback_start "$NOW"
|
||
state_set handback_strikes "0"
|
||
state_set tier2_started "false"
|
||
state_set tier3_started "false"
|
||
state_set tier4_started "false"
|
||
|
||
read -r -a tier1 <<< "$(get_tier_containers 1)"
|
||
for container in "${tier1[@]}"; do
|
||
[[ -n "$container" ]] && local_start "$container"
|
||
done
|
||
fi
|
||
notify "Emerging from DARK state on $(hostname)" "Fallback" "warning"
|
||
else
|
||
log "$ICON_FALLBACK DARK — no internet, no remote — waiting"
|
||
fi
|
||
fi
|
||
|
||
# ── Sleep until next check ───────────────────────────────────────────────────────────────
|
||
log "Next check in ${FALLBACK_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
|
||
sleep "$FALLBACK_CHECK_INTERVAL" &
|
||
wait $!
|
||
|
||
done |