diff --git a/Failover/failover.sh b/Failover/failover.sh index d284829..eccf235 100644 --- a/Failover/failover.sh +++ b/Failover/failover.sh @@ -1,24 +1,73 @@ #!/bin/bash -# ----------------------------------------------------------------------------------------------- -# --------------------------------- Failover Script -------------------------------------------- -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= Failover =================================================== +# ============================================================================================== # Mutual container failover between two unRAID servers. -# Runs continuously on BOTH servers — each operates fully autonomously. -# No coordination between servers — decisions based solely on ping results. +# Each server runs this script independently — no direct coordination between servers. +# All decisions based solely on two pings: remote reachable + internet reachable. # -# States: -# NORMAL — remote up, internet up — own containers only -# FAILOVER — remote down, internet up — own + remote's containers (additive) -# NO_INTERNET — internet down — stop public-facing containers -# DARK — remote down + internet down — same as NO_INTERNET +# ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── +# Both servers run this script continuously as a background task via User Scripts. +# Every FAILOVER_CHECK_INTERVAL seconds each server: +# 1. Pings the remote server +# 2. Pings the internet +# 3. Determines its current state +# 4. Takes the appropriate action # -# Handback (remote returns after FAILOVER): -# Strike confirmation → pre-flight checks → rsync → start on remote → stop locally +# No SSH signaling, no shared state files, no coordination — each server acts autonomously +# based only on what it can see from its own network perspective. +# +# ── STATES ──────────────────────────────────────────────────────────────────────────────────── +# NORMAL — remote up, internet up +# Own containers only. DDNS ON. Silent operation. +# +# FAILOVER — remote down, internet up +# Start remote containers locally — tiered by outage duration. +# Remote DDNS started immediately (Tier 1). +# Own containers keep running — failover is additive. +# +# NO_INTERNET — internet down (remote may be up or down) +# Stop own DDNS immediately — can't update DNS without internet. +# Do not start remote containers — no internet = no point. +# Wait for recovery. +# +# DARK — remote down AND internet down +# Same actions as NO_INTERNET. +# Cannot determine if remote is truly down or just unreachable. +# +# ── DDNS RULES — ABSOLUTE ───────────────────────────────────────────────────────────────────── +# Each server owns its own DDNS — ON when that server has internet. +# Script controls DDNS exclusively — network state NEVER auto-starts DDNS. +# DDNS only starts after full handback sequence confirms containers are up. +# One DDNS per domain active at all times — never two, never zero for long. +# 1 minute TTL + 1 minute check interval = minimal user impact on failover. +# +# ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── +# When remote returns after FAILOVER: +# 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks +# prevents handing back during a brief network blip +# 2. Pre-flight checks — remote array started, Docker healthy, rootfs not full +# 3. Stop remote DDNS first — prevents split brain DNS during rsync +# 4. Stop remote containers — clean state, no dirty writes during rsync +# 5. Rsync writeback — full bandwidth, clean source, minimal data +# 6. Start local containers — confirmed up before DNS cuts over +# 7. Start local DDNS — DNS cuts back ONLY after containers confirmed up +# 8. Return to NORMAL +# +# ── TIERED FAILOVER ─────────────────────────────────────────────────────────────────────────── +# Tier 1 — Immediate — vital services + Live TV — can't wait +# Tier 2 — 2hr default — shared productivity services +# Tier 3 — 6hr default — secondary services +# Tier 4 — 18hr default — arrs + downloaders — minimal writeback on handback +# Delays configurable per host in Master.conf +# +# ── TO STOP THIS SCRIPT ─────────────────────────────────────────────────────────────────────── +# Click Abort beside the script in unRAID's User Scripts plugin page. +# Do NOT kill the process directly — state file may be left in inconsistent state. # # All configuration in Master.conf under Failover section. -# Run via User Scripts plugin as a background task — runs continuously until stopped. -# Supports --dry-run to walk through logic without taking any action. -# ----------------------------------------------------------------------------------------------- +# Supports --dry-run (no container changes) and --status (show current state and exit). +# ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -27,9 +76,9 @@ source "$SCRIPT_DIR/../common.sh" parse_args "$@" -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== # ━━━ $ICON_GEAR Setup ━━━ -# ----------------------------------------------------------------------------------------------- +# ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" @@ -39,474 +88,659 @@ if [[ "$EUID" -ne 0 ]]; then fi success "Running as root" - detect_hosts resolve_remote_ip -# Select correct arrays based on which server we are -if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then - FAILOVER_START_CONTAINERS=("${FAILOVER_HOST1_STARTS_FOR_HOST2[@]}") - FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST1_STOP_ON_NO_NET[@]}") - FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST1_RSYNC_JOBS[@]}") -else - FAILOVER_START_CONTAINERS=("${FAILOVER_HOST2_STARTS_FOR_HOST1[@]}") - FAILOVER_STOP_ON_NO_NET=("${FAILOVER_HOST2_STOP_ON_NO_NET[@]}") - FAILOVER_RSYNC_JOBS=("${FAILOVER_HOST2_RSYNC_JOBS[@]}") -fi +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made" -info "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}" -info "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}" -info "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}" +# ============================================================================================== +# STATE FILE HELPERS +# State file format — key=value, one per line, persists on /boot/ +# Tracks: state, failover_start, handback_strikes, tier2_started, tier3_started, tier4_started +# ============================================================================================== -# Ensure state file directory exists -touch "$FAILOVER_STATE_FILE" 2>/dev/null || { - error "Cannot create state file: $FAILOVER_STATE_FILE" - exit 1 +state_get() { + grep "^${1}=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2 } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_SUMMARY Status ━━━ -# ----------------------------------------------------------------------------------------------- +state_set() { + local key="$1" value="$2" + if grep -q "^${key}=" "$FAILOVER_STATE_FILE" 2>/dev/null; then + sed -i "s|^${key}=.*|${key}=${value}|" "$FAILOVER_STATE_FILE" + else + echo "${key}=${value}" >> "$FAILOVER_STATE_FILE" + fi +} + +state_init() { + mkdir -p "$(dirname "$FAILOVER_STATE_FILE")" + [[ ! -f "$FAILOVER_STATE_FILE" ]] && touch "$FAILOVER_STATE_FILE" + [[ -z "$(state_get state)" ]] && state_set state "NORMAL" + [[ -z "$(state_get failover_start)" ]] && state_set failover_start "0" + [[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0" + [[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false" + [[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false" + [[ -z "$(state_get tier4_started)" ]] && state_set tier4_started "false" +} + +# ============================================================================================== +# CONTAINER HELPERS +# ============================================================================================== + +# Start a container locally +local_start() { + local container="$1" + [[ -z "$container" ]] && return + local status + status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$status" == "true" ]]; then + log "$container already running locally" + return 0 + fi + info "$ICON_START Starting $container locally..." + if [[ "$DRY_RUN" == false ]]; then + docker start "$container" >/dev/null 2>&1 && \ + success "$ICON_STARTED $container started" || \ + error "Failed to start $container locally" + else + warn "DRY RUN — would start $container locally" + fi +} + +# Stop a container locally +local_stop() { + local container="$1" + [[ -z "$container" ]] && return + local status + status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + if [[ "$status" != "true" ]]; then + log "$container already stopped locally" + return 0 + fi + info "$ICON_STOP Stopping $container locally..." + if [[ "$DRY_RUN" == false ]]; then + docker stop "$container" >/dev/null 2>&1 && \ + success "$ICON_STOPPED $container stopped" || \ + error "Failed to stop $container locally" + else + warn "DRY RUN — would stop $container locally" + fi +} + +# Start a container on remote via SSH +remote_start() { + local container="$1" + [[ -z "$container" ]] && return + local status + status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null) + if [[ "$status" == "true" ]]; then + log "$container already running on $REMOTE_SERVER_NAME" + return 0 + fi + info "$ICON_START Starting $container on $REMOTE_SERVER_NAME..." + if [[ "$DRY_RUN" == false ]]; then + ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker start $container" >/dev/null 2>&1 && \ + success "$ICON_STARTED $container started on $REMOTE_SERVER_NAME" || \ + error "Failed to start $container on $REMOTE_SERVER_NAME" + else + warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME" + fi +} + +# Stop a container on remote via SSH +remote_stop() { + local container="$1" + [[ -z "$container" ]] && return + local status + status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null) + if [[ "$status" != "true" ]]; then + log "$container already stopped on $REMOTE_SERVER_NAME" + return 0 + fi + info "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..." + if [[ "$DRY_RUN" == false ]]; then + ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ + "docker stop $container" >/dev/null 2>&1 && \ + success "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \ + error "Failed to stop $container on $REMOTE_SERVER_NAME" + else + warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME" + fi +} + +# ============================================================================================== +# DDNS HELPERS +# DDNS is managed exclusively by this script — never by network state +# ============================================================================================== + +# Stop local DDNS — called on internet loss +local_ddns_stop() { + info "$ICON_NET Stopping local DDNS — internet lost" + for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do + local_stop "$container" + done +} + +# Start local DDNS — called at end of handback ONLY after containers confirmed up +local_ddns_start() { + info "$ICON_NET Starting local DDNS — containers confirmed up" + for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do + local_start "$container" + done +} + +# Stop remote DDNS — FIRST step of handback sequence, prevents split brain +remote_ddns_stop() { + info "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — handback starting" + for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do + remote_stop "$container" + done +} + +# ============================================================================================== +# TIERED FAILOVER HELPERS +# Containers selected based on which server is local and which arrays to use +# ============================================================================================== + +# Get the correct tier arrays based on local host +# HOST2 runs for HOST1, HOST1 runs for HOST2 +get_tier_containers() { + local tier="$1" + if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then + # HOST2 is running — covers HOST1 + case "$tier" in + 1) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE[@]}" ;; + 2) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_2HR[@]}" ;; + 3) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_6HR[@]}" ;; + 4) echo "${FAILOVER_HOST2_RUNS_FOR_HOST1_18HR[@]}" ;; + esac + else + # HOST1 is running — covers HOST2 + case "$tier" in + 1) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE[@]}" ;; + 2) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_2HR[@]}" ;; + 3) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_6HR[@]}" ;; + 4) echo "${FAILOVER_HOST1_RUNS_FOR_HOST2_18HR[@]}" ;; + esac + fi +} + +get_tier2_delay() { + [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ + echo "$HOST1_TIER2_DELAY" || echo "$HOST2_TIER2_DELAY" +} + +get_tier3_delay() { + [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ + echo "$HOST1_TIER3_DELAY" || echo "$HOST2_TIER3_DELAY" +} + +get_tier4_delay() { + [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \ + echo "$HOST1_TIER4_DELAY" || echo "$HOST2_TIER4_DELAY" +} + +get_writeback_jobs() { + if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then + echo "${FAILOVER_HOST1_WRITEBACK[@]}" + else + echo "${FAILOVER_HOST2_WRITEBACK[@]}" + fi +} + +# Select correct DDNS arrays based on local host +set_ddns_arrays() { + if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + LOCAL_DDNS_CONTAINERS=("${HOST1_DDNS_CONTAINERS[@]}") + REMOTE_DDNS_CONTAINERS=("${HOST2_DDNS_CONTAINERS[@]}") + else + LOCAL_DDNS_CONTAINERS=("${HOST2_DDNS_CONTAINERS[@]}") + REMOTE_DDNS_CONTAINERS=("${HOST1_DDNS_CONTAINERS[@]}") + fi +} + +# ============================================================================================== +# STATUS DISPLAY +# ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then + state_init + set_ddns_arrays + CURRENT_STATE=$(state_get state) + FAILOVER_START=$(state_get failover_start) + TIER2=$(state_get tier2_started) + TIER3=$(state_get tier3_started) + TIER4=$(state_get tier4_started) + STRIKES=$(state_get handback_strikes) + echo "" echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━" - echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" - echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" - echo "$ICON_PING External IP: $EXTERNAL_IP" - echo "$ICON_TIME Check interval: ${FAILOVER_CHECK_INTERVAL}s" - echo "$ICON_RETRY Handback strikes: $FAILOVER_HANDBACK_STRIKES" - echo "$ICON_FAILOVER Failover containers: ${FAILOVER_START_CONTAINERS[*]}" - echo "$ICON_STOP Stop on no-net: ${FAILOVER_STOP_ON_NO_NET[*]}" - echo "$ICON_SYNC Rsync jobs: ${FAILOVER_RSYNC_JOBS[*]}" - echo "$ICON_GEAR State file: $FAILOVER_STATE_FILE" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" + echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" + echo "$ICON_FAILOVER State: $CURRENT_STATE" + echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]}" + echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]}" + echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s" + echo "$ICON_FAILOVER Strikes: $STRIKES / $FAILOVER_HANDBACK_STRIKES" - if [[ -f "$FAILOVER_STATE_FILE" ]]; then - echo "" - echo "━━━ Persisted State ━━━" - cat "$FAILOVER_STATE_FILE" + if [[ "$CURRENT_STATE" == "FAILOVER" && "$FAILOVER_START" -gt 0 ]]; then + ELAPSED=$(( ($(date +%s) - FAILOVER_START) / 60 )) + echo "$ICON_TIME Outage: ${ELAPSED}min" + echo "$ICON_FAILOVER Tier 2: $TIER2 (delay: $(get_tier2_delay)min)" + echo "$ICON_FAILOVER Tier 3: $TIER3 (delay: $(get_tier3_delay)min)" + echo "$ICON_FAILOVER Tier 4: $TIER4 (delay: $(get_tier4_delay)min)" fi - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be started or stopped" - -# ----------------------------------------------------------------------------------------------- -# STATE FILE HELPERS -# State persists to /boot/ so it survives reboots. -# On restart the script re-evaluates from scratch using live ping results. -# State file is reference only — pings are always the source of truth. -# ----------------------------------------------------------------------------------------------- - -get_state() { - grep -E "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d'=' -f2 -} - -set_state() { - local new_state="$1" - local timestamp - timestamp=$(date '+%Y-%m-%d %H:%M:%S') - cat > "$FAILOVER_STATE_FILE" </dev/null | cut -d'=' -f2 -} - -set_handback_strikes() { - local count="$1" - if grep -q "^handback_strikes=" "$FAILOVER_STATE_FILE" 2>/dev/null; then - sed -i "s/^handback_strikes=.*/handback_strikes=$count/" "$FAILOVER_STATE_FILE" - else - echo "handback_strikes=$count" >> "$FAILOVER_STATE_FILE" - fi -} - -# ----------------------------------------------------------------------------------------------- -# CONTAINER HELPERS — LOCAL operations for failover -# These operate on the LOCAL server unlike stop_containers/start_containers in common.sh -# which operate on the REMOTE server via SSH. -# ----------------------------------------------------------------------------------------------- - -# Start a container locally — skip if already running -start_local_container() { - local container="$1" - - local status - status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - - if [[ "$status" == "true" ]]; then - log "$container already running locally — skipping" - return 0 - fi - - if [[ "$status" == "unknown" ]]; then - warn "$container not found on this host — skipping" - return 1 - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would start $container locally" - return 0 - fi - - echo "$ICON_START Starting $container locally..." - if docker start "$container" >/dev/null 2>&1; then - echo "$ICON_STARTED $container started" - return 0 - else - error "Failed to start $container locally" - return 1 - fi -} - -# Stop a container locally — skip if already stopped -stop_local_container() { - local container="$1" - - local status - status=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") - - if [[ "$status" == "false" ]]; then - log "$container already stopped locally — skipping" - return 0 - fi - - if [[ "$status" == "unknown" ]]; then - log "$container not found on this host — skipping" - return 0 - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would stop $container locally" - return 0 - fi - - echo "$ICON_STOP Stopping $container locally..." - if docker stop "$container" >/dev/null 2>&1; then - echo "$ICON_STOPPED $container stopped" - return 0 - else - error "Failed to stop $container locally" - return 1 - fi -} - -# Start a container on the remote server via SSH -start_remote_container() { - local container="$1" - - local status - status=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "docker inspect -f '{{.State.Running}}' $container 2>/dev/null || echo unknown" 2>/dev/null) - - if [[ "$status" == "true" ]]; then - log "$container already running on remote — skipping" - return 0 - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME" - return 0 - fi - - echo "$ICON_START Starting $container on $REMOTE_SERVER_NAME..." - if ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "docker start $container" >/dev/null 2>&1; then - echo "$ICON_STARTED $container started on $REMOTE_SERVER_NAME" - return 0 - else - error "Failed to start $container on $REMOTE_SERVER_NAME" - return 1 - fi -} - -# ----------------------------------------------------------------------------------------------- -# STATE ACTIONS -# Each state has a corresponding action function. -# These are idempotent — safe to call on every loop iteration. -# ----------------------------------------------------------------------------------------------- - -# NORMAL — remote is up, internet is up -# Ensure failover containers are stopped locally (cleanup after returning from FAILOVER) -action_normal() { - local prev_state="$1" - - if [[ "$prev_state" == "FAILOVER" ]]; then - info "$ICON_FAILOVER Returning from FAILOVER — cleaning up local failover containers" - for c in "${FAILOVER_START_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - stop_local_container "$c" - done - notify "Failover ended on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME is back online" "Failover" "normal" - fi - - # Ensure no-net containers are running (they may have been stopped) - if [[ "$prev_state" == "NO_INTERNET" || "$prev_state" == "DARK" ]]; then - info "$ICON_START Internet restored — starting previously stopped containers" - for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do - [[ -z "$c" ]] && continue - start_local_container "$c" - done - notify "Internet restored on $LOCAL_SERVER_NAME — normal containers restarted" "Failover" "normal" - fi -} - -# FAILOVER — remote is down, internet is up -# Start remote's containers locally (additive — own containers keep running) -action_failover() { - local prev_state="$1" - - if [[ "$prev_state" != "FAILOVER" ]]; then - # Just entered failover — notify and start containers - info "$ICON_FAILOVER Entering FAILOVER — $REMOTE_SERVER_NAME is unreachable" - - # Check local array before starting containers - if ! check_local_array; then - error "Local array not ready — cannot start failover containers safely" - notify "Failover triggered but local array not ready on $LOCAL_SERVER_NAME" "Failover" "warning" - return 1 - fi - - # Check local docker daemon - if ! timeout 10 docker ps >/dev/null 2>&1; then - error "Local Docker daemon not responding — cannot start failover containers" - notify "Failover triggered but Docker not ready on $LOCAL_SERVER_NAME" "Failover" "warning" - return 1 - fi - - notify "Failover ACTIVE on $LOCAL_SERVER_NAME — starting ${REMOTE_SERVER_NAME} containers: ${FAILOVER_START_CONTAINERS[*]}" "Failover" "warning" - fi - - # Start each failover container — idempotent, skips if already running - for c in "${FAILOVER_START_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - start_local_container "$c" - done -} - -# NO_INTERNET / DARK — this server has no internet -# Stop public-facing containers — no point serving if offline -action_no_internet() { - local prev_state="$1" - - if [[ "$prev_state" != "NO_INTERNET" && "$prev_state" != "DARK" ]]; then - warn "$ICON_WARN Internet lost on $LOCAL_SERVER_NAME — stopping public containers" - notify "Internet lost on $LOCAL_SERVER_NAME — stopping public-facing containers" "Failover" "warning" - fi - - for c in "${FAILOVER_STOP_ON_NO_NET[@]}"; do - [[ -z "$c" ]] && continue - stop_local_container "$c" - done - - # If we were in FAILOVER, also stop the failover containers - if [[ "$prev_state" == "FAILOVER" ]]; then - info "Was in FAILOVER — stopping failover containers too" - for c in "${FAILOVER_START_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - stop_local_container "$c" - done - fi -} - -# HANDBACK — remote has returned, ready to hand containers back -# Confirmed by FAILOVER_HANDBACK_STRIKES consecutive remote-up checks -action_handback() { - info "$ICON_FAILOVER Initiating handback to $REMOTE_SERVER_NAME..." - notify "Handback starting on $LOCAL_SERVER_NAME — syncing data to $REMOTE_SERVER_NAME" "Failover" "normal" - - # Pre-flight checks before rsync +# ============================================================================================== +# HANDBACK SEQUENCE +# Called when remote returns after FAILOVER state +# Critical sequencing — do not reorder without understanding the consequences +# ============================================================================================== +run_handback() { echo "" - echo "━━━ $ICON_SHIELD Handback Pre-flight ━━━" + echo "━━━ $ICON_FAILOVER Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + info "Remote $REMOTE_SERVER_NAME has returned — beginning handback" + + # ── Step 1: Pre-flight checks ──────────────────────────────────────────────────────────── + echo "" + echo "━━━ $ICON_SHIELD Pre-flight ━━━" if ! check_remote_array; then - error "Remote array not ready — deferring handback" - notify "Handback deferred on $LOCAL_SERVER_NAME — remote array not ready" "Failover" "warning" - set_handback_strikes 0 + warn "Remote array not ready — aborting handback, will retry next cycle" + state_set handback_strikes 0 return 1 fi - success "Remote array ready" if ! check_remote_docker; then - error "Remote Docker not ready — deferring handback" - notify "Handback deferred on $LOCAL_SERVER_NAME — remote Docker not ready" "Failover" "warning" - set_handback_strikes 0 + warn "Remote Docker not ready — aborting handback, will retry next cycle" + state_set handback_strikes 0 return 1 fi - success "Remote Docker ready" - REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) - if [[ -n "$REMOTE_USAGE" ]] && [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then - error "Remote rootfs ${REMOTE_USAGE}% full — deferring handback" - notify "Handback deferred — remote rootfs ${REMOTE_USAGE}% full on $REMOTE_SERVER_NAME" "Failover" "warning" - set_handback_strikes 0 - return 1 - fi - success "Remote rootfs healthy" + success "Pre-flight checks passed" - # Rsync data back via rsync.sh — uses existing profile system + # ── Step 2: Stop remote DDNS FIRST ────────────────────────────────────────────────────── + # CRITICAL — prevents split brain DNS during rsync window + # Remote DDNS must be OFF before any data moves echo "" - echo "━━━ $ICON_SYNC Handback Rsync ━━━" - local rsync_failed=false + echo "━━━ $ICON_NET DDNS Handoff ━━━" + remote_ddns_stop + info "Remote DDNS stopped — DNS will not update during rsync" + sleep 5 # brief pause to ensure DDNS stop propagates - for path in "${FAILOVER_RSYNC_JOBS[@]}"; do - [[ -z "$path" ]] && continue - info "Syncing $path → $REMOTE_SERVER_NAME" + # ── Step 3: Stop remote containers ────────────────────────────────────────────────────── + # Clean state before rsync — no dirty writes during transfer + # Containers are only down during the rsync window — minimise this time + echo "" + echo "━━━ $ICON_STOP Stop Remote Containers ━━━" - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would rsync $path" - else - if bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$path" --no-log; then - success "Rsync complete: $path" + # Stop all tiers that were started — in reverse order + local ALL_FAILOVER_CONTAINERS=() + + if [[ "$(state_get tier4_started)" == "true" ]]; then + read -r -a t4 <<< "$(get_tier_containers 4)" + ALL_FAILOVER_CONTAINERS+=("${t4[@]}") + fi + if [[ "$(state_get tier3_started)" == "true" ]]; then + read -r -a t3 <<< "$(get_tier_containers 3)" + ALL_FAILOVER_CONTAINERS+=("${t3[@]}") + fi + if [[ "$(state_get tier2_started)" == "true" ]]; then + read -r -a t2 <<< "$(get_tier_containers 2)" + ALL_FAILOVER_CONTAINERS+=("${t2[@]}") + fi + + # Tier 1 always started — stop last (DDNS already stopped above) + read -r -a t1 <<< "$(get_tier_containers 1)" + # Remove DDNS containers from tier 1 list — already handled + for container in "${t1[@]}"; do + local is_ddns=false + for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do + [[ "$container" == "$ddns" ]] && is_ddns=true && break + done + [[ "$is_ddns" == false ]] && ALL_FAILOVER_CONTAINERS+=("$container") + done + + for container in "${ALL_FAILOVER_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + local_stop "$container" + done + + success "All failover containers stopped locally" + + # ── Step 4: Rsync writeback ────────────────────────────────────────────────────────────── + # Full bandwidth, clean source, minimal data + # Only critical appdata synced back — media files and downloads skipped + echo "" + echo "━━━ $ICON_SYNC Rsync Writeback ━━━" + + local writeback_jobs + read -r -a writeback_jobs <<< "$(get_writeback_jobs)" + + if [[ ${#writeback_jobs[@]} -eq 0 ]]; then + info "No writeback jobs configured — skipping rsync" + else + for job in "${writeback_jobs[@]}"; do + [[ -z "$job" ]] && continue + info "Syncing: $job" + if [[ "$DRY_RUN" == false ]]; then + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" else - error "Rsync failed: $path" - rsync_failed=true + warn "DRY RUN — would rsync: $job" fi - fi - done - - if [[ "$rsync_failed" == true ]]; then - error "One or more rsync jobs failed — deferring container handback" - notify "Handback rsync failed on $LOCAL_SERVER_NAME — containers not handed back yet" "Failover" "warning" - set_handback_strikes 0 - return 1 + done fi - # Start containers on remote - echo "" - echo "━━━ $ICON_START $ICON_CONTAINERS Start on Remote ━━━" - local start_failed=false + success "Writeback complete" - for c in "${FAILOVER_START_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - if ! start_remote_container "$c"; then - start_failed=true - fi + # ── Step 5: Start remote containers ───────────────────────────────────────────────────── + # Start in dependency order — databases before apps + echo "" + echo "━━━ $ICON_START Start Remote Containers ━━━" + + # Start tier 1 on remote (excluding DDNS — handled separately) + for container in "${t1[@]}"; do + [[ -z "$container" ]] && continue + local is_ddns=false + for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do + [[ "$container" == "$ddns" ]] && is_ddns=true && break + done + [[ "$is_ddns" == false ]] && remote_start "$container" done - if [[ "$start_failed" == true ]]; then - error "One or more containers failed to start on remote — not stopping local copies" - notify "Handback partial failure on $LOCAL_SERVER_NAME — some containers failed to start on $REMOTE_SERVER_NAME" "Failover" "warning" - set_handback_strikes 0 - return 1 + # Brief pause — give databases time to initialise before apps + sleep 10 + + if [[ "$(state_get tier2_started)" == "true" ]]; then + for container in "${t2[@]}"; do + [[ -n "$container" ]] && remote_start "$container" + done fi - # Stop local failover containers — only after remote confirmed started - echo "" - echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Local Failover ━━━" - for c in "${FAILOVER_START_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - stop_local_container "$c" - done + if [[ "$(state_get tier3_started)" == "true" ]]; then + for container in "${t3[@]}"; do + [[ -n "$container" ]] && remote_start "$container" + done + fi - notify "Handback complete on $LOCAL_SERVER_NAME — $REMOTE_SERVER_NAME has containers back" "Failover" "normal" - set_handback_strikes 0 - return 0 + if [[ "$(state_get tier4_started)" == "true" ]]; then + for container in "${t4[@]}"; do + [[ -n "$container" ]] && remote_start "$container" + done + fi + + success "Remote containers started" + + # ── Step 6: Start remote DDNS LAST ────────────────────────────────────────────────────── + # DNS cuts over ONLY after containers confirmed up + # This is the final step — after this users hit remote + echo "" + echo "━━━ $ICON_NET DNS Cutover ━━━" + sleep 5 # brief pause to ensure containers are accepting connections + for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do + remote_start "$container" + done + success "Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME" + + # ── Step 7: Return to NORMAL ───────────────────────────────────────────────────────────── + echo "" + state_set state "NORMAL" + state_set failover_start "0" + state_set handback_strikes "0" + state_set tier2_started "false" + state_set tier3_started "false" + state_set tier4_started "false" + + success "$ICON_DONE Handback complete — returned to NORMAL" + notify "Failover handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" "Failover" "normal" } -# ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_FAILOVER Main Loop ━━━ -# ----------------------------------------------------------------------------------------------- -echo "" -echo "━━━ $ICON_FAILOVER Failover Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━" -echo "$ICON_HOST Local: $LOCAL_SERVER_NAME" -echo "$ICON_NET Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)" -echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s" -echo "" +# ============================================================================================== +# MAIN STATE MACHINE +# ============================================================================================== +state_init +set_ddns_arrays -notify "Failover script started on $LOCAL_SERVER_NAME — monitoring $REMOTE_SERVER_NAME" "Failover" "normal" - -LOOP_COUNT=0 +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " $ICON_FAILOVER FAILOVER — $(date '+%Y-%m-%d %H:%M:%S')" +echo " $ICON_HOST $LOCAL_SERVER_NAME → monitoring $REMOTE_SERVER_NAME" +echo " $ICON_NET Remote: $REMOTE_SERVER" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" while true; do - LOOP_COUNT=$((LOOP_COUNT + 1)) - CURRENT_STATE=$(get_state) - [[ -z "$CURRENT_STATE" ]] && CURRENT_STATE="UNKNOWN" + NOW=$(date +%s) + CURRENT_STATE=$(state_get state) - echo "" - echo "━━━ $ICON_PING Check #${LOOP_COUNT} — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - log "Previous state: $CURRENT_STATE" - - # ── Two ping checks — the only inputs to state decisions ── + # ── Connectivity checks ────────────────────────────────────────────────────────────────── REMOTE_UP=false INTERNET_UP=false ping_remote && REMOTE_UP=true ping_internet && INTERNET_UP=true - log "Remote reachable: $REMOTE_UP | Internet reachable: $INTERNET_UP" + log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE" - # ── Determine new state ── - if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then - NEW_STATE="NORMAL" - elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then - NEW_STATE="FAILOVER" - elif [[ "$INTERNET_UP" == false ]]; then - NEW_STATE="NO_INTERNET" - else - NEW_STATE="DARK" - fi + # ── State machine ──────────────────────────────────────────────────────────────────────── - # ── Handle handback strike system ── - # When returning from FAILOVER to NORMAL, require consecutive confirmations - if [[ "$CURRENT_STATE" == "FAILOVER" && "$NEW_STATE" == "NORMAL" ]]; then - STRIKES=$(get_handback_strikes) - [[ -z "$STRIKES" ]] && STRIKES=0 - STRIKES=$((STRIKES + 1)) - set_handback_strikes "$STRIKES" + # ════════════════════════════════════════════════════════════════ + # NORMAL STATE + # ════════════════════════════════════════════════════════════════ + if [[ "$CURRENT_STATE" == "NORMAL" ]]; then - if [[ "$STRIKES" -lt "$FAILOVER_HANDBACK_STRIKES" ]]; then - info "$ICON_RETRY Remote appears back — strike $STRIKES/$FAILOVER_HANDBACK_STRIKES — waiting for confirmation" - NEW_STATE="FAILOVER" # Stay in FAILOVER until strikes confirmed - else - info "$ICON_RETRY Remote confirmed stable ($STRIKES/$FAILOVER_HANDBACK_STRIKES) — initiating handback" - NEW_STATE="HANDBACK" - fi - else - # Reset handback strikes if we're not in the confirmation window - set_handback_strikes 0 - fi + if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then + # All good — silent operation + log "$ICON_SUCCESS NORMAL — all systems up" - # ── Log state ── - if [[ "$NEW_STATE" != "$CURRENT_STATE" ]]; then - echo "$ICON_FAILOVER State: $CURRENT_STATE → $NEW_STATE" - else - info "State: $NEW_STATE (unchanged)" - fi + elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then + # Remote is down — enter FAILOVER + echo "" + echo "━━━ $ICON_FAILOVER Entering FAILOVER — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + warn "$REMOTE_SERVER_NAME is unreachable — internet is up — starting failover" - # ── Execute state action ── - case "$NEW_STATE" in - NORMAL) - action_normal "$CURRENT_STATE" - set_state "NORMAL" - ;; - FAILOVER) - action_failover "$CURRENT_STATE" - set_state "FAILOVER" - ;; - NO_INTERNET|DARK) - action_no_internet "$CURRENT_STATE" - set_state "$NEW_STATE" - ;; - HANDBACK) - if action_handback; then - set_state "NORMAL" - action_normal "FAILOVER" + state_set state "FAILOVER" + state_set failover_start "$NOW" + state_set handback_strikes "0" + state_set tier2_started "false" + state_set tier3_started "false" + state_set tier4_started "false" + + # Start Tier 1 immediately — DDNS first, then vital services + echo "" + echo "━━━ $ICON_START Tier 1 — Immediate ━━━" + read -r -a tier1 <<< "$(get_tier_containers 1)" + for container in "${tier1[@]}"; do + [[ -n "$container" ]] && local_start "$container" + done + + notify "FAILOVER started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 containers started" "Failover" "warning" + + elif [[ "$INTERNET_UP" == false ]]; then + # Lost internet — enter NO_INTERNET + echo "" + echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + warn "Internet connectivity lost — stopping local DDNS" + + state_set state "NO_INTERNET" + local_ddns_stop + + # Stop any containers configured to stop without internet + if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + for container in "${FAILOVER_HOST1_STOP_ON_NO_NET[@]}"; do + [[ -n "$container" ]] && local_stop "$container" + done else - # Handback failed — stay in FAILOVER, retry next cycle - set_state "FAILOVER" + for container in "${FAILOVER_HOST2_STOP_ON_NO_NET[@]}"; do + [[ -n "$container" ]] && local_stop "$container" + done fi - ;; - esac - echo "$ICON_TIME Next check in ${FAILOVER_CHECK_INTERVAL}s" + notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" "Failover" "warning" + fi + + # ════════════════════════════════════════════════════════════════ + # FAILOVER STATE + # ════════════════════════════════════════════════════════════════ + elif [[ "$CURRENT_STATE" == "FAILOVER" ]]; then + + FAILOVER_START=$(state_get failover_start) + ELAPSED_MIN=$(( (NOW - FAILOVER_START) / 60 )) + + if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then + # Remote returned — increment handback strikes + STRIKES=$(state_get handback_strikes) + STRIKES=$(( STRIKES + 1 )) + state_set handback_strikes "$STRIKES" + + info "$ICON_FAILOVER $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FAILOVER_HANDBACK_STRIKES" + + if [[ "$STRIKES" -ge "$FAILOVER_HANDBACK_STRIKES" ]]; then + run_handback + fi + + elif [[ "$INTERNET_UP" == false ]]; then + # Lost internet during failover — enter DARK + echo "" + echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + warn "Lost internet during failover — entering DARK state" + state_set state "DARK" + local_ddns_stop + notify "DARK state on $(hostname) — lost internet during failover" "Failover" "warning" + + else + # Still in failover — check tier escalation + state_set handback_strikes "0" # reset strikes — remote still down + + # Tier 2 + TIER2_DELAY=$(get_tier2_delay) + if [[ "$(state_get tier2_started)" == "false" && \ + "$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then + echo "" + echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━" + read -r -a tier2 <<< "$(get_tier_containers 2)" + for container in "${tier2[@]}"; do + [[ -n "$container" ]] && local_start "$container" + done + state_set tier2_started "true" + notify "Failover Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" "Failover" "warning" + fi + + # Tier 3 + TIER3_DELAY=$(get_tier3_delay) + if [[ "$(state_get tier3_started)" == "false" && \ + "$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then + echo "" + echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━" + read -r -a tier3 <<< "$(get_tier_containers 3)" + for container in "${tier3[@]}"; do + [[ -n "$container" ]] && local_start "$container" + done + state_set tier3_started "true" + notify "Failover Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" "Failover" "warning" + fi + + # Tier 4 + TIER4_DELAY=$(get_tier4_delay) + if [[ "$(state_get tier4_started)" == "false" && \ + "$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then + echo "" + echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━" + read -r -a tier4 <<< "$(get_tier_containers 4)" + for container in "${tier4[@]}"; do + [[ -n "$container" ]] && local_start "$container" + done + state_set tier4_started "true" + notify "Failover Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" "Failover" "warning" + fi + + log "$ICON_FAILOVER FAILOVER active — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)" + fi + + # ════════════════════════════════════════════════════════════════ + # NO_INTERNET STATE + # ════════════════════════════════════════════════════════════════ + elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then + + if [[ "$INTERNET_UP" == true ]]; then + # Internet recovered + echo "" + echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + + if [[ "$REMOTE_UP" == true ]]; then + # Remote up, internet up — return to NORMAL + info "Remote is up — returning to NORMAL" + state_set state "NORMAL" + # DDNS does NOT auto-start here — it only starts via handback sequence + # or explicit NORMAL state management + # If local DDNS should be running in NORMAL — start it now + local_ddns_start + notify "Internet recovered on $(hostname) — returning to NORMAL" "Failover" "normal" + else + # Internet back but remote still down — enter FAILOVER + info "Remote still down — entering FAILOVER" + state_set state "FAILOVER" + state_set failover_start "$NOW" + state_set handback_strikes "0" + state_set tier2_started "false" + state_set tier3_started "false" + state_set tier4_started "false" + + echo "" + echo "━━━ $ICON_START Tier 1 — Immediate ━━━" + read -r -a tier1 <<< "$(get_tier_containers 1)" + for container in "${tier1[@]}"; do + [[ -n "$container" ]] && local_start "$container" + done + notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FAILOVER" "Failover" "warning" + fi + else + log "$ICON_NET NO_INTERNET — waiting for connectivity" + fi + + # ════════════════════════════════════════════════════════════════ + # DARK STATE + # ════════════════════════════════════════════════════════════════ + elif [[ "$CURRENT_STATE" == "DARK" ]]; then + + if [[ "$INTERNET_UP" == true ]]; then + echo "" + echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + + if [[ "$REMOTE_UP" == true ]]; then + info "Remote up, internet up — returning to NORMAL" + # Was in failover before DARK — need to handback + state_set state "FAILOVER" + state_set handback_strikes "0" + # Will trigger handback on next cycle via FAILOVER + remote up logic + else + info "Internet back but remote still down — entering FAILOVER" + state_set state "FAILOVER" + state_set failover_start "$NOW" + state_set handback_strikes "0" + state_set tier2_started "false" + state_set tier3_started "false" + state_set tier4_started "false" + + read -r -a tier1 <<< "$(get_tier_containers 1)" + for container in "${tier1[@]}"; do + [[ -n "$container" ]] && local_start "$container" + done + fi + notify "Emerging from DARK state on $(hostname)" "Failover" "warning" + else + log "$ICON_FAILOVER DARK — no internet, no remote — waiting" + fi + fi + + # ── Sleep until next check ─────────────────────────────────────────────────────────────── + log "Next check in ${FAILOVER_CHECK_INTERVAL}s — $(date '+%H:%M:%S')" sleep "$FAILOVER_CHECK_INTERVAL" + done \ No newline at end of file diff --git a/Master.conf b/Master.conf index 2926a25..4086e5a 100644 --- a/Master.conf +++ b/Master.conf @@ -33,7 +33,7 @@ # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly -# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP +# DOCKER WATCHDOG Two-tier self-healing container monitoring # DOCKER NETWORK CONNECT Connect containers to extra networks on boot # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── @@ -53,10 +53,11 @@ # ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── # TRANSCODE MANAGER Ramdisk and SSD fallback transcode management # -# ── MONITORS ──────────────────────────────────────────────────────────────────────────────── +# ── MONITORS ─────────────────────────────────────────────────────────────────────────────── # CERTIFICATE MONITOR SSL certificate expiry monitoring # BACKUP VERIFY Random sample checksum verification against remote # SMART HEALTH Drive SMART attribute monitoring +# ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report # BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary # HEALTH DIGEST Aggregated system health digest — always/smart/weekly # EMBY SESSION REPORT Weekly Emby usage statistics via API @@ -290,76 +291,227 @@ declare -A PROFILE_SKIP_DISK_CHECK=( # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Mutual container failover between two unRAID servers 50 miles apart. +# Mutual container failover between two unRAID servers. # Each server runs Failover/failover.sh independently — no coordination between servers. -# All decisions are based solely on two ping checks: remote reachable + internet reachable. +# All decisions based solely on two pings: remote reachable + internet reachable. # -# States: -# NORMAL — remote up, internet up — own containers only, silent operation -# FAILOVER — remote down, internet up — start remote's containers locally (additive) -# NO_INTERNET — internet down — stop public-facing containers, wait for recovery -# DARK — remote down + internet down — same actions as NO_INTERNET +# ── STATES ──────────────────────────────────────────────────────────────────────────────────── +# NORMAL — remote up, internet up — own containers only, DDNS ON, silent +# FAILOVER — remote down, internet up — start remote containers (tiered by time) +# NO_INTERNET — internet down — stop own DDNS immediately, wait for recovery +# DARK — remote down AND internet down — same as NO_INTERNET # -# Handback: strike confirmation → pre-flight → rsync → start remote → stop local - +# ── DDNS RULES — ABSOLUTE ───────────────────────────────────────────────────────────────────── +# Each server owns its own DDNS — ON when that server has internet +# Script controls DDNS exclusively — network state NEVER auto-starts DDNS +# Internet loss → stop own DDNS immediately +# Failover → start remote DDNS as first action (Tier 1) +# Handback → stop remote DDNS FIRST → rsync → start local containers +# → start local DDNS LAST — only after containers confirmed up +# One DDNS per domain active at all times — never two, never zero for long +# 1 minute TTL + 1 minute check interval = minimal user impact +# +# ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── +# Strike confirmation → pre-flight → stop remote DDNS → stop remote containers +# → rsync writeback → start local containers → start local DDNS → NORMAL +# Containers only down during rsync window — minimise this time +# +# ── TIERED FAILOVER ─────────────────────────────────────────────────────────────────────────── +# Tier 1 — Immediate — vital services + Live TV — people are watching, can't wait +# Tier 2 — configurable delay — shared productivity services +# Tier 3 — configurable delay — secondary services +# Tier 4 — configurable delay — arrs + downloaders — workflow continuity +# Delays set independently per host below + EXTERNAL_IP="8.8.8.8" # external IP to ping for internet connectivity check FAILOVER_CHECK_INTERVAL=120 # seconds between state checks - FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up confirmations before handback + # 1 minute TTL + 2 minute interval = minimal gap + FAILOVER_HANDBACK_STRIKES=2 # consecutive remote-up checks before handback + # 2 strikes x 120s = 4 min confirmation window FAILOVER_STATE_FILE="/boot/config/failover_state.db" - + # persists on /boot/ — survives reboots + # tracks: state, failover_start, strikes, tier flags + # ━━━ Failover Test ━━━ -# Used by Failover/failover_test.sh — controlled simulation of the failover lifecycle. -# failover_test.sh blocks remote connectivity via iptables then observes failover.sh behavior. +# Used by Failover/failover_test.sh — controlled simulation via iptables block. # All failover logic stays in failover.sh — test script is the harness only. -# -# ⚠️ Run during a maintenance window — real containers start and stop during the test. +# ⚠️ Run during maintenance window — real containers start and stop during the test. # Use --dry-run first to walk through phases without touching anything. - - # Seconds to hold the iptables block — must be longer than FAILOVER_CHECK_INTERVAL - # so failover.sh has time to detect the outage and change state - FAILOVER_TEST_BLOCK_WAIT=150 # 150s = FAILOVER_CHECK_INTERVAL + 30s buffer - - # Seconds to wait for handback after restoring connectivity - # Must cover FAILOVER_HANDBACK_STRIKES x FAILOVER_CHECK_INTERVAL plus rsync time - # 2 strikes x 120s = 240s minimum — add buffer for rsync handback jobs - FAILOVER_TEST_HANDBACK_WAIT=360 # 360s = 6 minutes — adjust if rsync takes longer - - -FAILOVER_HOST1_STARTS_FOR_HOST2=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" + FAILOVER_TEST_BLOCK_WAIT=150 # seconds to hold iptables block + # must be > FAILOVER_CHECK_INTERVAL + buffer + FAILOVER_TEST_HANDBACK_WAIT=360 # seconds to wait for handback completion + # covers FAILOVER_HANDBACK_STRIKES x INTERVAL + rsync + +# ━━━ DDNS — Script Controlled Exclusively ━━━ +# Each server owns its own DDNS containers — one domain per server. +# DDNS is started and stopped ONLY by this script — never by network state returning. +# HOST1 DDNS starts last in handback (after containers confirmed up). +# HOST1 DDNS stops first on internet loss. +# HOST2 DDNS starts when HOST2 detects HOST1 is down (Tier 1). +# HOST2 DDNS stops before handback rsync begins. + +HOST1_DDNS_CONTAINERS=( + "Gmer4Lfe.com" # HOST1's own DDNS — ON when HOST1 has internet + # covers gmer4lfe.com pointing to HOST1 IP ) + +HOST2_DDNS_CONTAINERS=( + "Gmer4Lfe.us" # HOST2's own DDNS — ON when HOST2 has internet + # covers gmer4lfe.us pointing to HOST2 IP +) + +# ━━━ Containers to stop on internet loss ━━━ +# Own DDNS handled separately above — list additional containers here if needed +# These stop when this server loses internet — regardless of remote state FAILOVER_HOST1_STOP_ON_NO_NET=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" -) -FAILOVER_HOST1_RSYNC_JOBS=( -# "/mnt/user/appdata-Failover/Jayred365" -# "/mnt/user/Media_Server/Emby-Jayred" -) - -# HOST2 (unRAID-Jayred365 — Secondary) -FAILOVER_HOST2_STARTS_FOR_HOST1=( - "Emby" - "Gmer4Lfe.com" - "Gmer4Lfe.us" + # "container-name" # add containers that should stop without internet ) + FAILOVER_HOST2_STOP_ON_NO_NET=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" + # "container-name" ) -FAILOVER_HOST2_RSYNC_JOBS=( -# "/mnt/user/appdata-Failover/Gmer4Lfe" + +# ━━━ HOST1 runs these for HOST2 when HOST2 goes down ━━━ +# HOST2's DDNS listed in Tier 1 — starts immediately as first action +# List HOST2's specific services here — HOST2's own containers only +# Do NOT list shared services that HOST1 already runs + +# Tier 1 — Immediate — starts as soon as HOST2 is detected down +FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( + "Gmer4Lfe.us" # HOST2's DDNS — start first, covers HOST2's domain + "VaultWarden-Jayred365" # HOST2's password manager — immediate access needed + # "container-placeholder" # add HOST2 specific services here ) - + +# Tier 2 — starts after HOST2_TIER2_DELAY minutes +FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( + # "container-placeholder" +) + +# Tier 3 — starts after HOST2_TIER3_DELAY minutes +FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( + # "container-placeholder" +) + +# Tier 4 — starts after HOST2_TIER4_DELAY minutes +FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( + # "container-placeholder" +) + +# ━━━ HOST2 runs these for HOST1 when HOST1 goes down ━━━ +# HOST1's DDNS listed in Tier 1 — starts immediately to cover HOST1's domain +# Live TV in Tier 1 — people are watching, cannot wait for tiered startup +# Auth stack in Tier 1 — everything proxied through NPM needs auth + +# Tier 1 — Immediate — vital services and Live TV cannot wait +FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( + "Gmer4Lfe.com" # HOST1's DDNS — start first, covers HOST1's domain + "Emby" # media server — users are actively watching + "NginxProxyManager" # reverse proxy — all services route through this + "Lldap-Gmer4Lfe" # auth directory — required by Authelia + "Mariadb-Authelia" # auth database — required by Authelia + "Redis-Authelia" # auth cache — required by Authelia + "Authelia" # authentication — required for all proxied services + "Authelia-Secondary" # auth redundancy + "Redis-Authelia-Secondary" # auth secondary cache + "VaultWarden-Gmer4Lfe" # password manager — critical, immediate access needed + "Dispatcharr" # Live TV — people are watching, cannot wait + "Dispatcharr-Basic" # Live TV basic profile + "Dispatcharr-Iptv-Users" # Live TV IPTV users + "ErsatzTV-Emby" # Live TV scheduling and channel management +) + +# Tier 2 — starts after HOST1_TIER2_DELAY minutes +# Productivity services — important but can wait a couple of hours +FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( + "Postgres-NextCloud" # NextCloud database — must start before NextCloud + "NextCloud" # file access and collaboration + "PostgreSQL_Immich" # Immich database + "Immich-Gmer4Lfe" # photo management + "Jellyseerr" # media request management + # "container-placeholder" +) + +# Tier 3 — starts after HOST1_TIER3_DELAY minutes +# Secondary services — useful but not immediately critical +FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( + "Organizrv2-Gmer4Lfe" # dashboard — nice to have + "AdGuard-Home" # DNS filtering + "UptimeKuma-Gmer4Lfe" # uptime monitoring + "Gitea" # git server + "Collabora-CODE" # document editing for NextCloud + # "container-placeholder" +) + +# Tier 4 — starts after HOST1_TIER4_DELAY minutes +# Full workflow mode — arrs and downloaders +# Minimal writeback on handback — start fresh is cleaner than syncing download state +FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( + "Sonarr" # TV show management + "Radarr" # movie management + "Lidarr" # music management + "Readarr" # book management + "Prowlarr" # indexer management + "Bazarr" # subtitle management + "SABnzbd-Gmer4Lfe" # usenet downloader + "Qbittorrent-Gmer4Lfe" # torrent downloader + "LidaTube" # YouTube music downloader + "Pinchflat" # YouTube channel downloader + "ChannelTube" # YouTube channel management + # "container-placeholder" +) + +# ━━━ Tier Delay Settings ━━━ +# How long primary must be down before each tier activates — set in minutes +# Tier 1 is always immediate — no delay +# Set independently per host — a large server may want longer delays than a small one +# Adjust based on your tolerance for resource usage on the covering server + +# Delays for HOST1's containers running on HOST2 (HOST1 is down) +HOST1_TIER2_DELAY=120 # 2 hours — NextCloud, Immich can wait +HOST1_TIER3_DELAY=360 # 6 hours — dashboard, monitoring, Gitea +HOST1_TIER4_DELAY=1080 # 18 hours — full workflow, arrs and downloaders + +# Delays for HOST2's containers running on HOST1 (HOST2 is down) +HOST2_TIER2_DELAY=120 +HOST2_TIER3_DELAY=360 +HOST2_TIER4_DELAY=1080 + +# ━━━ Rsync Writeback Jobs ━━━ +# Run during handback — syncs critical appdata back to primary before containers restart. +# Containers are stopped before this runs — clean source, no competing writes. +# Full bandwidth available — DDNS stopped, containers stopped, nothing competing. +# +# Priority: +# Critical — Emby userdata/playstates (small, fast, important) +# Critical — Auth stack data +# Skip — Media files (already on primary, never moved) +# Skip — Downloads (start fresh — cleaner than syncing partial state) +# +# Format: "/path/to/source" — matched to rsync profile by directory basename + +# HOST1 writeback — run by HOST2 during HOST1 handback +FAILOVER_HOST1_WRITEBACK=( + "/mnt/user/appdata-Failover/Critical-Data" # auth stack — Authelia, Mariadb, Redis, LLDAP, NPM + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres + "/mnt/user/appdata-Failover/Emby" # Emby userdata, playstates, metadata + "/mnt/user/appdata-Failover/Gmer4Lfe" # server specific appdata — Organizr, UptimeKuma +# "/mnt/user/appdata-Failover/Arrs_Stack" # skip — arrs start fresh on handback +) + +# HOST2 writeback — run by HOST1 during HOST2 handback +FAILOVER_HOST2_WRITEBACK=( + # "/mnt/user/appdata-Failover/Jayred365" # HOST2 specific appdata + # "container-placeholder" +) + # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Docker Daily Restart ━━━ -# Containers restarted every day by Docker_Essentials/docker_daily_restart.sh. -# Keeps services fresh and clears memory leaks that accumulate over time. -# Case-sensitive — must match exact Docker container names shown in the unRAID Docker tab. +# Containers restarted every day — keeps services fresh, clears memory leaks. +# Case-sensitive — must match exact Docker container names in the unRAID Docker tab. DAILY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" @@ -370,8 +522,8 @@ DAILY_RESTART_CONTAINERS=( ) # ━━━ Docker Weekly Restart ━━━ -# Containers restarted once per week by Docker_Essentials/docker_weekly_restart.sh. -# For less critical services that benefit from periodic restart but don't need daily cycling. +# Less critical services that benefit from periodic restart but don't need daily cycling. +# Case-sensitive — must match exact Docker container names in the unRAID Docker tab. WEEKLY_RESTART_CONTAINERS=( "NextCloud" "Organizrv2-Gmer4Lfe" @@ -380,20 +532,18 @@ WEEKLY_RESTART_CONTAINERS=( ) # ━━━ Docker Watchdog ━━━ -# Two-tier self-healing container monitoring: -# Tier 1 — strict monitoring of explicitly configured containers -# Tier 2 — global health scan of ALL running containers +# Two-tier self-healing container monitoring. +# Tier 1 — strict monitoring of explicitly configured containers +# Tier 2 — global health scan of ALL running containers # -# Cross-cutting intelligence applies to both tiers: +# Cross-cutting intelligence: # Startup grace — skip restarts while system is still booting -# Dependency order — restart database before app, not the other way around +# Dependency order — restart database before app # Restart loop — stop restarting after limit hit → skip list → notify critical # Skip list — persistent across reboots, auto-clears when container recovers -# Batch notify — one clean summary per run instead of one ping per event - -# ── Tier 1 — Strict Monitoring ──────────────────────────────────────────────────────────── - -# Memory hard limits in MB — immediate restart if exceeded, no strike system +# Batch notify — one clean summary per run + +# Memory hard limits in MB — immediate restart if exceeded # 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( @@ -402,14 +552,14 @@ declare -A WATCHDOG_CONTAINERS=( ["Tdarr"]=6144 ["Code-Server"]=1024 ) - -# HTTP responsiveness checks — omit container to skip its HTTP check + declare -A WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ) - + # Containers that must always be running — strike system, persistent skip list on /boot/ # Skip list auto-clears when container recovers — no manual intervention for normal recovery +# These are your core auth and proxy stack — everything depends on them being up WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" "Lldap-Gmer4Lfe" @@ -419,81 +569,74 @@ WATCHDOG_REQUIRED_CONTAINERS=( "Authelia-Secondary" "Redis-Authelia-Secondary" ) - -# Strike thresholds + +# Strike state file — /tmp resets on reboot which is correct for strike tracking WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive CPU strikes before restart - SOFT_MEM_THRESHOLD=80 # warn when container reaches this % of hard limit - RESP_FAIL_LIMIT=2 # consecutive failed HTTP checks before restart - CURL_TIMEOUT=5 # seconds before curl gives up per check - -# ── Tier 2 — Global Health Scan ─────────────────────────────────────────────────────────── - -# Master toggle — false disables Tier 2 entirely + +# CPU thresholds — normalised against total core count automatically at runtime + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive CPU strikes before container restart + +# Memory soft threshold — warn when container reaches this % of its hard limit +# Hard limit exceeded triggers immediate restart regardless of strikes + SOFT_MEM_THRESHOLD=80 + +# HTTP responsiveness check settings + RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart + CURL_TIMEOUT=5 # seconds before curl gives up per check + +# Tier 2 master toggle — false disables global scan entirely WATCHDOG_SCAN_ALL=true - -# Containers to skip in Tier 2 — add containers expected to be in a non-running state -# or managed by other systems that should not be auto-restarted + +# Containers to skip in Tier 2 scan entirely +# Add intentionally stopped containers or containers managed by other systems WATCHDOG_SCAN_IGNORE=( # "container-name" ) - + # Individual Tier 2 check toggles — disable checks that cause false positives WATCHDOG_RESTART_UNHEALTHY=true # restart containers with unhealthy Docker health status WATCHDOG_RESTART_DEAD=true # remove and restart containers in dead state WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero exit code WATCHDOG_NOTIFY_OOM=true # restart and notify when OOM killed by kernel WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker restart count is climbing - + # Crash loop threshold — notify critical if Docker has restarted container this many times WATCHDOG_CRASH_LIMIT=5 - -# ── Cross-cutting Intelligence ──────────────────────────────────────────────────────────── - -# Startup grace period — skip restarts while system is still booting + +# Startup grace — skip restarts while system is still booting # Prevents false positives while containers are coming up after array start - WATCHDOG_STARTUP_GRACE=300 # seconds after boot before watchdog acts on failures - + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures + # Restart loop protection — stops hammering broken containers -# Tracks watchdog-initiated restarts per container in a bounded /boot/ file -# After limit hit → container added to skip list → notify critical → manual intervention +# After limit hit → skip list → notify critical → manual intervention needed # Skip list auto-clears when container is found running again WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window WATCHDOG_CONTAINER_RESTART_WINDOW=1 # hours — rolling window for restart count WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" - # /boot/ survives reboots — bounded, auto-purges old entries - + # /boot/ survives reboots — bounded, auto-purges + # Dependency ordering — skip restarting a container if its dependency is also down # Dependency gets restarted first, dependent picked up on the next watchdog cycle +# Prevents Authelia restarting before its database is ready — it would just fail again # Format: ["dependent"]="dependency1 dependency2" declare -A WATCHDOG_DEPENDENCIES=( ["Authelia"]="Mariadb-Authelia Redis-Authelia" ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" ["NextCloud"]="Postgres-NextCloud" ) - + # Notification batching — one clean summary per run instead of one ping per event # true = batch all events into a single notification at end of run # false = send individual notification per event as it happens WATCHDOG_BATCH_NOTIFY=true - -# ━━━ Docker Network Connect ━━━ -NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" -) -NETWORK_CONNECT_NETWORKS=( - "nextcloud-aio" -) # ━━━ Docker Network Connect ━━━ -# Connects containers to extra Docker networks on array start. +# Connects containers to extra Docker networks on array start — many-to-many. +# Every container in the list connects to every network in the list. # Useful when containers need to communicate across networks they were not originally # configured with — e.g. memcached needing access to the nextcloud-aio network. -# Every container in the list connects to every network in the list (many-to-many). -# Comment out entries to disable without removing them. NETWORK_CONNECT_CONTAINERS=( "memcached" "Npm-CrowdSec" @@ -508,42 +651,37 @@ NETWORK_CONNECT_NETWORKS=( # ============================================================================================== # ━━━ Reboot ━━━ -# Seconds of warning broadcast to all logged-in users before server_reboot.sh reboots. -# Gives users time to save work or finish what they are doing before the system goes down. +# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. +# Gives users time to save work before the system goes down. REBOOT_SLEEP=300 # ━━━ Mover ━━━ -# Seconds to wait after warning users before mover_stop.sh sends SIGTERM to the mover. +# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. # Gives the mover time to finish its current file operation cleanly before being killed. MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ -# Path for the rsyslog filter file created by docker_syslog_filter.sh. -# The filter suppresses noisy Docker veth and docker0 messages from syslog on boot. -# Without this filter, every Docker network interface change floods the syslog. +# Path for the rsyslog filter file that suppresses Docker veth noise from syslog. +# Without this filter every Docker network interface change floods the syslog on boot. FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ -# Config file path and max children value for php_fpm_max_children.sh. # Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. # Set based on available RAM — too high can cause memory pressure on low-RAM systems. PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ -# System log files cleared by clear_logs.sh — Docker container logs are cleared too. -# Run weekly to prevent logs from filling the rootfs over time. +# System log files cleared weekly to prevent rootfs fill over time. LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # ━━━ WebGUI Watchdog ━━━ -# Monitors the unRAID WebGUI and restarts services if it becomes unresponsive. -# Escalation path: nginx restart → recheck → emhttp restart → recheck → notify warning. -# emhttp is the core unRAID management daemon — restarting it is more disruptive than nginx -# but both recover cleanly. Notification sent on any restart so you know what happened. - WEBGUI_URL="http://localhost" # adjust port if non-standard e.g. http://localhost:8080 +# Escalation: nginx restart → recheck → emhttp restart → recheck → notify warning. +# emhttp is the core unRAID daemon — restarting is more disruptive but recovers cleanly. + WEBGUI_URL="http://localhost" # adjust if running non-standard port WEBGUI_TIMEOUT=5 # seconds before curl gives up on the WebGUI check WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before rechecking - WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart — emhttp takes longer + WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart — takes longer # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── @@ -553,11 +691,11 @@ NETWORK_CONNECT_NETWORKS=( # Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES. # Run by Media/media_shares_permissions.sh via the media_management.sh orchestrator. # 777 and nobody:users is standard for unRAID media shares accessible by Docker containers. +# Applied recursively so large shares take time — run overnight via orchestrator. PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" # Shares to apply permissions to — add or remove paths as your library grows. -# These are applied recursively so large shares take time — run overnight via orchestrator. MEDIA_PERMISSION_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -616,7 +754,8 @@ ANIME_FILE_PATTERNS=( '*.log' '*.json' ) -# File patterns deleted by the media profile — includes *.iso and *.lrc not needed in anime +# File patterns deleted by the media profile +# Includes *.iso and *.lrc not needed in anime profile MEDIA_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -660,7 +799,6 @@ MEDIA_MAINTENANCE_JOBS=( LIDARR_MUSIC_ROOT="/mnt/user/Music-New" # must match the root path set in Lidarr LIDARR_ORPHAN_AGE=7 # days before untracked file is eligible for deletion LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") - # file extensions considered valid music files LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") # never deleted — cover art, metadata, lyrics @@ -684,43 +822,32 @@ MEDIA_MAINTENANCE_JOBS=( # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Session-based storage allocator using filesystem symlink indirection. -# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# Only new sessions care about where the symlink currently points. -# -# How it works: -# ramdisk_setup.sh — run once at array start, creates tmpfs and sets symlink -# transcode_manager.sh — every 3 min, monitors usage and manages symlink direction -# transcode_cleanup.sh — every 5 min, removes old inactive files from both locations -# -# ⚠️ Docker mount warning: -# Do NOT add a static SSD transcode path as a second volume mount in your Emby container. -# If the SSD path is mounted inside the container Emby can see it and will use it -# independently of the symlink — breaking symlink-based routing entirely. -# The symlink IS your emergency lever — one mount only: -# /mnt/ram-transcode → /ext-ram-transcode - +# ⚠️ One mount only in Emby container: /mnt/ram-transcode → /ext-ram-transcode +# Do NOT add a static SSD path — Emby will use it independently of the symlink. +# The symlink IS your emergency lever — flip it manually if needed: +# ln -sfn /mnt/cache/Temp_Storage/Emby/Transcodes /mnt/ram-transcode + RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point created at array start RAMDISK_SIZE="8G" # ceiling — tmpfs only uses RAM actually needed TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — location never changes TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback location - -# Usage thresholds in GB — hysteresis gap prevents flip-flop near threshold + +# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD required before allowing flip to SSD - + # Cleanup age thresholds — files must be older than these AND not open by any process TRANSCODE_MAX_AGE=20 # minutes before a transcode file is eligible for cleanup - TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible - -# Flip frequency alert — too many flips per hour may indicate ramdisk needs to be larger + TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution buffer + +# Flip frequency alert — too many flips per hour indicates ramdisk needs to be larger TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour - + # Permissions — must match your Emby container user TRANSCODE_OWNER="nobody:users" TRANSCODE_CHMOD="755" # renamed from TRANSCODE_MODE to avoid ambiguity with manager mode - + # Operating mode — controls symlink routing behavior # smart — auto-flips between ramdisk and SSD based on usage thresholds (default) # ramdisk — always uses ramdisk, never flips to SSD regardless of usage @@ -729,28 +856,23 @@ MEDIA_MAINTENANCE_JOBS=( # ssd — always uses SSD, never uses ramdisk # useful during ramdisk maintenance, testing, or after a flip issue # switch to this mode to drain ramdisk sessions gracefully - TRANSCODE_MANAGER_MODE="smart" - + TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + # Emby container check — skips threshold checks when Emby is not running # Prevents unnecessary symlink flips when no transcoding is happening TRANSCODE_CHECK_EMBY=true - TRANSCODE_EMBY_CONTAINER="Emby" # exact Docker container name — case sensitive - + TRANSCODE_EMBY_CONTAINER="Emby" # exact Docker container name — case sensitive + # ============================================================================================== -# ── MONITORS ─────────────────────────────────────────────────────────────────────────────────── +# ── MONITORS ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Monitoring scripts — watch and report only, never take action. -# Lives in Monitor/ folder — distinct from unRAID_Essentials (which acts) and -# Docker_Essentials (which manages containers). -# These scripts are the data sources for the future plugin dashboard. # ━━━ Certificate Monitor ━━━ # Checks SSL cert expiry via direct openssl connection — no NPM dependency. # Reads the actual cert the server is presenting — catches real-world issues API checks miss. # Each domain and subdomain is a separate entry — they have independent certs. -# Add your public-facing domains — uncomment and replace with your actual domains. CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" + "Gmer4Lfe.com" "Gmer4Lfe.us" ) CERT_WARN_DAYS=30 # notify warning when cert expires within this many days @@ -763,9 +885,6 @@ CERT_MONITOR_DOMAINS=( # Leave BACKUP_VERIFY_SHARES empty to automatically use DAILY_SYNC_SHARES as the target list. BACKUP_VERIFY_SHARES=( # leave empty to use DAILY_SYNC_SHARES automatically - # or specify individual shares to verify: - # /mnt/user/Movies - # /mnt/user/Tv_Shows ) BACKUP_VERIFY_SAMPLE=10 # number of files to randomly sample per share per run BACKUP_VERIFY_MIN_SIZE=1M # skip files smaller than this — avoids tiny junk files @@ -777,49 +896,45 @@ BACKUP_VERIFY_SHARES=( SMART_TEMP_WARN=45 # degrees C — warn if drive temperature exceeds this SMART_TEMP_CRIT=55 # degrees C — critical if drive temperature exceeds this SMART_IGNORE_DRIVES=( - "sda" # uncomment to ignore sda — common choice if sda is your unRAID boot USB + "sda" # boot USB — SMART not meaningful on flash drives ) -# ━━━ Bandwidth Monitor ━━━ -# Logs daily rsync transfer totals to a bounded file on /boot/ — minimal flash wear. -# bandwidth_monitor.sh --log-transfer is called by rsync.sh after each successful sync. -# bandwidth_monitor.sh --report generates the weekly summary standalone. -# File stays bounded to BANDWIDTH_LOG_RETENTION lines — old entries auto-purged on each write. - BANDWIDTH_LOG="/boot/config/bandwidth_history.db" - BANDWIDTH_LOG_RETENTION=90 # days to keep — file never grows beyond ~90 lines - BANDWIDTH_WARN_GB=50 # flag in reports if a single sync transfer exceeds this GB - - # ━━━ ZFS Memory Snapshot ━━━ +# ━━━ ZFS Memory Snapshot ━━━ +# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. +# system_watchdog.sh handles threshold-based intervention. +# Output written to ZFS_REPORT_LOG for historical review in addition to console output. +# Pools in ZFS_REPORT_IGNORE_POOLS are excluded from reporting — still monitored by unRAID. ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" - ZFS_REPORT_ARC_WARN_PCT=90 - ZFS_REPORT_FREE_WARN_GB=10 - ZFS_REPORT_AVAIL_WARN_GB=20 - ZFS_REPORT_DOCKER_TOP=10 - -# Pools to exclude from health reporting — still monitored by unRAID but skipped in report -# Useful for pools that are expected to be heavily used or are managed separately + ZFS_REPORT_ARC_WARN_PCT=90 # warn in report if ARC utilization above this % + ZFS_REPORT_FREE_WARN_GB=10 # warn in report if free RAM drops below this GB + ZFS_REPORT_AVAIL_WARN_GB=20 # warn in report if available RAM drops below this GB + ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show in report ZFS_REPORT_IGNORE_POOLS=( - "disk10" # Docker overlay storage — high usage is normal - "disk9" # Cache pool — usage varies widely, not meaningful to report + # Pools excluded from health reporting — expected to run at high usage + # All pools still monitored by unRAID regardless of this list + "disk10" + "disk9" "disk8" "disk6" "disk5" ) +# ━━━ Bandwidth Monitor ━━━ +# Called by rsync.sh after each sync — one bounded write per run, minimal flash wear. +# Log format: YYYY-MM-DD|HH:MM|profile|duration_seconds|status — version-proof +# File stays bounded to BANDWIDTH_LOG_RETENTION days — old entries auto-purged on write. + BANDWIDTH_LOG="/boot/config/bandwidth_history.db" + BANDWIDTH_LOG_RETENTION=90 # days to keep — file never grows beyond ~90 lines + BANDWIDTH_WARN_GB=50 # flag in reports if a single sync transfer exceeds this GB + # ━━━ Health Digest ━━━ -# Aggregated system health summary from across the ecosystem. -# Reads existing state files — no new writes to flash drive. -# +# Aggregated system health summary — reads existing state files, no new writes to flash. # Three profiles — switch by changing DIGEST_PROFILE, no cron changes needed: -# always — sends every run (schedule daily = daily digest, weekly = weekly digest) -# smart — sends only if findings worth reporting (intelligent quiet operation) -# weekly — sends once per week on DIGEST_DAY only, silent all other days -# -# Data sources (reads only — no writes): -# Transcode ramdisk state, container watchdog strikes, system watchdog strikes, -# failover state, container skip list, bandwidth history, SSL cert days remaining +# always — sends every run +# smart — sends only if findings worth reporting +# weekly — sends once per week on DIGEST_DAY only DIGEST_PROFILE="weekly" # always | smart | weekly - DIGEST_DAY="Sunday" # day name for weekly profile — must match date +%A output + DIGEST_DAY="Sunday" # must match date +%A output # Smart profile triggers — set true to send digest when this condition is found DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active @@ -828,29 +943,19 @@ ZFS_REPORT_IGNORE_POOLS=( DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB # ━━━ Emby Session Report ━━━ -# Weekly Emby usage report via API — no persistent writes, queries fresh each run. -# Shows active streams, library counts, transcode vs direct play ratio. -# Requires an API key from Emby Settings → API Keys in the Emby WebUI. +# Requires API key from Emby Settings → API Keys in the Emby WebUI. +# No persistent writes — queries fresh each run. EMBY_URL="http://localhost:8096" - EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" # paste your Emby API key here + EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" EMBY_REPORT_DAYS=7 # number of days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show in report # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Last line of defense — reboots the system cleanly when it is about to become unstable. -# Runs every 15 minutes via cron. Works alongside docker_watchdog.sh: -# docker_watchdog.sh — container level, minimal disruption, tries to self-heal first -# system_watchdog.sh — system level, last resort, reboots when healing has failed -# -# Strike system: sustained threshold hits trigger reboot — single spikes are ignored. -# Each check that exceeds its threshold adds a strike. Strikes reset when recovered. -# When strike limit is hit the reboot sequence begins. -# -# Reboot loop protection: tracks reboot timestamps on /boot/ (survives reboots). -# If the server reboots too many times in the window it shuts down instead — a reboot -# loop means something is fundamentally wrong that a reboot is not fixing. +# Last line of defense — reboots cleanly when system is about to become unstable. +# Strike system: sustained threshold hits trigger reboot — single spikes ignored. +# Reboot loop protection: shuts down instead if reboot limit hit in window. # ━━━ State Files ━━━ # Strike counts reset on reboot — /tmp is correct (fresh start after each reboot)