diff --git a/Failover/README-Failover.md b/Fallback/README-Failover.md similarity index 100% rename from Failover/README-Failover.md rename to Fallback/README-Failover.md diff --git a/Failover/failover.sh b/Fallback/fallback.sh similarity index 86% rename from Failover/failover.sh rename to Fallback/fallback.sh index 3685273..2d514d8 100644 --- a/Failover/failover.sh +++ b/Fallback/fallback.sh @@ -8,7 +8,7 @@ # # ── HOW IT WORKS ────────────────────────────────────────────────────────────────────────────── # Both servers run this script continuously as a background task via User Scripts. -# Every FAILOVER_CHECK_INTERVAL seconds each server: +# Every FALLBACK_CHECK_INTERVAL seconds each server: # 1. Pings the remote server # 2. Pings the internet # 3. Determines its current state @@ -44,7 +44,7 @@ # # ── HANDBACK SEQUENCE ───────────────────────────────────────────────────────────────────────── # When remote returns after FAILOVER: -# 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks +# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks # 2. Pre-flight checks — version parity, remote array, remote Docker daemon # 3. Stop remote DDNS first — prevents split brain DNS during rsync # 4. Stop remote containers — clean state before rsync @@ -61,7 +61,7 @@ # Tier delays configurable per host in master_host*.conf # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# FAILOVER_ENABLED gate — exits cleanly if disabled in master.conf +# FALLBACK_ENABLED gate — exits cleanly if disabled in master.conf # Version parity check — refuses handback if unRAID versions mismatch # Remote Docker daemon — checks remote daemon before issuing any remote commands # Timeout protection — all docker and SSH commands wrapped in timeouts @@ -79,18 +79,18 @@ # FAILOVER_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# FAILOVER_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.) -# FAILOVER_CHECK_INTERVAL — seconds between checks -# FAILOVER_HANDBACK_STRIKES — consecutive remote-up checks before handback -# FAILOVER_STATE_FILE — /boot/config path — survives reboots -# FAILOVER_RSYNC_ENABLED — gate for writeback rsync jobs +# FALLBACK_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.) +# FALLBACK_CHECK_INTERVAL — seconds between checks +# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback +# FALLBACK_STATE_FILE — /boot/config path — survives reboots +# FALLBACK_RSYNC_ENABLED — gate for writeback rsync jobs # EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8) # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# failover.sh — normal start (continuous loop) -# failover.sh --dry-run — preview state changes without acting on containers -# failover.sh --status — show current state and exit -# failover.sh --log — verbose cycle output +# fallback.sh — normal start (continuous loop) +# fallback.sh --dry-run — preview state changes without acting on containers +# fallback.sh --status — show current state and exit +# fallback.sh --log — verbose cycle output # # ── TO STOP THIS SCRIPT ─────────────────────────────────────────────────────────────────────── # Click Abort in unRAID User Scripts — do NOT kill directly, state file may corrupt. @@ -113,10 +113,10 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -# FAILOVER_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt) -if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then - warn "FAILOVER_ENABLED=false — failover monitoring disabled" - warn "Set FAILOVER_ENABLED=true in master.conf when both servers are ready" +# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt) +if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then + warn "FALLBACK_ENABLED=false — fallback monitoring disabled" + warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready" exit 0 fi @@ -143,26 +143,26 @@ CONTAINER_VERIFY_WAIT=5 # seconds after start before verifying container is up # ============================================================================================== # State file on /boot/config — survives reboots. # Format: key=value one per line. -# Keys: state, failover_start, handback_strikes, tier2_started, tier3_started, tier4_started +# Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started state_get() { - grep "^${1}=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2 + grep "^${1}=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2 } state_set() { local key="$1" value="$2" - if grep -q "^${key}=" "$FAILOVER_STATE_FILE" 2>/dev/null; then - sed -i "s|^${key}=.*|${key}=${value}|" "$FAILOVER_STATE_FILE" + if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then + sed -i "s|^${key}=.*|${key}=${value}|" "$FALLBACK_STATE_FILE" else - echo "${key}=${value}" >> "$FAILOVER_STATE_FILE" + echo "${key}=${value}" >> "$FALLBACK_STATE_FILE" fi } state_init() { - mkdir -p "$(dirname "$FAILOVER_STATE_FILE")" - [[ ! -f "$FAILOVER_STATE_FILE" ]] && touch "$FAILOVER_STATE_FILE" + mkdir -p "$(dirname "$FALLBACK_STATE_FILE")" + [[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE" [[ -z "$(state_get state)" ]] && state_set state "NORMAL" - [[ -z "$(state_get failover_start)" ]] && state_set failover_start "0" + [[ -z "$(state_get fallback_start)" ]] && state_set fallback_start "0" [[ -z "$(state_get handback_strikes)" ]] && state_set handback_strikes "0" [[ -z "$(state_get tier2_started)" ]] && state_set tier2_started "false" [[ -z "$(state_get tier3_started)" ]] && state_set tier3_started "false" @@ -202,14 +202,14 @@ local_start() { return 0 else error "$container started but crashed immediately" - notify "$container failed to stay running during failover on $(hostname)" \ - "Failover" "warning" + notify "$container failed to stay running during fallback on $(hostname)" \ + "Fallback" "warning" return 1 fi else error "Failed to start $container locally" - notify "Failed to start $container locally during failover on $(hostname)" \ - "Failover" "warning" + notify "Failed to start $container locally during fallback on $(hostname)" \ + "Fallback" "warning" return 1 fi } @@ -268,13 +268,13 @@ remote_start() { else error "$container started on $REMOTE_SERVER_NAME but crashed immediately" notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \ - "Failover" "warning" + "Fallback" "warning" return 1 fi else error "Failed to start $container on $REMOTE_SERVER_NAME" notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \ - "Failover" "warning" + "Fallback" "warning" return 1 fi } @@ -341,7 +341,7 @@ remote_ddns_stop() { get_tier_containers() { local tier="$1" local remote_id="${REMOTE_ID}" - local var_name="FAILOVER_${MY_ID}_RUNS_FOR_${remote_id}_TIER${tier}" + local var_name="FALLBACK_${MY_ID}_COVERS_${remote_id}_TIER${tier}" eval "echo \"\${${var_name}[@]:-}\"" } @@ -371,10 +371,10 @@ get_writeback_jobs_for_tier() { if [[ "$tier" -eq 4 ]]; then # Tier 4 — daily sync shares of the remote + any extra writeback paths local shares_var="${remote_id}_DAILY_SYNC_SHARES" - local extra_var="FAILOVER_${remote_id}_WRITEBACK_TIER4" + local extra_var="FALLBACK_${remote_id}_WRITEBACK_TIER4" eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\"" else - local var_name="FAILOVER_${remote_id}_WRITEBACK_TIER${tier}" + local var_name="FALLBACK_${remote_id}_WRITEBACK_TIER${tier}" eval "echo \"\${${var_name}[@]:-}\"" fi } @@ -389,7 +389,7 @@ set_ddns_arrays() { # Select internet-loss stop list based on MY_ID get_stop_on_no_net() { - local var_name="FAILOVER_${MY_ID}_STOP_ON_NO_NET" + local var_name="FALLBACK_${MY_ID}_STOP_ON_NO_NET" eval "echo \"\${${var_name}[@]:-}\"" } @@ -400,7 +400,7 @@ if [[ "$SHOW_STATUS" == true ]]; then state_init set_ddns_arrays CURRENT_STATE=$(state_get state) - FAILOVER_START_TS=$(state_get failover_start) + FALLBACK_START_TS=$(state_get fallback_start) TIER2=$(state_get tier2_started) TIER3=$(state_get tier3_started) TIER4=$(state_get tier4_started) @@ -409,25 +409,25 @@ if [[ "$SHOW_STATUS" == true ]]; then local_ver=$(grep -oP '(?<=version=")[^"]+' /etc/unraid-version 2>/dev/null || echo "unknown") echo "" - echo "━━━━━ $ICON_SUMMARY FAILOVER STATUS ━━━━━" + echo "━━━━━ $ICON_SUMMARY FALLBACK STATUS ━━━━━" echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" echo "$ICON_GEAR unRAID ver: $local_ver" - echo "$ICON_FAILOVER State: $CURRENT_STATE" + echo "$ICON_FALLBACK State: $CURRENT_STATE" echo "$ICON_NET Local DDNS: ${LOCAL_DDNS_CONTAINERS[*]:-none}" echo "$ICON_NET Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}" - echo "$ICON_TIME Interval: ${FAILOVER_CHECK_INTERVAL}s" - echo "$ICON_FAILOVER Strikes: $STRIKES / $FAILOVER_HANDBACK_STRIKES" + echo "$ICON_TIME Interval: ${FALLBACK_CHECK_INTERVAL}s" + echo "$ICON_FALLBACK Strikes: $STRIKES / $FALLBACK_HANDBACK_STRIKES" - if [[ "$CURRENT_STATE" == "FAILOVER" && "$FAILOVER_START_TS" -gt 0 ]]; then - ELAPSED=$(( ($(date +%s) - FAILOVER_START_TS) / 60 )) + if [[ "$CURRENT_STATE" == "FALLBACK" && "$FALLBACK_START_TS" -gt 0 ]]; then + ELAPSED=$(( ($(date +%s) - FALLBACK_START_TS) / 60 )) echo "$ICON_TIME Outage: ${ELAPSED}min" - echo "$ICON_FAILOVER Tier 2: $TIER2 (delay: $(get_tier2_delay)min)" - echo "$ICON_FAILOVER Tier 3: $TIER3 (delay: $(get_tier3_delay)min)" - echo "$ICON_FAILOVER Tier 4: $TIER4 (delay: $(get_tier4_delay)min)" + echo "$ICON_FALLBACK Tier 2: $TIER2 (delay: $(get_tier2_delay)min)" + echo "$ICON_FALLBACK Tier 3: $TIER3 (delay: $(get_tier3_delay)min)" + echo "$ICON_FALLBACK Tier 4: $TIER4 (delay: $(get_tier4_delay)min)" fi - echo "$ICON_GEAR Enabled: $FAILOVER_ENABLED" + echo "$ICON_GEAR Enabled: $FALLBACK_ENABLED" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 @@ -442,7 +442,7 @@ fi run_handback() { echo "" - echo "━━━ $ICON_FAILOVER Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━" warn "$REMOTE_SERVER_NAME has returned — beginning handback" # ── Step 1: Pre-flight checks ──────────────────────────────────────────────────────────── @@ -484,20 +484,20 @@ run_handback() { echo "" echo "━━━ $ICON_STOP Stop Remote Containers ━━━" - local ALL_FAILOVER_CONTAINERS=() + local ALL_FALLBACK_CONTAINERS=() # Collect all started tiers in reverse order (highest tier stops first) if [[ "$(state_get tier4_started)" == "true" ]]; then read -r -a t4 <<< "$(get_tier_containers 4)" - ALL_FAILOVER_CONTAINERS+=("${t4[@]}") + ALL_FALLBACK_CONTAINERS+=("${t4[@]}") fi if [[ "$(state_get tier3_started)" == "true" ]]; then read -r -a t3 <<< "$(get_tier_containers 3)" - ALL_FAILOVER_CONTAINERS+=("${t3[@]}") + ALL_FALLBACK_CONTAINERS+=("${t3[@]}") fi if [[ "$(state_get tier2_started)" == "true" ]]; then read -r -a t2 <<< "$(get_tier_containers 2)" - ALL_FAILOVER_CONTAINERS+=("${t2[@]}") + ALL_FALLBACK_CONTAINERS+=("${t2[@]}") fi # Tier 1 always started — stop last (DDNS already handled above) @@ -507,15 +507,15 @@ run_handback() { for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do [[ "$container" == "$ddns" ]] && is_ddns=true && break done - [[ "$is_ddns" == false ]] && ALL_FAILOVER_CONTAINERS+=("$container") + [[ "$is_ddns" == false ]] && ALL_FALLBACK_CONTAINERS+=("$container") done - for container in "${ALL_FAILOVER_CONTAINERS[@]}"; do + for container in "${ALL_FALLBACK_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue local_stop "$container" done - log "All failover containers stopped locally" + log "All fallback containers stopped locally" # ── Step 4: Rsync writeback ────────────────────────────────────────────────────────────── # Tiered writeback with skip window — short outages do not benefit from writeback. @@ -525,14 +525,14 @@ run_handback() { echo "━━━ $ICON_SYNC Rsync Writeback ━━━" local outage_minutes - outage_minutes=$(( ($(date +%s) - $(state_get failover_start)) / 60 )) + outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 )) local tier1_wb_delay tier1_wb_delay=$(get_tier1_writeback_delay) warn "Outage duration: ${outage_minutes}min" - if ! check_rsync_enabled "FAILOVER"; then - warn "FAILOVER_RSYNC_ENABLED=false — skipping all writeback jobs" + if ! check_rsync_enabled "FALLBACK"; then + warn "FALLBACK_RSYNC_ENABLED=false — skipping all writeback jobs" warn "Handback will complete without syncing state back to primary" else run_writeback_tier() { @@ -548,7 +548,7 @@ run_handback() { log "Syncing: $job" if [[ "$DRY_RUN" == false ]]; then if [[ "$(basename "$job")" == "Emby" ]]; then - bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-failover + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback else bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" fi @@ -621,15 +621,15 @@ run_handback() { # ── Step 7: Return to NORMAL ───────────────────────────────────────────────────────────── echo "" state_set state "NORMAL" - state_set failover_start "0" + state_set fallback_start "0" state_set handback_strikes "0" state_set tier2_started "false" state_set tier3_started "false" state_set tier4_started "false" warn "$ICON_DONE Handback complete — returned to NORMAL" - notify "Failover handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \ - "Failover" "normal" + notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \ + "Fallback" "normal" } # ============================================================================================== @@ -640,16 +640,16 @@ set_ddns_arrays echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " $ICON_FAILOVER FAILOVER — $(date '+%Y-%m-%d %H:%M:%S')" +echo " $ICON_FALLBACK FALLBACK — $(date '+%Y-%m-%d %H:%M:%S')" echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)" echo " $ICON_NET Remote IP: $REMOTE_SERVER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -FAILOVER_RUNNING=true -trap 'FAILOVER_RUNNING=false; warn "Failover received shutdown signal — stopping cleanly"; exit 0' \ +FALLBACK_RUNNING=true +trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \ SIGTERM SIGINT -while [[ "$FAILOVER_RUNNING" == true ]]; do +while [[ "$FALLBACK_RUNNING" == true ]]; do NOW=$(date +%s) CURRENT_STATE=$(state_get state) @@ -673,11 +673,11 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then # Remote is down — enter FAILOVER echo "" - echo "━━━ $ICON_FAILOVER Entering FAILOVER — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting failover" + echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback" - state_set state "FAILOVER" - state_set failover_start "$NOW" + state_set state "FALLBACK" + state_set fallback_start "$NOW" state_set handback_strikes "0" state_set tier2_started "false" state_set tier3_started "false" @@ -690,8 +690,8 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do [[ -n "$container" ]] && local_start "$container" done - notify "FAILOVER started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \ - "Failover" "warning" + notify "FALLBACK started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \ + "Fallback" "warning" elif [[ "$INTERNET_UP" == false ]]; then # Lost internet — enter NO_INTERNET @@ -709,16 +709,16 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do done notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \ - "Failover" "warning" + "Fallback" "warning" fi # ════════════════════════════════════════════════════════════════ # FAILOVER STATE # ════════════════════════════════════════════════════════════════ - elif [[ "$CURRENT_STATE" == "FAILOVER" ]]; then + elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then - FAILOVER_START_TS=$(state_get failover_start) - ELAPSED_MIN=$(( (NOW - FAILOVER_START_TS) / 60 )) + FALLBACK_START_TS=$(state_get fallback_start) + ELAPSED_MIN=$(( (NOW - FALLBACK_START_TS) / 60 )) if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then # Remote returned — increment handback strikes @@ -726,21 +726,21 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do STRIKES=$(( STRIKES + 1 )) state_set handback_strikes "$STRIKES" - warn "$ICON_FAILOVER $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FAILOVER_HANDBACK_STRIKES" + warn "$ICON_FALLBACK $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FALLBACK_HANDBACK_STRIKES" - if [[ "$STRIKES" -ge "$FAILOVER_HANDBACK_STRIKES" ]]; then + if [[ "$STRIKES" -ge "$FALLBACK_HANDBACK_STRIKES" ]]; then run_handback fi elif [[ "$INTERNET_UP" == false ]]; then - # Lost internet during failover — enter DARK + # Lost internet during fallback — enter DARK echo "" echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" - warn "Lost internet during failover — entering DARK state" + warn "Lost internet during fallback — entering DARK state" state_set state "DARK" local_ddns_stop - notify "DARK state on $(hostname) — lost internet during failover" \ - "Failover" "warning" + notify "DARK state on $(hostname) — lost internet during fallback" \ + "Fallback" "warning" else # Still in failover — check tier escalation @@ -756,8 +756,8 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do [[ -n "$container" ]] && local_start "$container" done state_set tier2_started "true" - notify "Failover Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \ - "Failover" "warning" + notify "Fallback Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \ + "Fallback" "warning" fi TIER3_DELAY=$(get_tier3_delay) @@ -770,8 +770,8 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do [[ -n "$container" ]] && local_start "$container" done state_set tier3_started "true" - notify "Failover Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \ - "Failover" "warning" + notify "Fallback Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \ + "Fallback" "warning" fi TIER4_DELAY=$(get_tier4_delay) @@ -784,11 +784,11 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do [[ -n "$container" ]] && local_start "$container" done state_set tier4_started "true" - notify "Failover Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \ - "Failover" "warning" + notify "Fallback Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \ + "Fallback" "warning" fi - log "$ICON_FAILOVER FAILOVER — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)" + log "$ICON_FALLBACK FALLBACK — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)" fi # ════════════════════════════════════════════════════════════════ @@ -805,11 +805,11 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do state_set state "NORMAL" local_ddns_start notify "Internet recovered on $(hostname) — returning to NORMAL" \ - "Failover" "normal" + "Fallback" "normal" else - warn "Internet back but $REMOTE_SERVER_NAME still down — entering FAILOVER" - state_set state "FAILOVER" - state_set failover_start "$NOW" + warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK" + state_set state "FALLBACK" + state_set fallback_start "$NOW" state_set handback_strikes "0" state_set tier2_started "false" state_set tier3_started "false" @@ -821,8 +821,8 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do for container in "${tier1[@]}"; do [[ -n "$container" ]] && local_start "$container" done - notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FAILOVER" \ - "Failover" "warning" + notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FALLBACK" \ + "Fallback" "warning" fi else log "$ICON_NET NO_INTERNET — waiting for connectivity" @@ -838,14 +838,14 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━" if [[ "$REMOTE_UP" == true ]]; then - warn "Remote up, internet up — transitioning through FAILOVER for handback" - # Was in failover before DARK — route through FAILOVER state for handback - state_set state "FAILOVER" + warn "Remote up, internet up — transitioning through FALLBACK for handback" + # Was in failover before DARK — route through FALLBACK state for handback + state_set state "FALLBACK" state_set handback_strikes "0" else - warn "Internet back but $REMOTE_SERVER_NAME still down — entering FAILOVER" - state_set state "FAILOVER" - state_set failover_start "$NOW" + warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK" + state_set state "FALLBACK" + state_set fallback_start "$NOW" state_set handback_strikes "0" state_set tier2_started "false" state_set tier3_started "false" @@ -856,15 +856,15 @@ while [[ "$FAILOVER_RUNNING" == true ]]; do [[ -n "$container" ]] && local_start "$container" done fi - notify "Emerging from DARK state on $(hostname)" "Failover" "warning" + notify "Emerging from DARK state on $(hostname)" "Fallback" "warning" else - log "$ICON_FAILOVER DARK — no internet, no remote — waiting" + log "$ICON_FALLBACK DARK — no internet, no remote — waiting" fi fi # ── Sleep until next check ─────────────────────────────────────────────────────────────── - log "Next check in ${FAILOVER_CHECK_INTERVAL}s — $(date '+%H:%M:%S')" - sleep "$FAILOVER_CHECK_INTERVAL" & + log "Next check in ${FALLBACK_CHECK_INTERVAL}s — $(date '+%H:%M:%S')" + sleep "$FALLBACK_CHECK_INTERVAL" & wait $! done \ No newline at end of file diff --git a/Failover/failover_test.sh b/Fallback/fallback_test.sh similarity index 79% rename from Failover/failover_test.sh rename to Fallback/fallback_test.sh index fae4454..dc47a6b 100644 --- a/Failover/failover_test.sh +++ b/Fallback/fallback_test.sh @@ -7,22 +7,22 @@ # # ── WHAT THIS SCRIPT IS ─────────────────────────────────────────────────────────────────────── # A test harness only — contains no failover logic. -# All failover logic lives in failover.sh and is exercised by this test. -# Any changes to failover.sh are automatically reflected here. +# All failover logic lives in fallback.sh and is exercised by this test. +# Any changes to fallback.sh are automatically reflected here. # # ── TEST SEQUENCE ───────────────────────────────────────────────────────────────────────────── # Phase 1 — Pre-flight verify both servers reachable, daemons healthy, -# version parity, failover.sh exists, state is NORMAL +# version parity, fallback.sh exists, state is NORMAL # Phase 2 — Block Remote iptables rule drops all traffic to remote IP -# Phase 3 — Failover Detection wait for failover.sh to detect outage and enter FAILOVER +# Phase 3 — Fallback Detection wait for fallback.sh to detect outage and enter FAILOVER # Phase 4 — Container Start verify Tier 1 failover containers started locally # Phase 5 — Restore remove iptables rule, remote becomes reachable -# Phase 6 — Handback wait for failover.sh to complete handback to NORMAL +# Phase 6 — Handback wait for fallback.sh to complete handback to NORMAL # Phase 7 — Container Handback verify Tier 1 containers stopped locally after handback # Phase 8 — Report full pass/fail summary per phase # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# FAILOVER_ENABLED gate — aborts if failover monitoring is disabled +# FALLBACK_ENABLED gate — aborts if fallback monitoring is disabled # iptables safety trap — rule ALWAYS removed on exit (crash, error, ctrl-c, normal) # remote connectivity always restored regardless of outcome # Version parity check — pre-flight verifies both servers on compatible unRAID versions @@ -38,17 +38,17 @@ # Use --dry-run to walk through the sequence without any real changes. # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# FAILOVER_TEST_BLOCK_WAIT — seconds to wait for failover.sh to detect outage -# FAILOVER_TEST_HANDBACK_WAIT — seconds to wait for failover.sh to complete handback -# FAILOVER_CHECK_INTERVAL — check interval of the running failover.sh (informational) -# FAILOVER_HANDBACK_STRIKES — strikes required before handback (informational) -# FAILOVER_STATE_FILE — state file path to read current state +# FALLBACK_TEST_BLOCK_WAIT — seconds to wait for fallback.sh to detect outage +# FALLBACK_TEST_HANDBACK_WAIT — seconds to wait for fallback.sh to complete handback +# FALLBACK_CHECK_INTERVAL — check interval of the running fallback.sh (informational) +# FALLBACK_HANDBACK_STRIKES — strikes required before handback (informational) +# FALLBACK_STATE_FILE — state file path to read current state # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# failover_test.sh — run full test sequence -# failover_test.sh --dry-run — walk through all phases without changes -# failover_test.sh --status — show current failover state and test config -# failover_test.sh --log — verbose output +# fallback_test.sh — run full test sequence +# fallback_test.sh --dry-run — walk through all phases without changes +# fallback_test.sh --status — show current fallback state and test config +# fallback_test.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -57,7 +57,7 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -FAILOVER_SCRIPT="$SCRIPT_DIR/failover.sh" +FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh" DOCKER_TIMEOUT=15 # ============================================================================================== @@ -95,10 +95,10 @@ if [[ "$EUID" -ne 0 ]]; then exit 1 fi -# FAILOVER_ENABLED gate — no point testing if failover is disabled -if [[ "${FAILOVER_ENABLED:-false}" == false ]]; then - warn "FAILOVER_ENABLED=false — failover test aborted" - warn "Enable failover in master.conf before running this test" +# FALLBACK_ENABLED gate — no point testing if failover is disabled +if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then + warn "FALLBACK_ENABLED=false — fallback test aborted" + warn "Enable fallback in master.conf before running this test" exit 0 fi @@ -118,11 +118,11 @@ validate_unraid_cmd \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" -if [[ ! -f "$FAILOVER_SCRIPT" ]]; then - error "failover.sh not found at $FAILOVER_SCRIPT" +if [[ ! -f "$FALLBACK_SCRIPT" ]]; then + error "fallback.sh not found at $FALLBACK_SCRIPT" exit 1 fi -log "failover.sh found at $FAILOVER_SCRIPT" +log "fallback.sh found at $FALLBACK_SCRIPT" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no iptables rules or container changes will be made" @@ -137,21 +137,21 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)" echo "$ICON_GEAR unRAID ver: $local_ver" - echo "$ICON_FAILOVER Block wait: ${FAILOVER_TEST_BLOCK_WAIT}s" - echo "$ICON_FAILOVER Handback wait: ${FAILOVER_TEST_HANDBACK_WAIT}s" - echo "$ICON_FAILOVER Check interval: ${FAILOVER_CHECK_INTERVAL}s" - echo "$ICON_FAILOVER Handback strikes: ${FAILOVER_HANDBACK_STRIKES}" + echo "$ICON_FALLBACK Block wait: ${FALLBACK_TEST_BLOCK_WAIT}s" + echo "$ICON_FALLBACK Handback wait: ${FALLBACK_TEST_HANDBACK_WAIT}s" + echo "$ICON_FALLBACK Check interval: ${FALLBACK_CHECK_INTERVAL}s" + echo "$ICON_FALLBACK Handback strikes: ${FALLBACK_HANDBACK_STRIKES}" echo "$ICON_GEAR Dry Run: $DRY_RUN" - if [[ -f "$FAILOVER_STATE_FILE" ]]; then - CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - echo "$ICON_FAILOVER Current state: ${CURRENT_STATE:-unknown}" + if [[ -f "$FALLBACK_STATE_FILE" ]]; then + CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) + echo "$ICON_FALLBACK Current state: ${CURRENT_STATE:-unknown}" else - echo "$ICON_FAILOVER Current state: no state file" + echo "$ICON_FALLBACK Current state: no state file" fi # Show Tier 1 containers for this host - TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1" + TIER1_VAR="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1" eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")" echo "$ICON_CONTAINERS Tier 1 to test: ${TIER1_CONTAINERS[*]:-none configured}" echo "━━━━━━━━━━━━━━━━━━━━━━━" @@ -169,7 +169,7 @@ phase_pass() { PHASES_PASS+=("$1"); warn "$ICON_DONE Phase: $1 — PASSED ✅"; phase_fail() { PHASES_FAIL+=("$1"); error "Phase: $1 — FAILED ❌"; } # Get Tier 1 containers for this server's failover responsibility -TIER1_VAR="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1" +TIER1_VAR="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1" eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")" # ============================================================================================== @@ -177,7 +177,7 @@ eval "TIER1_CONTAINERS=(\"\${${TIER1_VAR}[@]:-}\")" # ============================================================================================== echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -echo " $ICON_SHIELD FAILOVER TEST — $(date '+%Y-%m-%d %H:%M:%S')" +echo " $ICON_SHIELD FALLBACK TEST — $(date '+%Y-%m-%d %H:%M:%S')" echo " $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" @@ -215,15 +215,15 @@ if ! check_remote_docker_daemon; then exit 1 fi -# Failover state must be NORMAL before test -if [[ -f "$FAILOVER_STATE_FILE" ]]; then - CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) +# Fallback state must be NORMAL before test +if [[ -f "$FALLBACK_STATE_FILE" ]]; then + CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ "$CURRENT_STATE" != "NORMAL" ]]; then - error "Failover state is $CURRENT_STATE — must be NORMAL before running test" + error "Fallback state is $CURRENT_STATE — must be NORMAL before running test" phase_fail "Pre-flight" exit 1 fi - log "Failover state is NORMAL" + log "Fallback state is NORMAL" else warn "No state file found — assuming NORMAL (first run)" fi @@ -231,7 +231,7 @@ fi # Tier 1 containers configured if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID" - error "Check FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER1 in master_host*.conf" + error "Check FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER1 in master_host*.conf" phase_fail "Pre-flight" exit 1 fi @@ -266,33 +266,33 @@ else fi # ============================================================================================== -# ━━━ Phase 3 — Failover Detection ━━━ +# ━━━ Phase 3 — Fallback Detection ━━━ # ============================================================================================== echo "" -echo "━━━ $ICON_FAILOVER Phase 3 — Failover Detection ━━━" -warn "Waiting ${FAILOVER_TEST_BLOCK_WAIT}s for failover.sh to detect outage..." -log "failover.sh check interval: ${FAILOVER_CHECK_INTERVAL}s" +echo "━━━ $ICON_FALLBACK Phase 3 — Fallback Detection ━━━" +warn "Waiting ${FALLBACK_TEST_BLOCK_WAIT}s for fallback.sh to detect outage..." +log "fallback.sh check interval: ${FALLBACK_CHECK_INTERVAL}s" if [[ "$DRY_RUN" == false ]]; then - sleep "$FAILOVER_TEST_BLOCK_WAIT" + sleep "$FALLBACK_TEST_BLOCK_WAIT" - if [[ -f "$FAILOVER_STATE_FILE" ]]; then - NEW_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - if [[ "$NEW_STATE" == "FAILOVER" ]]; then - log "State changed to FAILOVER — outage detected correctly ✅" - phase_pass "Failover Detection" + if [[ -f "$FALLBACK_STATE_FILE" ]]; then + NEW_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ "$NEW_STATE" == "FALLBACK" ]]; then + log "State changed to FALLBACK — outage detected correctly ✅" + phase_pass "Fallback Detection" else - error "State is $NEW_STATE — expected FAILOVER after ${FAILOVER_TEST_BLOCK_WAIT}s" - warn "Is failover.sh running? Check User Scripts plugin" - phase_fail "Failover Detection" + error "State is $NEW_STATE — expected FALLBACK after ${FALLBACK_TEST_BLOCK_WAIT}s" + warn "Is fallback.sh running? Check User Scripts plugin" + phase_fail "Fallback Detection" fi else - error "No state file found after wait — failover.sh may not be running" - phase_fail "Failover Detection" + error "No state file found after wait — fallback.sh may not be running" + phase_fail "Fallback Detection" fi else - warn "DRY RUN — would wait ${FAILOVER_TEST_BLOCK_WAIT}s then check for FAILOVER state" - phase_pass "Failover Detection" + warn "DRY RUN — would wait ${FALLBACK_TEST_BLOCK_WAIT}s then check for FALLBACK state" + phase_pass "Fallback Detection" fi # ============================================================================================== @@ -354,22 +354,22 @@ fi # ━━━ Phase 6 — Handback ━━━ # ============================================================================================== echo "" -echo "━━━ $ICON_FAILOVER Phase 6 — Handback ━━━" -warn "Waiting ${FAILOVER_TEST_HANDBACK_WAIT}s for failover.sh to complete handback..." -log "Requires $FAILOVER_HANDBACK_STRIKES consecutive checks at ${FAILOVER_CHECK_INTERVAL}s" -log "Minimum handback time: $(( FAILOVER_HANDBACK_STRIKES * FAILOVER_CHECK_INTERVAL ))s" +echo "━━━ $ICON_FALLBACK Phase 6 — Handback ━━━" +warn "Waiting ${FALLBACK_TEST_HANDBACK_WAIT}s for fallback.sh to complete handback..." +log "Requires $FALLBACK_HANDBACK_STRIKES consecutive checks at ${FALLBACK_CHECK_INTERVAL}s" +log "Minimum handback time: $(( FALLBACK_HANDBACK_STRIKES * FALLBACK_CHECK_INTERVAL ))s" if [[ "$DRY_RUN" == false ]]; then - sleep "$FAILOVER_TEST_HANDBACK_WAIT" + sleep "$FALLBACK_TEST_HANDBACK_WAIT" - if [[ -f "$FAILOVER_STATE_FILE" ]]; then - FINAL_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ -f "$FALLBACK_STATE_FILE" ]]; then + FINAL_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) if [[ "$FINAL_STATE" == "NORMAL" ]]; then log "State returned to NORMAL — handback completed ✅" phase_pass "Handback" else - error "State is $FINAL_STATE — expected NORMAL after ${FAILOVER_TEST_HANDBACK_WAIT}s" - warn "Handback may still be in progress — check failover.sh output" + error "State is $FINAL_STATE — expected NORMAL after ${FALLBACK_TEST_HANDBACK_WAIT}s" + warn "Handback may still be in progress — check fallback.sh output" phase_fail "Handback" fi else @@ -377,7 +377,7 @@ if [[ "$DRY_RUN" == false ]]; then phase_fail "Handback" fi else - warn "DRY RUN — would wait ${FAILOVER_TEST_HANDBACK_WAIT}s then verify NORMAL state" + warn "DRY RUN — would wait ${FALLBACK_TEST_HANDBACK_WAIT}s then verify NORMAL state" phase_pass "Handback" fi @@ -418,7 +418,7 @@ TOTAL_END=$(date +%s) # ━━━ Test Report ━━━ # ============================================================================================== echo "" -echo "━━━━━ $ICON_SUMMARY FAILOVER TEST REPORT ━━━━━" +echo "━━━━━ $ICON_SUMMARY FALLBACK TEST REPORT ━━━━━" echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((TOTAL_END - TOTAL_START)))" @@ -440,12 +440,12 @@ if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$FAIL_COUNT" -eq 0 ]]; then warn "$ICON_DONE ALL $TOTAL_PHASES PHASES PASSED" - notify "Failover test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \ - "Failover Test" "normal" + notify "Fallback test PASSED on $(hostname) — all $TOTAL_PHASES phases completed" \ + "Fallback Test" "normal" else error "$FAIL_COUNT/$TOTAL_PHASES PHASES FAILED" - notify "Failover test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \ - "Failover Test" "warning" + notify "Fallback test FAILED on $(hostname) — $FAIL_COUNT/$TOTAL_PHASES phases failed: ${PHASES_FAIL[*]}" \ + "Fallback Test" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/Initial_run/partnership_onboard.sh b/Initial_run/partnership_onboard.sh index b78a331..5d6ba62 100755 --- a/Initial_run/partnership_onboard.sh +++ b/Initial_run/partnership_onboard.sh @@ -58,7 +58,7 @@ detect_hosts START=$(date +%s) echo "" -echo "━━━ $ICON_FAILOVER Partnership Onboard Setup — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "━━━ $ICON_FALLBACK Partnership Onboard Setup — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo " This server: $MY_ID ($LOCAL_SERVER_NAME)" echo " Partner: $REMOTE_ID ($REMOTE_SERVER_NAME)" diff --git a/Monitors/continuous_scripts_status.sh b/Monitors/continuous_scripts_status.sh index cad613b..6d5af52 100644 --- a/Monitors/continuous_scripts_status.sh +++ b/Monitors/continuous_scripts_status.sh @@ -371,43 +371,43 @@ fi # ============================================================================================== # ━━━ Failover ━━━ # ============================================================================================== -section "🔀 FAILOVER" +section "🔀 FALLBACK" -FAILOVER_PID=$(get_lock_pid "failover") -FAILOVER_RUNNING=false +FALLBACK_PID=$(get_lock_pid "fallback") +FALLBACK_RUNNING=false -if is_script_running "failover"; then - FAILOVER_RUNNING=true - FAILOVER_AGE=$(get_lock_age "failover") - FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE") - echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME" +if is_script_running "fallback"; then + FALLBACK_RUNNING=true + FALLBACK_AGE=$(get_lock_age "fallback") + FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE") + echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME" else - if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then - echo " ⏸️ Disabled — FAILOVER_ENABLED=false in master.conf" + if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then + echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf" else - echo " ❌ NOT RUNNING — failover.sh is not active" + echo " ❌ NOT RUNNING — fallback.sh is not active" echo " Start via: bash Orchestrators/array_start.sh" fi fi echo "" -# Failover state -FAILOVER_STATE="UNKNOWN" -FAILOVER_STATE_SECONDS=0 +# Fallback state +FALLBACK_STATE="UNKNOWN" +FALLBACK_STATE_SECONDS=0 -if [[ -f "$FAILOVER_STATE_FILE" ]]; then - FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FAILOVER_LAST_EPOCH=$(grep "^failover_start=" "$FAILOVER_STATE_FILE" \ +if [[ -f "$FALLBACK_STATE_FILE" ]]; then + FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) + FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \ 2>/dev/null | cut -d= -f2) - if [[ -n "$FAILOVER_LAST_EPOCH" && "$FAILOVER_LAST_EPOCH" -gt 0 ]]; then - FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH )) + if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then + FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH )) fi fi -STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}") +STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}") -# Tier delays via REMOTE_ID — same logic as failover.sh +# Tier delays via REMOTE_ID — same logic as fallback.sh REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY" REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY" REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY" @@ -415,33 +415,33 @@ TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}" TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}" TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}" -case "$FAILOVER_STATE" in +case "$FALLBACK_STATE" in NORMAL) echo " ✅ State: NORMAL" ;; - FAILOVER) - echo " ⚠️ State: FAILOVER — $REMOTE_SERVER_NAME is down" + FALLBACK) + echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down" echo " ⏱️ Duration: $STATE_DURATION" - FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 )) + FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 )) echo "" echo " 🔄 Tier status:" echo " Tier 1 (immediate): ✅ active" - if (( FAILOVER_MINS >= TIER2_DELAY )); then + if (( FALLBACK_MINS >= TIER2_DELAY )); then echo " Tier 2 (${TIER2_DELAY}min): ✅ active" else - REMAINING=$(( TIER2_DELAY - FAILOVER_MINS )) + REMAINING=$(( TIER2_DELAY - FALLBACK_MINS )) echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min" fi - if (( FAILOVER_MINS >= TIER3_DELAY )); then + if (( FALLBACK_MINS >= TIER3_DELAY )); then echo " Tier 3 (${TIER3_DELAY}min): ✅ active" else - REMAINING=$(( TIER3_DELAY - FAILOVER_MINS )) + REMAINING=$(( TIER3_DELAY - FALLBACK_MINS )) echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min" fi - if (( FAILOVER_MINS >= TIER4_DELAY )); then + if (( FALLBACK_MINS >= TIER4_DELAY )); then echo " Tier 4 (${TIER4_DELAY}min): ✅ active" else - REMAINING=$(( TIER4_DELAY - FAILOVER_MINS )) + REMAINING=$(( TIER4_DELAY - FALLBACK_MINS )) echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min" fi ;; @@ -454,7 +454,7 @@ case "$FAILOVER_STATE" in echo " ⏱️ Duration: $STATE_DURATION" ;; *) - echo " ❓ State: ${FAILOVER_STATE:-unknown}" + echo " ❓ State: ${FALLBACK_STATE:-unknown}" ;; esac @@ -475,7 +475,7 @@ else echo " 🌐 Tailscale: not available" fi -echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}" +echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}" # ============================================================================================== # ━━━ Footer ━━━ @@ -486,11 +486,11 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ ISSUES=0 [[ "$SYS_RUNNING" == false ]] && (( ISSUES++ )) [[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ )) -[[ "$FAILOVER_RUNNING" == false && "${FAILOVER_ENABLED:-true}" != false ]] && (( ISSUES++ )) +[[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ )) [[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ )) [[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ )) [[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ )) -[[ "$FAILOVER_STATE" != "NORMAL" && "$FAILOVER_STATE" != "UNKNOWN" ]] && (( ISSUES++ )) +[[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ )) if [[ "$ISSUES" -eq 0 ]]; then echo " ✅ $MY_ID — all continuous scripts healthy" diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index 89a6441..64a34c9 100644 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -20,7 +20,7 @@ # master.conf to switch behaviour. No cron changes needed. # # ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── -# FAILOVER_STATE_FILE — current failover state +# FALLBACK_STATE_FILE — current fallback state # SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed) # WATCHDOG_STATE_FILE — active container watchdog strikes # SYS_WATCHDOG_STATE_FILE — active system watchdog strikes @@ -44,7 +44,7 @@ # DIGEST_PROFILE — always | smart | weekly # DIGEST_DAY — day name for weekly profile (e.g. Sunday) # DIGEST_SMART_ON_WATCHDOG — send on active watchdog strikes -# DIGEST_SMART_ON_FAILOVER — send on non-NORMAL failover state +# DIGEST_SMART_ON_FALLBACK — send on non-NORMAL fallback state # DIGEST_SMART_ON_CERT_WARN — send on cert warning # DIGEST_SMART_ON_BANDWIDTH — send on high bandwidth day # CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT @@ -101,7 +101,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_DIGEST Profile: $DIGEST_PROFILE" echo "$ICON_DIGEST Digest day: $DIGEST_DAY" - echo "$ICON_DIGEST Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG failover=$DIGEST_SMART_ON_FAILOVER cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH" + echo "$ICON_DIGEST Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG fallback=$DIGEST_SMART_ON_FALLBACK cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH" echo "$ICON_CERT Cert domains: ${CERT_MONITOR_DOMAINS[*]:-none}" echo "$ICON_BANDWIDTH Bandwidth warn: ${BANDWIDTH_WARN_GB}GB" echo "$ICON_GEAR Dry Run: $DRY_RUN" @@ -148,17 +148,17 @@ ISSUES=() # need attention DIGEST_LINES=() # full report lines # ── Failover State ──────────────────────────────────────────────────────────────────────────── -if [[ -f "$FAILOVER_STATE_FILE" ]]; then - FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - if [[ -n "$FAILOVER_STATE" ]]; then - DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE") - if [[ "$FAILOVER_STATE" != "NORMAL" ]]; then - ISSUES+=("Failover state: $FAILOVER_STATE") - [[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true +if [[ -f "$FALLBACK_STATE_FILE" ]]; then + FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ -n "$FALLBACK_STATE" ]]; then + DIGEST_LINES+=("$ICON_FALLBACK Fallback: $FALLBACK_STATE") + if [[ "$FALLBACK_STATE" != "NORMAL" ]]; then + ISSUES+=("Fallback state: $FALLBACK_STATE") + [[ "$DIGEST_SMART_ON_FALLBACK" == true ]] && SHOULD_SEND=true fi fi else - DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found") + DIGEST_LINES+=("$ICON_FALLBACK Fallback: state file not found") fi # ── Container Skip List ─────────────────────────────────────────────────────────────────────── diff --git a/Orchestrators/array_start.sh b/Orchestrators/array_start.sh index 883340f..37d6cc1 100644 --- a/Orchestrators/array_start.sh +++ b/Orchestrators/array_start.sh @@ -20,7 +20,7 @@ # CONTINUOUS (run until array stops): # unRAID_Essentials/system_watchdog.sh — system health monitor (last line of defense) # Docker_Essentials/docker_watchdog.sh — container health monitor -# Failover/failover.sh — mutual failover monitor +# Fallback/fallback.sh — mutual failover monitor # # ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── # inotify_tuning.sh — must run BEFORE Code-Server and other containers start @@ -30,7 +30,7 @@ # docker_network_connect — must run BEFORE watchdogs check container states # system_watchdog.sh — before docker_watchdog (system > container priority) # docker_watchdog.sh — before failover (containers must be healthy for failover) -# failover.sh — last — needs everything else stable to make decisions +# fallback.sh — last — needs everything else stable to make decisions # # ── ONE-SHOT vs CONTINUOUS DETECTION ───────────────────────────────────────────────────────── # Script is launched in background with bash script.sh & diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index ea5f4b1..8818266 100644 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -8,7 +8,7 @@ # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) -# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running) +# 2. emby-fallback rsync — dirty Emby sync (watch states, library — Emby stays running) # 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window # 4. partnership --check — read both state files, detect changes, act accordingly # diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 572866e..541f308 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -14,7 +14,7 @@ # 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split # 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth # 🌐 Rsync — weekly transfer totals, per-share breakdown, failures -# 🛡️ Watchdog — system watchdog, docker watchdog, failover state +# 🛡️ Watchdog — system watchdog, docker watchdog, fallback state # 🔐 Security — SSL cert expiry per domain # 📊 Emby — weekly stream count, active now, top users # ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync @@ -22,7 +22,7 @@ # # ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── # DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history -# /boot/config — failover state, watchdog reboot log +# /boot/config — fallback state, watchdog reboot log # /tmp — watchdog strike state files # /proc, /sys — system memory, uptime, inotify # /var/local/emhttp/ — unRAID array info @@ -113,7 +113,7 @@ if [[ "$SHOW_STATUS" == true ]]; then "$SYS_WATCHDOG_REBOOT_LOG:watchdog reboot log" \ "$WATCHDOG_STATE_FILE:docker watchdog state" \ "$WATCHDOG_CONTAINER_RESTART_LOG:container restart log" \ - "$FAILOVER_STATE_FILE:failover state"; do + "$FALLBACK_STATE_FILE:fallback state"; do path="${f%%:*}" label="${f##*:}" if [[ -f "$path" ]] && [[ -s "$path" ]]; then @@ -578,22 +578,22 @@ fi REPORT+=("") # ── Failover ───────────────────────────────────────────────────────────────────────────────── -line "🔀 Failover" -FO_PID=$(_get_lock_pid "failover") -if _is_running "failover"; then - FO_AGE=$(_lock_age "failover") +line "🔀 Fallback" +FO_PID=$(_get_lock_pid "fallback") +if _is_running "fallback"; then + FO_AGE=$(_lock_age "fallback") line " ✅ Running │ PID: $FO_PID │ Uptime: $(_fmt_uptime "$FO_AGE")" -elif [[ "${FAILOVER_ENABLED:-true}" == false ]]; then - line " ⏸️ Not running — FAILOVER_ENABLED=false" +elif [[ "${FALLBACK_ENABLED:-true}" == false ]]; then + line " ⏸️ Not running — FALLBACK_ENABLED=false" else - issue "failover NOT RUNNING" + issue "fallback NOT RUNNING" fi FO_STATE="UNKNOWN" FO_STATE_SECS=0 -if [[ -f "$FAILOVER_STATE_FILE" ]]; then - FO_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FO_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) +if [[ -f "$FALLBACK_STATE_FILE" ]]; then + FO_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) + FO_EPOCH=$(grep "^last_change_epoch=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) [[ -n "$FO_EPOCH" ]] && FO_STATE_SECS=$(( NOW - FO_EPOCH )) fi @@ -602,8 +602,8 @@ FO_DUR=$(_fmt_uptime "${FO_STATE_SECS:-0}") case "$FO_STATE" in NORMAL) line " ✅ State: NORMAL │ Duration: $FO_DUR" ;; - FAILOVER) - issue " State: FAILOVER — $REMOTE_SERVER_NAME down for $FO_DUR" + FALLBACK) + issue " State: FALLBACK — $REMOTE_SERVER_NAME down for $FO_DUR" FO_MINS=$(( FO_STATE_SECS / 60 )) # Use REMOTE_ID-based tier delay vars — no HOST1/HOST2 hardcoding T2_VAR="${REMOTE_ID}_TIER2_DELAY"; T3_VAR="${REMOTE_ID}_TIER3_DELAY"; T4_VAR="${REMOTE_ID}_TIER4_DELAY" diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index adc0dc3..9a0db04 100644 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -19,7 +19,7 @@ # Emby builds a warm image cache on HOST2 throughout the week. # Syncing nightly resets cache — cold loads every morning for users. # Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep. -# emby-failover dirty sync covers watch states + library every 15min between weekly syncs. +# emby-fallback dirty sync covers watch states + library every 15min between weekly syncs. # # ── CONTAINER UPDATES ───────────────────────────────────────────────────────────────────────── # Containers already stopped for sync — updates pull at zero extra downtime. diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index 83a6be7..f786723 100644 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -394,7 +394,7 @@ derive_partner_folder_name() { local short="${hostname,,}" [[ "$short" == unraid-* ]] && short="${short:7}" # Capitalise first char for readability: jayred365 → Jayred365-Failover - echo "${short^}-Failover" + echo "${short^}-Fallback" } folderview3_ensure_plugin() { @@ -544,12 +544,12 @@ folderview3_remove_partner_folder() { } # Gather all partner failover containers for this server (all tiers) -gather_partner_failover_containers() { +gather_partner_fallback_containers() { local out_var="$1" eval "${out_var}=()" local tier var for tier in TIER1 TIER2 TIER3 TIER4; do - var="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_${tier}" + var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_${tier}" # Check if var is set and is an array if declare -p "$var" >/dev/null 2>&1; then local -a _tmp_arr @@ -583,11 +583,11 @@ do_final_sync() { # Use CRITICAL_SYNC_SHARES from HOST* conf if available # Falls back to known paths — these are the critical ones bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ - "/mnt/user/appdata-Failover/Critical-Data" \ - --profile=critical-failover --log + "/mnt/user/appdata-Fallback/Critical-Data" \ + --profile=critical-fallback --log bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ "/mnt/user/Media_Server/Emby" \ - --profile=emby-failover --log + --profile=emby-fallback --log else warn "DRY RUN — would run final Critical-Data and Emby sync" fi @@ -867,7 +867,7 @@ fi # ============================================================================================== if [[ "$MODE" == "onboard" ]]; then echo "" - echo "━━━ $ICON_FAILOVER Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "━━━ $ICON_FALLBACK Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # Check already onboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then @@ -958,11 +958,11 @@ if [[ "$MODE" == "onboard" ]]; then echo "━━━ $ICON_CONTAINERS FolderView3 Integration ━━━" PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") declare -a PARTNER_CONTAINERS=() - gather_partner_failover_containers PARTNER_CONTAINERS + gather_partner_fallback_containers PARTNER_CONTAINERS if [[ ${#PARTNER_CONTAINERS[@]} -gt 0 ]]; then folderview3_create_partner_folder "$PARTNER_FOLDER_NAME" "${PARTNER_CONTAINERS[@]}" else - log "No FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER* containers configured — skipping folder creation" + log "No FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER* containers configured — skipping folder creation" fi fi @@ -995,7 +995,7 @@ fi # ============================================================================================== if [[ "$MODE" == "offboard" ]]; then echo "" - echo "━━━ $ICON_FAILOVER Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "━━━ $ICON_FALLBACK Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # Check already offboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then @@ -1196,7 +1196,7 @@ fi # ============================================================================================== if [[ "$MODE" == "transfer" ]]; then echo "" - echo "━━━ $ICON_FAILOVER Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "━━━ $ICON_FALLBACK Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — OWNERSHIP TRANSFER" diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 960ecd7..dcd6297 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -20,7 +20,7 @@ # PROFILE_CONTAINER_DELAY — seconds before starting delayed containers # PROFILE_EXCLUDE_DIRS — paths excluded from transfer # PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync -# (critical-failover, emby-failover profiles) +# (critical-fallback, emby-fallback profiles) # Was running → restart. Was stopped → leave stopped. # # ── RSYNC ENABLE/DISABLE ────────────────────────────────────────────────────────────────────── @@ -36,7 +36,7 @@ # bandwidth_monitor.sh flags syncs exceeding BANDWIDTH_WARN_GB. # # ── DIRTY SYNC REMOTE RESTART ───────────────────────────────────────────────────────────────── -# Profiles using dirty sync (critical-failover, emby-failover) define +# Profiles using dirty sync (critical-fallback, emby-fallback) define # PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after sync completes. # This ensures the remote picks up config changes synced during the dirty window. # Was running → restart. Was stopped → leave stopped. @@ -56,9 +56,9 @@ # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # rsync.sh /mnt/user/Movies — media share, global defaults -# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile -# rsync.sh /mnt/user/appdata-Failover/Arrs_Stack --dry-run --log -# rsync.sh /mnt/user/appdata-Failover/Critical-Data --profile=critical-failover +# rsync.sh /mnt/user/appdata-Fallback/Arrs_Stack — matched to [arrs_stack] profile +# rsync.sh /mnt/user/appdata-Fallback/Arrs_Stack --dry-run --log +# rsync.sh /mnt/user/appdata-Fallback/Critical-Data --profile=critical-fallback # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -269,7 +269,7 @@ fi # ============================================================================================== # ━━━ Remote Restart (dirty sync profiles) ━━━ # ============================================================================================== -# For dirty sync profiles (critical-failover, emby-failover) — restart containers on remote +# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote # that were running before sync so they pick up config changes from the dirty sync window. # Was running → restart. Was stopped → leave stopped. if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then diff --git a/Tools/claude_startup.sh b/Tools/claude_startup.sh new file mode 100755 index 0000000..daa8e5e --- /dev/null +++ b/Tools/claude_startup.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# ============================================================================================== +# ================================= Claude Code Startup ======================================== +# ============================================================================================== +# Restores Claude Code's persistent data after an Unraid reboot and launches Claude. +# +# Unraid's root filesystem lives in RAM — /root/.claude and /root/.local are wiped on every +# reboot. This script symlinks both directories back to persistent appdata storage before +# launching Claude, so memory, sessions, and settings survive across reboots. +# +# ── FIRST RUN ───────────────────────────────────────────────────────────────────────────────── +# If persistent storage has no data yet, migrates from the current live locations: +# /root/.claude → PERSIST_DIR/.claude (memory, sessions, settings) +# /root/.local/share/claude → PERSIST_DIR/local/share/claude (installed binaries) +# Subsequent runs skip the migration and just create the symlinks. +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# claude_startup.sh — set up persistent symlinks and launch Claude +# claude_startup.sh --setup — set up only, do not launch (for array_start.sh use) +# ============================================================================================== + +PERSIST_DIR="/mnt/user/appdata/claude-code" +CLAUDE_DATA="$PERSIST_DIR/.claude" +CLAUDE_BIN="$PERSIST_DIR/local/share/claude" + +LAUNCH=true +[[ "$1" == "--setup" ]] && LAUNCH=false + +# Standalone — no common.sh dependency +_log() { echo " ✅ $*"; } +_warn() { echo " ⚠️ $*"; } +_err() { echo " ❌ $*" >&2; } + +echo "" +echo "━━━ Claude Code Startup ━━━" +echo "" + +# ── Array must be mounted ───────────────────────────────────────────────────────────────────── +if ! mountpoint -q /mnt/user 2>/dev/null; then + _err "Array not mounted — /mnt/user not available" + exit 1 +fi + +# ── Create persistent dirs ──────────────────────────────────────────────────────────────────── +mkdir -p "$CLAUDE_DATA" "$CLAUDE_BIN" + +# ── Migrate .claude on first run ────────────────────────────────────────────────────────────── +if [[ ! -L /root/.claude && -d /root/.claude ]]; then + _warn "First run — migrating /root/.claude → $CLAUDE_DATA" + cp -a /root/.claude/. "$CLAUDE_DATA/" + rm -rf /root/.claude + _log "Migrated .claude (memory, sessions, settings)" +elif [[ -z "$(ls -A "$CLAUDE_DATA" 2>/dev/null)" && -d /root/.claude ]]; then + _warn "Persistent storage empty — copying current .claude data" + cp -a /root/.claude/. "$CLAUDE_DATA/" + _log "Copied .claude data to persistent storage" +fi + +# ── Migrate Claude binaries on first run ────────────────────────────────────────────────────── +if [[ ! -L /root/.local/share/claude && -d /root/.local/share/claude ]]; then + _warn "First run — migrating Claude binaries → $CLAUDE_BIN" + cp -a /root/.local/share/claude/. "$CLAUDE_BIN/" + _log "Migrated Claude binaries" +fi + +# ── Create symlinks ─────────────────────────────────────────────────────────────────────────── +# Remove any real directories first — ln -sfn silently creates inside a dir instead of +# replacing it, which produces a circular symlink on subsequent runs after migration. +mkdir -p /root/.local/share /root/.local/bin + +[[ -d /root/.claude && ! -L /root/.claude ]] && rm -rf /root/.claude +ln -sfn "$CLAUDE_DATA" /root/.claude +_log ".claude → $CLAUDE_DATA" + +[[ -d /root/.local/share/claude && ! -L /root/.local/share/claude ]] && rm -rf /root/.local/share/claude +ln -sfn "$CLAUDE_BIN" /root/.local/share/claude +_log "claude binary → $CLAUDE_BIN" + +# ── Point the claude binary at the latest installed version ─────────────────────────────────── +LATEST=$(ls "$CLAUDE_BIN/versions/" 2>/dev/null | sort -V | tail -1) +if [[ -z "$LATEST" ]]; then + _err "No Claude versions found in $CLAUDE_BIN/versions/" + _err "Install Claude Code first: npm install -g @anthropic-ai/claude-code" + exit 1 +fi + +ln -sfn "$CLAUDE_BIN/versions/$LATEST" /root/.local/bin/claude +_log "claude v$LATEST ready" + +echo "" + +# ── Setup-only mode (used by array_start.sh or other callers) ───────────────────────────────── +if [[ "$LAUNCH" == false ]]; then + _log "Setup complete — run 'claude' to start" + echo "" + exit 0 +fi + +# ── Launch ──────────────────────────────────────────────────────────────────────────────────── +export PATH="$HOME/.local/bin:$PATH" +exec claude diff --git a/Tools/failover_state_reset.sh b/Tools/fallback_state_reset.sh similarity index 73% rename from Tools/failover_state_reset.sh rename to Tools/fallback_state_reset.sh index fde0d0b..78c60b6 100644 --- a/Tools/failover_state_reset.sh +++ b/Tools/fallback_state_reset.sh @@ -2,22 +2,22 @@ # ============================================================================================== # ============================= Failover State Reset =========================================== # ============================================================================================== -# Resets the failover state file to NORMAL and clears all tier flags. -# Use when the failover state file is stuck in a non-NORMAL state after: +# Resets the fallback state file to NORMAL and clears all tier flags. +# Use when the fallback state file is stuck in a non-NORMAL state after: # - Failover testing that left state as FAILOVER # - A failed handback that did not complete cleanly # - Manual intervention that left state inconsistent -# - failover.sh was killed mid-cycle and state is unknown +# - fallback.sh was killed mid-cycle and state is unknown # # ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── # Writes a fresh state file with: # state=NORMAL -# failover_start=0 +# fallback_start=0 # handback_strikes=0 # tier2_started=false / tier3_started=false / tier4_started=false # # Does NOT start or stop any containers — state file only. -# After reset, failover.sh will resume from NORMAL on its next cycle. +# After reset, fallback.sh will resume from NORMAL on its next cycle. # # ── ⚠️ ONLY RUN WHEN SAFE ──────────────────────────────────────────────────────────────────── # Verify BEFORE resetting: @@ -26,22 +26,22 @@ # ✓ No active failover actually in progress # ✓ Both servers can see each other # -# Resetting state while a real failover is happening causes failover.sh to stop +# Resetting state while a real fallback is happening causes fallback.sh to stop # covering the remote server — services go offline until next detection cycle. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# failover.sh running check — warns if failover.sh is active when reset is attempted +# fallback.sh running check — warns if fallback.sh is active when reset is attempted # acquire_lock — prevents concurrent resets -# flock on state write — prevents race with failover.sh mid-cycle read +# flock on state write — prevents race with fallback.sh mid-cycle read # Confirmation required — interactive: type YES | non-interactive: --force flag # validate_unraid_cmd — notify validated before use # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# failover_state_reset.sh — interactive reset (prompts for YES) -# failover_state_reset.sh --dry-run — show current state, show what would be written -# failover_state_reset.sh --status — show current state file contents and exit -# failover_state_reset.sh --force — non-interactive reset (no prompt, use in scripts) -# failover_state_reset.sh --force --dry-run — dry run without prompt +# fallback_state_reset.sh — interactive reset (prompts for YES) +# fallback_state_reset.sh --dry-run — show current state, show what would be written +# fallback_state_reset.sh --status — show current state file contents and exit +# fallback_state_reset.sh --force — non-interactive reset (no prompt, use in scripts) +# fallback_state_reset.sh --force --dry-run — dry run without prompt # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -85,31 +85,31 @@ detect_hosts # ━━━ Current State ━━━ # ============================================================================================== echo "" -echo "━━━ $ICON_FAILOVER Current Failover State ━━━" +echo "━━━ $ICON_FALLBACK Current Fallback State ━━━" echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "" -if [[ ! -f "$FAILOVER_STATE_FILE" ]]; then - warn "State file not found: $FAILOVER_STATE_FILE" +if [[ ! -f "$FALLBACK_STATE_FILE" ]]; then + warn "State file not found: $FALLBACK_STATE_FILE" warn "Will be created fresh on reset" CURRENT_STATE="NOT FOUND" else - log "State file: $FAILOVER_STATE_FILE" + log "State file: $FALLBACK_STATE_FILE" echo "" while IFS='=' read -r key value; do [[ -z "$key" ]] && continue echo " $ICON_INFO $key = $value" - done < "$FAILOVER_STATE_FILE" - CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + done < "$FALLBACK_STATE_FILE" + CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) fi if [[ "$SHOW_STATUS" == true ]]; then echo "" - # Check if failover.sh is running — informational in status mode - if pgrep -f "failover.sh" >/dev/null 2>&1; then - warn "failover.sh is currently RUNNING — any reset would race with active cycle" + # Check if fallback.sh is running — informational in status mode + if pgrep -f "fallback.sh" >/dev/null 2>&1; then + warn "fallback.sh is currently RUNNING — any reset would race with active cycle" else - log "failover.sh is not running" + log "fallback.sh is not running" fi exit 0 fi @@ -120,18 +120,18 @@ fi echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" -# Check if failover.sh is actively running -FAILOVER_RUNNING=false -if pgrep -f "failover.sh" >/dev/null 2>&1; then - FAILOVER_RUNNING=true - warn "⚠️ failover.sh is currently RUNNING" +# Check if fallback.sh is actively running +FALLBACK_RUNNING=false +if pgrep -f "fallback.sh" >/dev/null 2>&1; then + FALLBACK_RUNNING=true + warn "⚠️ fallback.sh is currently RUNNING" warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration" - warn "Consider stopping failover.sh first (click Abort in User Scripts)" - warn "Then reset state, then restart failover.sh" + warn "Consider stopping fallback.sh first (click Abort in User Scripts)" + warn "Then reset state, then restart fallback.sh" echo "" warn "If you are sure you want to proceed anyway, confirm below" else - log "failover.sh is not running — safe to reset ✅" + log "fallback.sh is not running — safe to reset ✅" fi # Check current state — if already NORMAL warn user @@ -144,11 +144,11 @@ fi # ━━━ Confirmation ━━━ # ============================================================================================== echo "" -warn "This will reset failover state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)" +warn "This will reset fallback state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)" warn "Verify before proceeding:" warn " ✓ Right containers running on the right server" warn " ✓ DDNS pointing at correct server" -warn " ✓ No real failover actually in progress" +warn " ✓ No real fallback actually in progress" warn " ✓ Both servers can reach each other" echo "" @@ -165,7 +165,7 @@ if [[ "$DRY_RUN" == false ]]; then else # Non-interactive — no terminal, cannot prompt error "Non-interactive mode — use --force flag to skip confirmation" - error "Usage: failover_state_reset.sh --force" + error "Usage: fallback_state_reset.sh --force" exit 1 fi fi @@ -174,10 +174,10 @@ fi # ━━━ Reset State File ━━━ # ============================================================================================== echo "" -echo "━━━ $ICON_FAILOVER Resetting State File ━━━" +echo "━━━ $ICON_FALLBACK Resetting State File ━━━" NEW_STATE_CONTENT="state=NORMAL -failover_start=0 +fallback_start=0 handback_strikes=0 tier2_started=false tier3_started=false @@ -186,31 +186,31 @@ last_reset=$(date '+%Y-%m-%d %H:%M:%S') reset_by=$MY_ID" if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would write to $FAILOVER_STATE_FILE:" + warn "DRY RUN — would write to $FALLBACK_STATE_FILE:" echo "" echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do echo " $line" done else - mkdir -p "$(dirname "$FAILOVER_STATE_FILE")" + mkdir -p "$(dirname "$FALLBACK_STATE_FILE")" - # flock prevents race with failover.sh mid-cycle read/write + # flock prevents race with fallback.sh mid-cycle read/write ( flock -x 200 - echo "$NEW_STATE_CONTENT" > "$FAILOVER_STATE_FILE" - ) 200>"${FAILOVER_STATE_FILE}.lock" + echo "$NEW_STATE_CONTENT" > "$FALLBACK_STATE_FILE" + ) 200>"${FALLBACK_STATE_FILE}.lock" warn "State file reset to NORMAL ✅" - log "Written to: $FAILOVER_STATE_FILE" + log "Written to: $FALLBACK_STATE_FILE" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" -echo "━━━━━ $ICON_SUMMARY FAILOVER STATE RESET SUMMARY ━━━━━" +echo "━━━━━ $ICON_SUMMARY FALLBACK STATE RESET SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_FAILOVER File: $FAILOVER_STATE_FILE" +echo "$ICON_FALLBACK File: $FALLBACK_STATE_FILE" echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')" echo "" @@ -218,12 +218,12 @@ if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else warn "$ICON_DONE State reset to NORMAL" - log "failover.sh will resume from NORMAL on next cycle" + log "fallback.sh will resume from NORMAL on next cycle" log "No containers were started or stopped" echo "" - [[ "$FAILOVER_RUNNING" == true ]] && \ - warn "⚠️ failover.sh was running during reset — monitor next cycle carefully" - notify "Failover state manually reset to NORMAL on $(hostname) ($MY_ID)" \ - "Failover State Reset" "warning" + [[ "$FALLBACK_RUNNING" == true ]] && \ + warn "⚠️ fallback.sh was running during reset — monitor next cycle carefully" + notify "Fallback state manually reset to NORMAL on $(hostname) ($MY_ID)" \ + "Fallback State Reset" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/common.sh b/common.sh index e39a00a..3a01867 100644 --- a/common.sh +++ b/common.sh @@ -65,11 +65,11 @@ # Tier 1: array disk paths (/mnt/disk*/sharename) # Tier 2: ZFS pools with find -maxdepth 2 (catches nested cache paths) # Tier 3: shfs fallback — verifies /mnt/user is mounted -# Fixes: nested paths like /mnt/cache/appdata-Failover/Critical-Data +# Fixes: nested paths like /mnt/cache/appdata-Fallback/Critical-Data # Fixes: ZFS cache pools not detected when share is not at pool root # # v3.3 PROFILE_REMOTE_RESTART_CONTAINERS support added -# Dirty sync profiles (critical-failover, emby-failover) restart remote +# Dirty sync profiles (critical-fallback, emby-fallback) restart remote # containers after sync if they were running before — picks up config changes # Was stopped → stays stopped. Was running → gets restarted. ✅ # @@ -160,7 +160,7 @@ ICON_RAM="💨" # ramdisk operations — fast ephemeral storage ICON_LINK="🔗" # symlink state and management # Failover Operations -ICON_FAILOVER="🔀" # failover state changes and operations +ICON_FALLBACK="🔀" # fallback state changes and operations # Docker Network Operations ICON_DOCKER_NET="🔌" # Docker network connect operations @@ -626,13 +626,13 @@ check_connectivity() { info "$ICON_PING $REMOTE_SERVER_NAME is reachable" } -# Non-fatal ping — used by failover.sh which handles its own state machine. +# Non-fatal ping — used by fallback.sh which handles its own state machine. # Returns 0 if reachable, 1 if not — does NOT exit. ping_remote() { ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null } -# Non-fatal external connectivity check — used by failover.sh. +# Non-fatal external connectivity check — used by fallback.sh. # Returns 0 if internet reachable, 1 if not — does NOT exit. ping_internet() { ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null @@ -649,7 +649,7 @@ ping_internet() { # check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED # check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED # check_rsync_enabled "CRITICAL" ← checks RSYNC_ENABLED + CRITICAL_RSYNC_ENABLED -# check_rsync_enabled "FAILOVER" ← checks RSYNC_ENABLED + FAILOVER_RSYNC_ENABLED +# check_rsync_enabled "FALLBACK" ← checks RSYNC_ENABLED + FALLBACK_RSYNC_ENABLED # check_rsync_enabled ← checks RSYNC_ENABLED only (direct rsync.sh call) # # Returns: 0 = enabled, proceed | 1 = disabled, skip cleanly @@ -895,7 +895,7 @@ check_local_disk_temps() { # Three-tier detection handles all share locations: # Tier 1: Array disk paths — /mnt/disk*/sharename (XFS or ZFS per-disk) # Tier 2: ZFS standalone pools — finds nested paths up to 2 levels deep -# catches /mnt/cache/appdata-Failover/Critical-Data correctly +# catches /mnt/cache/appdata-Fallback/Critical-Data correctly # Tier 3: shfs fallback — if share exists under /mnt/user, resolve backing pool # or verify shfs itself is mounted if pool can't be determined # @@ -1287,7 +1287,7 @@ acquire_lock() { else error "Another instance of $script_name is already running (PID $existing_pid) — exiting" case "$script_name" in - failover|transcode_management|daily_sync_maintenance|system_watchdog) + fallback|transcode_management|daily_sync_maintenance|system_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" ;; esac diff --git a/master.conf b/master.conf index e3770ee..3c398b6 100644 --- a/master.conf +++ b/master.conf @@ -49,7 +49,7 @@ # # ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── # FAILOVER Mutual container failover shared settings -# FAILOVER TEST Simulated outage settings for failover_test.sh +# FALLBACK TEST Simulated outage settings for fallback_test.sh # # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── # DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent @@ -99,7 +99,7 @@ # DATA_DIR is the same path on all servers — persistent script state and statistics. # Array share — survives reboots, no flash drive wear. # Created automatically if it doesn't exist. -# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. +# Only truly critical files (fallback state, watchdog reboot log) stay on /boot/config. DATA_DIR="/mnt/user/appdata/unraid_scripts/data" # ── Version Parity ── @@ -254,7 +254,7 @@ "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop -# "Failover/failover.sh" # mutual failover — enable when HOST2 ready +# "Fallback/fallback.sh" # mutual failover — enable when HOST2 ready ) # ━━━ Daily Sync Maintenance ━━━ @@ -352,14 +352,14 @@ # DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild # WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) # CRITICAL_RSYNC_ENABLED=true ← 15min auth stack sync still runs -# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works +# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works # → Run individual: bash Rsync/rsync.sh /mnt/user/Movies # → When ready: DAILY_RSYNC_ENABLED=true RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready - FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback + FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. @@ -395,53 +395,53 @@ # arrs_stack — arr databases — lower bandwidth, containers stopped for consistency # critical-data — auth stack — full stop both sides, Authelia delayed start # called by weekly_sync_maintenance.sh — full clean sync weekly -# critical-failover — dirty sync — auth stays running both sides, WAL excluded +# critical-fallback — dirty sync — auth stays running both sides, WAL excluded # called by critical_sync_maintenance.sh every 15min # host1-appdata — HOST1 server-specific appdata — defined in master_host1.conf # host2-appdata — HOST2 server-specific appdata — defined in master_host2.conf # important-data — NextCloud + Postgres — NextCloud delayed start after Postgres # emby — weekly clean sync — both Emby stopped, full mirror # called by weekly_sync_maintenance.sh only — do NOT schedule separately -# emby-failover — dirty sync — Emby stays running, WAL excluded +# emby-fallback — dirty sync — Emby stays running, WAL excluded # called by critical_sync_maintenance.sh every 15min declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [critical-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" ) # Per-profile bandwidth limits in KB/s declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 # lower — runs alongside other syncs [critical-data]=9500 # high — small dataset, sync fast - [critical-failover]=9500 # high — small dataset, sync fast + [critical-fallback]=9500 # high — small dataset, sync fast [important-data]=9500 # high — database sync [emby]=8000 # medium — large full mirror - [emby-failover]=9500 # high — small critical dataset + [emby-fallback]=9500 # high — small critical dataset ) # Retry attempts per profile declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 - [critical-failover]=3 + [critical-fallback]=3 [important-data]=3 [emby]=3 - [emby-failover]=3 + [emby-fallback]=3 ) # Seconds between retry attempts declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 - [critical-failover]=120 # shorter — frequent dirty sync, retry faster + [critical-fallback]=120 # shorter — frequent dirty sync, retry faster [important-data]=300 [emby]=300 - [emby-failover]=120 # shorter — frequent dirty sync, retry faster + [emby-fallback]=120 # shorter — frequent dirty sync, retry faster ) # Containers stopped on BOTH LOCAL and REMOTE before rsync. @@ -451,10 +451,10 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" - [critical-failover]="" # dirty sync — auth stays running both sides + [critical-fallback]="" # dirty sync — auth stays running both sides [important-data]="Postgres-NextCloud NextCloud" [emby]="Emby" - [emby-failover]="" # dirty sync — Emby stays running both sides + [emby-fallback]="" # dirty sync — Emby stays running both sides ) # Containers needing a delay after rsync before starting. @@ -462,20 +462,20 @@ declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis - [critical-failover]="" + [critical-fallback]="" [important-data]="NextCloud" # wait for Postgres [emby]="" - [emby-failover]="" + [emby-fallback]="" ) # Seconds before starting delayed containers declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=15 # Mariadb + Redis need time to accept connections - [critical-failover]=5 + [critical-fallback]=5 [important-data]=10 # Postgres needs time before NextCloud [emby]=5 - [emby-failover]=5 + [emby-fallback]=5 ) # Directories excluded from rsync per profile. @@ -483,19 +483,19 @@ declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" - [critical-failover]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" + [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" [important-data]="logs *.tmp" [emby]="logs transcodes cache crash*" - [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" + [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" ) # Remote restart after dirty sync — restart these on remote IF they were running before sync. # Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. -# Used by dirty sync profiles (critical-failover, emby-failover) so remote picks up changes. +# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( - [critical-failover]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" - [emby-failover]="Emby" + [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" + [emby-fallback]="Emby" ) # Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. @@ -505,7 +505,7 @@ # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Mutual container failover between two unRAID servers. -# Each server runs Failover/failover.sh independently via array_start.sh. +# Each server runs Fallback/fallback.sh independently via array_start.sh. # All decisions based on two pings: remote reachable + internet reachable. # # States: NORMAL | FAILOVER | NO_INTERNET | DARK @@ -519,16 +519,16 @@ # Shared settings (intervals, state file, thresholds) live here. EXTERNAL_IP="8.8.8.8" - FAILOVER_CHECK_INTERVAL=120 # seconds between failover state checks - FAILOVER_HANDBACK_STRIKES=2 # consecutive healthy checks before initiating handback - FAILOVER_STATE_FILE="/boot/config/failover_state.db" - FAILOVER_ENABLED=false # HOST2 being rebuilt — set true when back online and tested + FALLBACK_CHECK_INTERVAL=120 # seconds between fallback state checks + FALLBACK_HANDBACK_STRIKES=2 # consecutive healthy checks before initiating handback + FALLBACK_STATE_FILE="/boot/config/fallback_state.db" + FALLBACK_ENABLED=false # HOST2 being rebuilt — set true when back online and tested # false = suppresses "not running" warnings in status scripts # ━━━ Failover Test ━━━ -# Controlled simulation of a failover event — run manually via failover_test.sh. - FAILOVER_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity - FAILOVER_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback +# Controlled simulation of a failover event — run manually via fallback_test.sh. + FALLBACK_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity + FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── @@ -1046,7 +1046,7 @@ DIGEST_PROFILE="weekly" # always | smart | weekly DIGEST_DAY="Sunday" DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active - DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL + DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB diff --git a/master_host1.conf b/master_host1.conf index 073e5bb..d028267 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -105,7 +105,7 @@ # Paths HOST2 should collect during the grace window after offboard. # Notified on offboard — no auto-deletion, HOST2 must collect manually within PARTNERSHIP_GRACE_HOURS. HOST1_PARTNERSHIP_MIRROR_BACKUPS=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" + # "/mnt/user/appdata-Fallback/Jayred365-Emby" ) # ============================================================================================== @@ -148,7 +148,7 @@ # Order matters — Emby first (larger transfer), then Critical-Data (auth stack). HOST1_WEEKLY_SYNC_SHARES=( "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror - "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack + "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — auth stack ) # ━━━ Critical Sync Shares ━━━ @@ -156,8 +156,8 @@ # Format: "/path/to/share" or "/path/to/share|profile-name" # Order matters — Critical-Data first (auth stack), then Emby dirty sync. HOST1_CRITICAL_SYNC_SHARES=( - "/mnt/user/appdata-Failover/Critical-Data|critical-failover" # auth dirty sync — stays running - "/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running + "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" # auth dirty sync — stays running + "/mnt/user/Media_Server/Emby|emby-fallback" # Emby dirty sync — stays running ) # ━━━ Backup Verify ━━━ @@ -172,7 +172,7 @@ # HOST1-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf. # Use for appdata unique to HOST1 (Organizrv2, VaultWarden, UptimeKuma etc.) # Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf. -# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Failover/HOST1-Appdata --profile=host1-appdata +# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST1-Appdata --profile=host1-appdata PROFILE_RSYNC_OPTS[host1-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host1-appdata]:-8000}" PROFILE_BW_LIMIT[host1-appdata]=8000 PROFILE_RETRY_COUNT[host1-appdata]=3 @@ -282,7 +282,7 @@ # ============================================================================================== # ━━━ DDNS ━━━ -# DDNS containers HOST1 manages — started/stopped by failover.sh per DDNS absolute rules: +# DDNS containers HOST1 manages — started/stopped by fallback.sh per DDNS absolute rules: # Internet loss → stop immediately # Failover → HOST2 starts HOST1's DDNS as Tier 1 (before any other containers) # Handback → stop HOST1's DDNS on HOST2 → rsync → start containers → start local DDNS last @@ -293,28 +293,28 @@ # ━━━ Internet Loss ━━━ # Containers stopped immediately on HOST1 when internet connection is lost. # Prevents external-facing services from operating without connectivity. - FAILOVER_HOST1_STOP_ON_NO_NET=( + FALLBACK_HOST1_STOP_ON_NO_NET=( "Gmer4Lfe.com" ) -# ━━━ Failover Tiers — HOST1 Runs for HOST2 ━━━ +# ━━━ Fallback Tiers — HOST1 Runs for HOST2 ━━━ # Containers HOST1 starts when HOST2 goes down. # Tier 1 is always immediate — vital services cannot wait. # Higher tiers activate after HOST2_TIER*_DELAY minutes (set in master_host2.conf). - FAILOVER_HOST1_RUNS_FOR_HOST2_TIER1=( + FALLBACK_HOST1_COVERS_HOST2_TIER1=( "Gmer4Lfe.us" "VaultWarden-Jayred365" ) - FAILOVER_HOST1_RUNS_FOR_HOST2_TIER2=( + FALLBACK_HOST1_COVERS_HOST2_TIER2=( # "container-placeholder" ) - FAILOVER_HOST1_RUNS_FOR_HOST2_TIER3=( + FALLBACK_HOST1_COVERS_HOST2_TIER3=( # "container-placeholder" ) - FAILOVER_HOST1_RUNS_FOR_HOST2_TIER4=( + FALLBACK_HOST1_COVERS_HOST2_TIER4=( # "container-placeholder" ) @@ -334,20 +334,20 @@ HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr # Tier 4 automatically syncs HOST1_DAILY_SYNC_SHARES — only list paths NOT in that array. - FAILOVER_HOST1_WRITEBACK_TIER1=( + FALLBACK_HOST1_WRITEBACK_TIER1=( "/mnt/user/Media_Server/Emby" # watch states built up during outage ) - FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage + FALLBACK_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Fallback/Important-Data" # NextCloud + Postgres — files added during outage ) - FAILOVER_HOST1_WRITEBACK_TIER3=( + FALLBACK_HOST1_WRITEBACK_TIER3=( # "location-placeholder" ) - FAILOVER_HOST1_WRITEBACK_TIER4=( - "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage + FALLBACK_HOST1_WRITEBACK_TIER4=( + "/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage ) # ============================================================================================== @@ -576,7 +576,7 @@ # /boot read-only detected → reboot immediately. # Unexpected read-only /boot means state files and config writes are silently failing. -# Failover state, watchdog reboot log, and lock files all go stale silently. +# Fallback state, watchdog reboot log, and lock files all go stale silently. HOST1_SYS_WATCHDOG_CHECK_BOOT=true # ━━━ Tier 2 — Urgent OOM Check ━━━ diff --git a/master_host2.conf b/master_host2.conf index 240e5ba..cb0272c 100644 --- a/master_host2.conf +++ b/master_host2.conf @@ -14,7 +14,7 @@ # # ── STATUS ──────────────────────────────────────────────────────────────────────────────────── # HOST2 is currently being rebuilt — most sections scaffolded, fill in when back online. -# When ready: set FAILOVER_ENABLED=true and DAILY_RSYNC_ENABLED=true in master.conf. +# When ready: set FALLBACK_ENABLED=true and DAILY_RSYNC_ENABLED=true in master.conf. # # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── # @@ -137,7 +137,7 @@ HOST2_WEEKLY_SYNC_SHARES=( # fill in when HOST2 is back online # "/mnt/user/Media_Server/Emby" - # "/mnt/user/appdata-Failover/Critical-Data" + # "/mnt/user/appdata-Fallback/Critical-Data" ) # ━━━ Critical Sync Shares ━━━ @@ -145,8 +145,8 @@ # Format: "/path/to/share" or "/path/to/share|profile-name" HOST2_CRITICAL_SYNC_SHARES=( # fill in when HOST2 is back online - # "/mnt/user/appdata-Failover/Critical-Data|critical-failover" - # "/mnt/user/Media_Server/Emby|emby-failover" + # "/mnt/user/appdata-Fallback/Critical-Data|critical-fallback" + # "/mnt/user/Media_Server/Emby|emby-fallback" ) # ━━━ Backup Verify ━━━ @@ -161,7 +161,7 @@ # HOST2-specific appdata sync profile — extends the shared PROFILE_* arrays in master.conf. # Use for appdata unique to HOST2. # Shared appdata (auth stack, Emby) use dedicated profiles defined in master.conf. -# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Failover/HOST2-Appdata --profile=host2-appdata +# Run manually: bash Rsync/rsync.sh /mnt/user/appdata-Fallback/HOST2-Appdata --profile=host2-appdata PROFILE_RSYNC_OPTS[host2-appdata]="-av --info=progress2 --bwlimit=${PROFILE_BW_LIMIT[host2-appdata]:-8000}" PROFILE_BW_LIMIT[host2-appdata]=8000 PROFILE_RETRY_COUNT[host2-appdata]=3 @@ -245,7 +245,7 @@ # ============================================================================================== # ━━━ DDNS ━━━ -# DDNS containers HOST2 manages — started/stopped by failover.sh per DDNS absolute rules: +# DDNS containers HOST2 manages — started/stopped by fallback.sh per DDNS absolute rules: # Internet loss → stop immediately # Failover → HOST1 starts HOST2's DDNS as Tier 1 (before any other containers) # Handback → stop HOST2's DDNS on HOST1 → rsync → start containers → start local DDNS last @@ -256,15 +256,15 @@ # ━━━ Internet Loss ━━━ # Containers stopped immediately on HOST2 when internet connection is lost. # Prevents external-facing services from operating without connectivity. - FAILOVER_HOST2_STOP_ON_NO_NET=( + FALLBACK_HOST2_STOP_ON_NO_NET=( "Gmer4Lfe.us" ) -# ━━━ Failover Tiers — HOST2 Runs for HOST1 ━━━ +# ━━━ Fallback Tiers — HOST2 Runs for HOST1 ━━━ # Containers HOST2 starts when HOST1 goes down. # Tier 1 is always immediate — vital services cannot wait. # Higher tiers activate after HOST1_TIER*_DELAY minutes (set in master_host1.conf). - FAILOVER_HOST2_RUNS_FOR_HOST1_TIER1=( + FALLBACK_HOST2_COVERS_HOST1_TIER1=( "Gmer4Lfe.com" "Emby" "VaultWarden-Gmer4Lfe" @@ -274,18 +274,18 @@ "ErsatzTV-Emby" ) - FAILOVER_HOST2_RUNS_FOR_HOST1_TIER2=( + FALLBACK_HOST2_COVERS_HOST1_TIER2=( "Postgres-NextCloud" "NextCloud" "PostgreSQL_Immich" "Immich-Gmer4Lfe" ) - FAILOVER_HOST2_RUNS_FOR_HOST1_TIER3=( + FALLBACK_HOST2_COVERS_HOST1_TIER3=( "Gitea" ) - FAILOVER_HOST2_RUNS_FOR_HOST1_TIER4=( + FALLBACK_HOST2_COVERS_HOST1_TIER4=( "Sonarr" "Radarr" "Lidarr" @@ -315,20 +315,20 @@ HOST2_TIER1_WRITEBACK_DELAY=60 # skip writeback if outage under 1hr # Tier 4 automatically syncs HOST2_DAILY_SYNC_SHARES — only list paths NOT in that array. - FAILOVER_HOST2_WRITEBACK_TIER1=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" + FALLBACK_HOST2_WRITEBACK_TIER1=( + # "/mnt/user/appdata-Fallback/Jayred365-Emby" ) - FAILOVER_HOST2_WRITEBACK_TIER2=( - # "/mnt/user/appdata-Failover/Jayred365-Important" + FALLBACK_HOST2_WRITEBACK_TIER2=( + # "/mnt/user/appdata-Fallback/Jayred365-Important" ) - FAILOVER_HOST2_WRITEBACK_TIER3=( + FALLBACK_HOST2_WRITEBACK_TIER3=( # "location-placeholder" ) - FAILOVER_HOST2_WRITEBACK_TIER4=( - "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage + FALLBACK_HOST2_WRITEBACK_TIER4=( + "/mnt/user/appdata-Fallback/Arrs_Stack" # arr databases — downloads queued during outage ) # ============================================================================================== diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 11abe22..7ce1761 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -19,7 +19,7 @@ # critical_sync_maintenance.sh every 15 min — auth + Emby dirty sync + partnership # arrs_failed_stalled_recovery.sh every 6 hours — blocklist + re-search failed imports # daily_sync_maintenance.sh 1am daily — git + rsync + media + restart -# rsync.sh --profile=emby-failover every 30 min — Emby watch state dirty sync +# rsync.sh --profile=emby-fallback every 30 min — Emby watch state dirty sync # weekly_sync_maintenance.sh 2:30am Sunday — clean sync + image updates # sunday_morning_coffee_report.sh 7am Sunday — full weekly digest # weekly_health_digest.sh 8am daily — profile-controlled health notification @@ -100,7 +100,7 @@ # docker_network_connect.sh ensure custom networks exist and containers are connected # system_watchdog.sh [continuous] three-tier server stability watchdog # docker_watchdog.sh [continuous] two-tier container self-healing watchdog -# failover.sh [continuous] mutual failover state machine +# fallback.sh [continuous] mutual fallback state machine # # bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh @@ -216,7 +216,7 @@ # Result: if HOST1 fails, users resume from at most 30 minutes stale. # # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ -# /mnt/user/Media_Server/Emby --profile=emby-failover +# /mnt/user/Media_Server/Emby --profile=emby-fallback # ── WEEKLY SYNC MAINTENANCE ─────────────────────────────────────────────────────────────────── @@ -284,7 +284,7 @@ # bandwidth event exceeded BANDWIDTH_WARN_GB # weekly notify once per week on DIGEST_DAY (Sunday), silent all other days # -# Data sources: failover state, watchdog strike files, container skip list, +# Data sources: fallback state, watchdog strike files, container skip list, # bandwidth log, transcode daily log, cert expiry (live openssl check) # # bash /mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh @@ -378,9 +378,9 @@ # FAILOVER # ────────────────────────────────────────────────────────────────────────────────────────────── -# failover.sh — [continuous] mutual failover state machine +# fallback.sh — [continuous] mutual fallback state machine # Started by array_start.sh on both servers independently. -# Every FAILOVER_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8 +# Every FALLBACK_CHECK_INTERVAL (120s) pings: remote Tailscale IP + 8.8.8.8 # States: NORMAL / FAILOVER / NO_INTERNET / DARK # FAILOVER: starts remote containers in tiers across 24 hours: # Tier 1 immediate: DDNS + Emby + auth (NPM/Authelia/LLDAP) + VaultWarden + Live TV @@ -391,33 +391,33 @@ # DDNS: one domain, one DDNS container active at all times. Split-brain prevented by design. # Handback: stop remote DDNS → rsync writeback → start containers → start local DDNS. # -# bash /mnt/user/appdata/unraid_scripts/Failover/failover.sh -# bash /mnt/user/appdata/unraid_scripts/Failover/failover.sh --status -# bash /mnt/user/appdata/unraid_scripts/Failover/failover.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback.sh +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback.sh --status +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback.sh --dry-run -# failover_test.sh — controlled full-lifecycle failover test using iptables simulation +# fallback_test.sh — controlled full-lifecycle fallback test using iptables simulation # ALWAYS run --dry-run first. Live test starts/stops real containers — brief service interruption. # iptables safety trap removes the block rule on ANY exit including ctrl-c and crashes. # Remote is always reachable after the test completes regardless of what happened. # 8 phases: pre-flight → block remote (iptables) → detect FAILOVER → verify Tier 1 started # → restore remote → wait handback → verify NORMAL restored → full pass/fail report. -# FAILOVER_TEST_BLOCK_WAIT must be > FAILOVER_CHECK_INTERVAL + buffer (default: 150s). -# FAILOVER_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s). +# FALLBACK_TEST_BLOCK_WAIT must be > FALLBACK_CHECK_INTERVAL + buffer (default: 150s). +# FALLBACK_TEST_HANDBACK_WAIT must cover: strike confirmation + rsync + container start (default: 360s). # -# bash /mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run -# bash /mnt/user/appdata/unraid_scripts/Failover/failover_test.sh +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_test.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_test.sh -# failover_state_reset.sh — reset failover state file to NORMAL +# fallback_state_reset.sh — reset fallback state file to NORMAL # Resets state file ONLY — does NOT start or stop any containers. -# Use when: state stuck in FAILOVER after testing, after killing failover.sh mid-cycle, +# Use when: state stuck in FAILOVER after testing, after killing fallback.sh mid-cycle, # after manual intervention left state inconsistent, after a failed handback. # Shows current state file before asking for confirmation. # Verify before resetting: right containers on right server, DDNS correct, -# both servers Tailscale visible, failover.sh not currently running. +# both servers Tailscale visible, fallback.sh not currently running. # -# bash /mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh --status -# bash /mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh -# bash /mnt/user/appdata/unraid_scripts/Failover/failover_state_reset.sh --force +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_state_reset.sh --status +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_state_reset.sh +# bash /mnt/user/appdata/unraid_scripts/Fallback/fallback_state_reset.sh --force # ────────────────────────────────────────────────────────────────────────────────────────────── @@ -436,21 +436,21 @@ # important-data Postgres-NextCloud + delayed: NextCloud # arrs_stack Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat # emby Emby both sides (weekly clean sync — both instances stopped) -# emby-failover nothing stopped (Emby stays running — dirty sync, WAL/SHM excluded) +# emby-fallback nothing stopped (Emby stays running — dirty sync, WAL/SHM excluded) # [no profile] no containers stopped (media shares, plain data) # # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ -# /mnt/user/appdata-Failover/Critical-Data +# /mnt/user/appdata-Fallback/Critical-Data # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ -# /mnt/user/appdata-Failover/Important-Data +# /mnt/user/appdata-Fallback/Important-Data # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ -# /mnt/user/appdata-Failover/Arrs_Stack +# /mnt/user/appdata-Fallback/Arrs_Stack # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ # /mnt/user/Media_Server/Emby --profile=emby # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ -# /mnt/user/Media_Server/Emby --profile=emby-failover +# /mnt/user/Media_Server/Emby --profile=emby-fallback # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh \ -# /mnt/user/appdata-Failover/Gmer4Lfe +# /mnt/user/appdata-Fallback/Gmer4Lfe # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Movies # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Tv_Shows # bash /mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/Music @@ -897,7 +897,7 @@ # Running state, PID, uptime, approximate cycle count # Active strikes per check. Required container states. Memory-monitored container usage. # Skip list contents (any entry needs human attention). -# Failover state, active tiers, outage duration, remote Tailscale visibility. +# Fallback state, active tiers, outage duration, remote Tailscale visibility. # No schedule — on demand only. Run any time you want a complete status snapshot. # # bash /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh @@ -1021,7 +1021,7 @@ # Orchestrators/critical_sync_maintenance.sh # # */30 * * * * every 30 minutes: -# Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-failover +# Rsync/rsync.sh /mnt/user/Media_Server/Emby --profile=emby-fallback # # 0 */6 * * * every 6 hours: # Orchestrators/arrs_failed_stalled_recovery.sh