#!/bin/bash # ============================================================================================== # ============================= Failover State Reset =========================================== # ============================================================================================== # Resets the fallback state file to NORMAL and clears all tier flags. # Use when the fallback state file is stuck in a non-NORMAL state after: # - Failover testing that left state as FAILOVER # - A failed handback that did not complete cleanly # - Manual intervention that left state inconsistent # - fallback.sh was killed mid-cycle and state is unknown # # ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── # Writes a fresh state file with: # state=NORMAL # fallback_start=0 # handback_strikes=0 # tier2_started=false / tier3_started=false / tier4_started=false # # Does NOT start or stop any containers — state file only. # After reset, fallback.sh will resume from NORMAL on its next cycle. # # ── ⚠️ ONLY RUN WHEN SAFE ──────────────────────────────────────────────────────────────────── # Verify BEFORE resetting: # ✓ Right containers running on the right server # ✓ DDNS pointing at the correct server # ✓ No active failover actually in progress # ✓ Both servers can see each other # # Resetting state while a real fallback is happening causes fallback.sh to stop # covering the remote server — services go offline until next detection cycle. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # fallback.sh running check — warns if fallback.sh is active when reset is attempted # acquire_lock — prevents concurrent resets # flock on state write — prevents race with fallback.sh mid-cycle read # Confirmation required — interactive: type YES | non-interactive: --force flag # validate_unraid_cmd — notify validated before use # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # fallback_state_reset.sh — interactive reset (prompts for YES) # fallback_state_reset.sh --dry-run — show current state, show what would be written # fallback_state_reset.sh --status — show current state file contents and exit # fallback_state_reset.sh --force — non-interactive reset (no prompt, use in scripts) # fallback_state_reset.sh --force --dry-run — dry run without prompt # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # ── Handle --force flag before parse_args ───────────────────────────────────────────────────── FORCE=false FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --force) FORCE=true ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock # detect_hosts() sets MY_ID — used in summary and notification detect_hosts [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" [[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped" # ============================================================================================== # ━━━ Current State ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_FALLBACK Current Fallback State ━━━" echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "" if [[ ! -f "$FALLBACK_STATE_FILE" ]]; then warn "State file not found: $FALLBACK_STATE_FILE" warn "Will be created fresh on reset" CURRENT_STATE="NOT FOUND" else log "State file: $FALLBACK_STATE_FILE" echo "" while IFS='=' read -r key value; do [[ -z "$key" ]] && continue echo " $ICON_INFO $key = $value" done < "$FALLBACK_STATE_FILE" CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) fi if [[ "$SHOW_STATUS" == true ]]; then echo "" # Check if fallback.sh is running — informational in status mode if pgrep -f "fallback.sh" >/dev/null 2>&1; then warn "fallback.sh is currently RUNNING — any reset would race with active cycle" else log "fallback.sh is not running" fi exit 0 fi # ============================================================================================== # ━━━ Safety Checks ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" # Check if fallback.sh is actively running FALLBACK_RUNNING=false if pgrep -f "fallback.sh" >/dev/null 2>&1; then FALLBACK_RUNNING=true warn "⚠️ fallback.sh is currently RUNNING" warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration" warn "Consider stopping fallback.sh first (click Abort in User Scripts)" warn "Then reset state, then restart fallback.sh" echo "" warn "If you are sure you want to proceed anyway, confirm below" else log "fallback.sh is not running — safe to reset ✅" fi # Check current state — if already NORMAL warn user if [[ "$CURRENT_STATE" == "NORMAL" ]]; then warn "State is already NORMAL — reset may not be necessary" warn "Proceeding anyway (will refresh the state file)" fi # ============================================================================================== # ━━━ Confirmation ━━━ # ============================================================================================== echo "" warn "This will reset fallback state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)" warn "Verify before proceeding:" warn " ✓ Right containers running on the right server" warn " ✓ DDNS pointing at correct server" warn " ✓ No real fallback actually in progress" warn " ✓ Both servers can reach each other" echo "" if [[ "$DRY_RUN" == false ]]; then if [[ "$FORCE" == true ]]; then log "FORCE flag set — skipping confirmation prompt" elif [[ -t 0 ]]; then # Interactive terminal — prompt for confirmation read -r -p "Type YES to confirm reset: " CONFIRM if [[ "$CONFIRM" != "YES" ]]; then warn "Reset cancelled" exit 0 fi else # Non-interactive — no terminal, cannot prompt error "Non-interactive mode — use --force flag to skip confirmation" error "Usage: fallback_state_reset.sh --force" exit 1 fi fi # ============================================================================================== # ━━━ Reset State File ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_FALLBACK Resetting State File ━━━" NEW_STATE_CONTENT="state=NORMAL fallback_start=0 handback_strikes=0 tier2_started=false tier3_started=false tier4_started=false last_reset=$(date '+%Y-%m-%d %H:%M:%S') reset_by=$MY_ID" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would write to $FALLBACK_STATE_FILE:" echo "" echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do echo " $line" done else mkdir -p "$(dirname "$FALLBACK_STATE_FILE")" # flock prevents race with fallback.sh mid-cycle read/write ( flock -x 200 echo "$NEW_STATE_CONTENT" > "$FALLBACK_STATE_FILE" ) 200>"${FALLBACK_STATE_FILE}.lock" warn "State file reset to NORMAL ✅" log "Written to: $FALLBACK_STATE_FILE" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY FALLBACK STATE RESET SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_FALLBACK File: $FALLBACK_STATE_FILE" echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else warn "$ICON_DONE State reset to NORMAL" log "fallback.sh will resume from NORMAL on next cycle" log "No containers were started or stopped" echo "" [[ "$FALLBACK_RUNNING" == true ]] && \ warn "⚠️ fallback.sh was running during reset — monitor next cycle carefully" notify "Fallback state manually reset to NORMAL on $(hostname) ($MY_ID)" \ "Fallback State Reset" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"