#!/bin/bash # ============================================================================================== # ============================= fallback State Reset =========================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Resets the fallback state file to NORMAL and clears all tier flags. Use when # the state file is stuck after fallback testing, a failed handback, manual # intervention, or fallback.sh being killed mid-cycle. # # Writes a fresh state file with: # state=NORMAL / fallback_start=0 / handback_strikes=0 # tier2_started=false / tier3_started=false / tier4_started=false # # Does NOT start or stop containers — state file only. After reset, fallback.sh # resumes from NORMAL on its next cycle. # # WARNING: Only run when you have verified the stack is actually in a normal # state — right containers on the right server, DDNS correct, no active fallback # in progress. Resetting state during a real fallback causes fallback.sh to stop # covering the remote server until the next detection cycle. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Active Fallback Detection # Checks whether fallback.sh is currently running and warns if so. A reset # during an active cycle causes fallback.sh to lose its state on the next read. # # Single Instance Lock # acquire_lock prevents concurrent resets. # # flock on State Write # The state file write is protected with flock — prevents a race condition # with fallback.sh reading the file mid-cycle. # # Confirmation Required # Interactive mode prompts for YES before writing. Use --force to bypass in # non-interactive contexts (cron, scripts). # # Notification Validated # platform_require_cmd confirms the notify script is present before use. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # fallback_state_reset.sh # Show current state and prompt for YES before resetting. # # fallback_state_reset.sh --dry-run # Show current state and what the new state file would contain. No write. # # fallback_state_reset.sh --status # Show current state file contents and exit. # # fallback_state_reset.sh --force # Reset without interactive confirmation. Safe for scripted use. # # fallback_state_reset.sh --force --dry-run # Dry run without the confirmation prompt. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # ── Handle --force flag before parse_args ───────────────────────────────────────────────────── FORCE=false FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --force) FORCE=true ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock # detect_hosts() sets MY_ID — used in summary and notification detect_hosts log "$ICON_GEAR Config: state-file=${FALLBACK_STATE_FILE}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" [[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped" # ============================================================================================== # ━━━ Current State ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_FALLBACK Current Fallback State ━━━" echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo "" if [[ ! -f "$FALLBACK_STATE_FILE" ]]; then warn "State file not found: $FALLBACK_STATE_FILE" warn "Will be created fresh on reset" CURRENT_STATE="NOT FOUND" else log "State file: $FALLBACK_STATE_FILE" echo "" while IFS='=' read -r key value; do [[ -z "$key" ]] && continue echo " $ICON_INFO $key = $value" done < "$FALLBACK_STATE_FILE" CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) fi if [[ "$SHOW_STATUS" == true ]]; then echo "" # Check if fallback.sh is running — informational in status mode if pgrep -f "fallback.sh" >/dev/null 2>&1; then warn "fallback.sh is currently RUNNING — any reset would race with active cycle" else echo "fallback.sh is not running ✅" fi exit 0 fi # ============================================================================================== # ━━━ Safety Checks ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SHIELD Safety Checks ━━━" # Check if fallback.sh is actively running FALLBACK_RUNNING=false if pgrep -f "fallback.sh" >/dev/null 2>&1; then FALLBACK_RUNNING=true warn "⚠️ fallback.sh is currently RUNNING" warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration" warn "Consider stopping fallback.sh first: kill \$(pgrep -f fallback.sh)" warn "Then reset state, then restart: bash Fallback/fallback.sh &" echo "" warn "If you are sure you want to proceed anyway, confirm below" else echo "fallback.sh is not running — safe to reset ✅" fi # Check current state — if already NORMAL warn user if [[ "$CURRENT_STATE" == "NORMAL" ]]; then warn "State is already NORMAL — reset may not be necessary" warn "Proceeding anyway (will refresh the state file)" fi # ============================================================================================== # ━━━ Confirmation ━━━ # ============================================================================================== echo "" warn "This will reset fallback state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)" warn "Verify before proceeding:" warn " ✓ Right containers running on the right server" warn " ✓ DDNS pointing at correct server" warn " ✓ No real fallback actually in progress" warn " ✓ Both servers can reach each other" echo "" if [[ "$DRY_RUN" == false ]]; then if [[ "$FORCE" == true ]]; then log "FORCE flag set — skipping confirmation prompt" elif [[ -t 0 ]]; then # Interactive terminal — prompt for confirmation read -r -p "Type YES to confirm reset: " CONFIRM if [[ "$CONFIRM" != "YES" ]]; then warn "Reset cancelled" exit 0 fi else # Non-interactive — no terminal, cannot prompt error "Non-interactive mode — use --force flag to skip confirmation" error "Usage: fallback_state_reset.sh --force" exit 1 fi fi # ============================================================================================== # ━━━ Reset State File ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_FALLBACK Resetting State File ━━━" NEW_STATE_CONTENT="state=NORMAL fallback_start=0 handback_strikes=0 tier2_started=false tier3_started=false tier4_started=false last_reset=$(date '+%Y-%m-%d %H:%M:%S') reset_by=$MY_ID" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would write to $FALLBACK_STATE_FILE:" echo "" echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do echo " $line" done else mkdir -p "$(dirname "$FALLBACK_STATE_FILE")" # flock prevents race with fallback.sh mid-cycle read/write ( flock -x 200 echo "$NEW_STATE_CONTENT" > "$FALLBACK_STATE_FILE" ) 200>"${FALLBACK_STATE_FILE}.lock" warn "State file reset to NORMAL ✅" log "Written to: $FALLBACK_STATE_FILE" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY FALLBACK STATE RESET SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_FALLBACK File: $FALLBACK_STATE_FILE" echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else warn "$ICON_DONE State reset to NORMAL" echo "fallback.sh will resume from NORMAL on next cycle" echo "No containers were started or stopped" echo "" [[ "$FALLBACK_RUNNING" == true ]] && \ warn "⚠️ fallback.sh was running during reset — monitor next cycle carefully" notify "Fallback state manually reset to NORMAL on $(hostname) ($MY_ID)" \ "Fallback State Reset" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0