250 lines
9.6 KiB
Bash
250 lines
9.6 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Failover State Reset ===========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Resets the fallback state file to NORMAL and clears all tier flags. Use when
|
|
# the state file is stuck after failover testing, a failed handback, manual
|
|
# intervention, or fallback.sh being killed mid-cycle.
|
|
#
|
|
# Writes a fresh state file with:
|
|
# state=NORMAL / fallback_start=0 / handback_strikes=0
|
|
# tier2_started=false / tier3_started=false / tier4_started=false
|
|
#
|
|
# Does NOT start or stop containers — state file only. After reset, fallback.sh
|
|
# resumes from NORMAL on its next cycle.
|
|
#
|
|
# WARNING: Only run when you have verified the stack is actually in a normal
|
|
# state — right containers on the right server, DDNS correct, no active failover
|
|
# in progress. Resetting state during a real failover causes fallback.sh to stop
|
|
# covering the remote server until the next detection cycle.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Active Fallback Detection
|
|
# Checks whether fallback.sh is currently running and warns if so. A reset
|
|
# during an active cycle causes fallback.sh to lose its state on the next read.
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent resets.
|
|
#
|
|
# flock on State Write
|
|
# The state file write is protected with flock — prevents a race condition
|
|
# with fallback.sh reading the file mid-cycle.
|
|
#
|
|
# Confirmation Required
|
|
# Interactive mode prompts for YES before writing. Use --force to bypass in
|
|
# non-interactive contexts (cron, scripts).
|
|
#
|
|
# Notification Validated
|
|
# validate_unraid_cmd confirms the notify script is present before use.
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# fallback_state_reset.sh
|
|
# Show current state and prompt for YES before resetting.
|
|
#
|
|
# fallback_state_reset.sh --dry-run
|
|
# Show current state and what the new state file would contain. No write.
|
|
#
|
|
# fallback_state_reset.sh --status
|
|
# Show current state file contents and exit.
|
|
#
|
|
# fallback_state_reset.sh --force
|
|
# Reset without interactive confirmation. Safe for scripted use.
|
|
#
|
|
# fallback_state_reset.sh --force --dry-run
|
|
# Dry run without the confirmation prompt.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
# ── Handle --force flag before parse_args ─────────────────────────────────────────────────────
|
|
FORCE=false
|
|
FILTERED_ARGS=()
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--force) FORCE=true ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID — used in summary and notification
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Current State ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Current Fallback State ━━━"
|
|
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
|
|
if [[ ! -f "$FALLBACK_STATE_FILE" ]]; then
|
|
warn "State file not found: $FALLBACK_STATE_FILE"
|
|
warn "Will be created fresh on reset"
|
|
CURRENT_STATE="NOT FOUND"
|
|
else
|
|
log "State file: $FALLBACK_STATE_FILE"
|
|
echo ""
|
|
while IFS='=' read -r key value; do
|
|
[[ -z "$key" ]] && continue
|
|
echo " $ICON_INFO $key = $value"
|
|
done < "$FALLBACK_STATE_FILE"
|
|
CURRENT_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
|
fi
|
|
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
# Check if fallback.sh is running — informational in status mode
|
|
if pgrep -f "fallback.sh" >/dev/null 2>&1; then
|
|
warn "fallback.sh is currently RUNNING — any reset would race with active cycle"
|
|
else
|
|
log "fallback.sh is not running"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Safety Checks ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
|
|
|
# Check if fallback.sh is actively running
|
|
FALLBACK_RUNNING=false
|
|
if pgrep -f "fallback.sh" >/dev/null 2>&1; then
|
|
FALLBACK_RUNNING=true
|
|
warn "⚠️ fallback.sh is currently RUNNING"
|
|
warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration"
|
|
warn "Consider stopping fallback.sh first (click Abort in User Scripts)"
|
|
warn "Then reset state, then restart fallback.sh"
|
|
echo ""
|
|
warn "If you are sure you want to proceed anyway, confirm below"
|
|
else
|
|
echo "fallback.sh is not running — safe to reset ✅"
|
|
fi
|
|
|
|
# Check current state — if already NORMAL warn user
|
|
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
|
warn "State is already NORMAL — reset may not be necessary"
|
|
warn "Proceeding anyway (will refresh the state file)"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Confirmation ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
warn "This will reset fallback state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)"
|
|
warn "Verify before proceeding:"
|
|
warn " ✓ Right containers running on the right server"
|
|
warn " ✓ DDNS pointing at correct server"
|
|
warn " ✓ No real fallback actually in progress"
|
|
warn " ✓ Both servers can reach each other"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
if [[ "$FORCE" == true ]]; then
|
|
log "FORCE flag set — skipping confirmation prompt"
|
|
elif [[ -t 0 ]]; then
|
|
# Interactive terminal — prompt for confirmation
|
|
read -r -p "Type YES to confirm reset: " CONFIRM
|
|
if [[ "$CONFIRM" != "YES" ]]; then
|
|
warn "Reset cancelled"
|
|
exit 0
|
|
fi
|
|
else
|
|
# Non-interactive — no terminal, cannot prompt
|
|
error "Non-interactive mode — use --force flag to skip confirmation"
|
|
error "Usage: fallback_state_reset.sh --force"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Reset State File ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Resetting State File ━━━"
|
|
|
|
NEW_STATE_CONTENT="state=NORMAL
|
|
fallback_start=0
|
|
handback_strikes=0
|
|
tier2_started=false
|
|
tier3_started=false
|
|
tier4_started=false
|
|
last_reset=$(date '+%Y-%m-%d %H:%M:%S')
|
|
reset_by=$MY_ID"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would write to $FALLBACK_STATE_FILE:"
|
|
echo ""
|
|
echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
else
|
|
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
|
|
|
# flock prevents race with fallback.sh mid-cycle read/write
|
|
(
|
|
flock -x 200
|
|
echo "$NEW_STATE_CONTENT" > "$FALLBACK_STATE_FILE"
|
|
) 200>"${FALLBACK_STATE_FILE}.lock"
|
|
|
|
warn "State file reset to NORMAL ✅"
|
|
log "Written to: $FALLBACK_STATE_FILE"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY FALLBACK STATE RESET SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_FALLBACK File: $FALLBACK_STATE_FILE"
|
|
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
else
|
|
warn "$ICON_DONE State reset to NORMAL"
|
|
echo "fallback.sh will resume from NORMAL on next cycle"
|
|
echo "No containers were started or stopped"
|
|
echo ""
|
|
[[ "$FALLBACK_RUNNING" == true ]] && \
|
|
warn "⚠️ fallback.sh was running during reset — monitor next cycle carefully"
|
|
notify "Fallback state manually reset to NORMAL on $(hostname) ($MY_ID)" \
|
|
"Fallback State Reset" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0 |