massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+168
-66
@@ -1,127 +1,229 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Failover State Reset ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Failover State Reset ===========================================
|
||||
# ==============================================================================================
|
||||
# Resets the failover state file to NORMAL and clears all tier flags.
|
||||
# Use when the failover state file is stuck in a non-NORMAL state after testing,
|
||||
# a failed handback, or manual intervention that left state inconsistent.
|
||||
# Use when the failover state file is stuck in a non-NORMAL state after:
|
||||
# - Failover testing that left state as FAILOVER
|
||||
# - A failed handback that did not complete cleanly
|
||||
# - Manual intervention that left state inconsistent
|
||||
# - failover.sh was killed mid-cycle and state is unknown
|
||||
#
|
||||
# ── WHAT THIS DOES ────────────────────────────────────────────────────────────────────────────
|
||||
# Writes a fresh state file with:
|
||||
# state=NORMAL
|
||||
# failover_start=0
|
||||
# handback_strikes=0
|
||||
# tier2_started=false / tier3_started=false / tier4_started=false
|
||||
#
|
||||
# Does NOT start or stop any containers — state file only.
|
||||
# After reset, failover.sh will resume from NORMAL on its next cycle.
|
||||
#
|
||||
# ⚠️ Only run this when you have manually verified both servers are in their
|
||||
# correct states — right containers running on the right server, DDNS correct.
|
||||
# Resetting state without verifying the actual state can cause failover.sh
|
||||
# to make incorrect decisions on its next cycle.
|
||||
# ── ⚠️ ONLY RUN WHEN SAFE ────────────────────────────────────────────────────────────────────
|
||||
# Verify BEFORE resetting:
|
||||
# ✓ Right containers running on the right server
|
||||
# ✓ DDNS pointing at the correct server
|
||||
# ✓ No active failover actually in progress
|
||||
# ✓ Both servers can see each other
|
||||
#
|
||||
# Supports --dry-run to show what would be reset without changing anything.
|
||||
# Supports --status to show the current state file contents.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Resetting state while a real failover is happening causes failover.sh to stop
|
||||
# covering the remote server — services go offline until next detection cycle.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# failover.sh running check — warns if failover.sh is active when reset is attempted
|
||||
# acquire_lock — prevents concurrent resets
|
||||
# flock on state write — prevents race with failover.sh mid-cycle read
|
||||
# Confirmation required — interactive: type YES | non-interactive: --force flag
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# failover_state_reset.sh — interactive reset (prompts for YES)
|
||||
# failover_state_reset.sh --dry-run — show current state, show what would be written
|
||||
# failover_state_reset.sh --status — show current state file contents and exit
|
||||
# failover_state_reset.sh --force — non-interactive reset (no prompt, use in scripts)
|
||||
# failover_state_reset.sh --force --dry-run — dry run without prompt
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
# ── Handle --force flag before parse_args ─────────────────────────────────────────────────────
|
||||
FORCE=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--force) FORCE=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Current State ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in summary and notification
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Current State ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Current Failover State ━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "━━━ $ICON_SUMMARY Current State ━━━"
|
||||
|
||||
if [[ ! -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
warn "State file not found: $FAILOVER_STATE_FILE"
|
||||
warn "Will be created fresh on reset"
|
||||
CURRENT_STATE="NOT FOUND"
|
||||
else
|
||||
info "State file: $FAILOVER_STATE_FILE"
|
||||
log "State file: $FAILOVER_STATE_FILE"
|
||||
echo ""
|
||||
while IFS='=' read -r key value; do
|
||||
[[ -z "$key" ]] && continue
|
||||
echo " $ICON_INFO $key = $value"
|
||||
done < "$FAILOVER_STATE_FILE"
|
||||
CURRENT_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
fi
|
||||
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
# Check if failover.sh is running — informational in status mode
|
||||
if pgrep -f "failover.sh" >/dev/null 2>&1; then
|
||||
warn "failover.sh is currently RUNNING — any reset would race with active cycle"
|
||||
else
|
||||
log "failover.sh is not running"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Confirmation ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Checks ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
warn "$ICON_WARN This will reset the failover state to NORMAL"
|
||||
warn "Only proceed if you have verified both servers are in their correct states"
|
||||
warn " — Right containers running on the right server"
|
||||
warn " — DDNS pointing at the correct server"
|
||||
warn " — No active failover in progress"
|
||||
echo "━━━ $ICON_SHIELD Safety Checks ━━━"
|
||||
|
||||
# Check if failover.sh is actively running
|
||||
FAILOVER_RUNNING=false
|
||||
if pgrep -f "failover.sh" >/dev/null 2>&1; then
|
||||
FAILOVER_RUNNING=true
|
||||
warn "⚠️ failover.sh is currently RUNNING"
|
||||
warn "Resetting state mid-cycle may cause incorrect decisions on the next iteration"
|
||||
warn "Consider stopping failover.sh first (click Abort in User Scripts)"
|
||||
warn "Then reset state, then restart failover.sh"
|
||||
echo ""
|
||||
warn "If you are sure you want to proceed anyway, confirm below"
|
||||
else
|
||||
log "failover.sh is not running — safe to reset ✅"
|
||||
fi
|
||||
|
||||
# Check current state — if already NORMAL warn user
|
||||
if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
|
||||
warn "State is already NORMAL — reset may not be necessary"
|
||||
warn "Proceeding anyway (will refresh the state file)"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Confirmation ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
warn "This will reset failover state to NORMAL on $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
warn "Verify before proceeding:"
|
||||
warn " ✓ Right containers running on the right server"
|
||||
warn " ✓ DDNS pointing at correct server"
|
||||
warn " ✓ No real failover actually in progress"
|
||||
warn " ✓ Both servers can reach each other"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
read -r -p "Type YES to confirm reset: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
info "Reset cancelled"
|
||||
exit 0
|
||||
if [[ "$FORCE" == true ]]; then
|
||||
log "FORCE flag set — skipping confirmation prompt"
|
||||
elif [[ -t 0 ]]; then
|
||||
# Interactive terminal — prompt for confirmation
|
||||
read -r -p "Type YES to confirm reset: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
warn "Reset cancelled"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
# Non-interactive — no terminal, cannot prompt
|
||||
error "Non-interactive mode — use --force flag to skip confirmation"
|
||||
error "Usage: failover_state_reset.sh --force"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_FAILOVER Reset State File ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Reset State File ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Resetting State File ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would write:"
|
||||
echo " state=NORMAL"
|
||||
echo " failover_start=0"
|
||||
echo " handback_strikes=0"
|
||||
echo " tier2_started=false"
|
||||
echo " tier3_started=false"
|
||||
echo " tier4_started=false"
|
||||
echo " last_reset=$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
else
|
||||
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
|
||||
cat > "$FAILOVER_STATE_FILE" << EOF
|
||||
state=NORMAL
|
||||
NEW_STATE_CONTENT="state=NORMAL
|
||||
failover_start=0
|
||||
handback_strikes=0
|
||||
tier2_started=false
|
||||
tier3_started=false
|
||||
tier4_started=false
|
||||
last_reset=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
EOF
|
||||
success "State file reset to NORMAL"
|
||||
reset_by=$MY_ID"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would write to $FAILOVER_STATE_FILE:"
|
||||
echo ""
|
||||
echo "$NEW_STATE_CONTENT" | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
else
|
||||
mkdir -p "$(dirname "$FAILOVER_STATE_FILE")"
|
||||
|
||||
# flock prevents race with failover.sh mid-cycle read/write
|
||||
(
|
||||
flock -x 200
|
||||
echo "$NEW_STATE_CONTENT" > "$FAILOVER_STATE_FILE"
|
||||
) 200>"${FAILOVER_STATE_FILE}.lock"
|
||||
|
||||
warn "State file reset to NORMAL ✅"
|
||||
log "Written to: $FAILOVER_STATE_FILE"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY FAILOVER STATE RESET SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_FAILOVER File: $FAILOVER_STATE_FILE"
|
||||
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
warn "DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS State reset to NORMAL"
|
||||
echo "$ICON_TIME Reset at: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
warn "$ICON_DONE State reset to NORMAL"
|
||||
log "failover.sh will resume from NORMAL on next cycle"
|
||||
log "No containers were started or stopped"
|
||||
echo ""
|
||||
echo "$ICON_INFO failover.sh will resume from NORMAL on next cycle"
|
||||
echo "$ICON_INFO No containers were started or stopped"
|
||||
notify "Failover state manually reset to NORMAL on $(hostname)" "Failover State Reset" "normal"
|
||||
[[ "$FAILOVER_RUNNING" == true ]] && \
|
||||
warn "⚠️ failover.sh was running during reset — monitor next cycle carefully"
|
||||
notify "Failover state manually reset to NORMAL on $(hostname) ($MY_ID)" \
|
||||
"Failover State Reset" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
Reference in New Issue
Block a user