A fallback dry run wrote real reboot-surviving state, and a failed tier writeback reported itself as a dry run

This commit is contained in:
Gmer4Lfe
2026-08-22 00:00:56 -04:00
parent 387382ec9f
commit 41c0598220
2 changed files with 47 additions and 2 deletions
+26 -2
View File
@@ -368,6 +368,22 @@ state_set() {
}
state_init() {
# A dry run must not leave the host believing it failed over. state_set() writes
# unconditionally, and this file survives reboots and is what the real daemon — and the
# Monitor and Fallback cards — read to decide what is happening. A --dry-run walk through
# FAILOVER would have written state=FALLBACK, the tier flags and the strike counter into it
# for real, and nothing would have put them back.
#
# Copied rather than merely redirected, so the preview still starts from the live state and
# can advance through tiers exactly as a real run would. The copy lands in the RAM cache and
# dies with the reboot.
if [[ "$DRY_RUN" == true ]]; then
local live="$FALLBACK_STATE_FILE"
FALLBACK_STATE_FILE="${VV_CACHE_ROOT:-/tmp/varaverk}/fallback_state.dryrun.$$"
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
if [[ -f "$live" ]]; then cp -f "$live" "$FALLBACK_STATE_FILE"; else : > "$FALLBACK_STATE_FILE"; fi
warn "DRY RUN — state writes redirected to $FALLBACK_STATE_FILE (live state untouched)"
fi
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
@@ -759,8 +775,16 @@ run_handback() {
for job in "${jobs[@]}"; do
[[ -z "$job" ]] && continue
log "Syncing: $job"
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|| warn "DRY RUN — would rsync: $job"
# if/else, not A && B || C. In the shorthand a REAL run whose rsync exits
# non-zero falls through to the || branch and logs "DRY RUN — would rsync",
# so a failed Tier writeback reported itself as a preview and the real
# failure went unsaid. The Tier 1 block below always had this right.
if [[ "$DRY_RUN" == false ]]; then
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|| error "Tier $tier writeback FAILED: $job"
else
warn "DRY RUN — would rsync: $job"
fi
done
else
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
+21
View File
@@ -325,6 +325,27 @@ else
warn "No state file found — assuming NORMAL (first run)"
fi
# fallback.sh must actually be RUNNING, not merely enabled
#
# Every phase after this one waits for the daemon to change state. FALLBACK_ENABLED=true says
# it is allowed to run; it does not say array_started.sh launched it, or that it is still alive.
# Without this the test passes pre-flight, drops a real iptables rule on the partner, waits
# FALLBACK_TEST_BLOCK_WAIT for a transition nothing is there to make, and fails Phase 3 blaming
# fallback detection. Only the EXIT trap gets connectivity back.
#
# In --dry-run nothing is blocked and nothing is waited on, so a dead daemon is worth saying but
# not worth aborting for — the walkthrough still shows the operator the shape of the run.
if pgrep -f "Fallback/fallback\.sh" >/dev/null 2>&1; then
log "fallback.sh daemon is running"
elif [[ "$DRY_RUN" == true ]]; then
warn "fallback.sh is NOT running — a real test would abort here"
else
error "fallback.sh is not running — nothing would detect the outage this test creates"
error "Start it with array_started.sh, or run with --dry-run to walk the phases"
phase_fail "Pre-flight"
exit 1
fi
# Tier 1 containers configured
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
error "No Tier 1 containers configured for $MY_ID$REMOTE_ID"