A fallback dry run wrote real reboot-surviving state, and a failed tier writeback reported itself as a dry run
This commit is contained in:
+26
-2
@@ -368,6 +368,22 @@ state_set() {
|
||||
}
|
||||
|
||||
state_init() {
|
||||
# A dry run must not leave the host believing it failed over. state_set() writes
|
||||
# unconditionally, and this file survives reboots and is what the real daemon — and the
|
||||
# Monitor and Fallback cards — read to decide what is happening. A --dry-run walk through
|
||||
# FAILOVER would have written state=FALLBACK, the tier flags and the strike counter into it
|
||||
# for real, and nothing would have put them back.
|
||||
#
|
||||
# Copied rather than merely redirected, so the preview still starts from the live state and
|
||||
# can advance through tiers exactly as a real run would. The copy lands in the RAM cache and
|
||||
# dies with the reboot.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
local live="$FALLBACK_STATE_FILE"
|
||||
FALLBACK_STATE_FILE="${VV_CACHE_ROOT:-/tmp/varaverk}/fallback_state.dryrun.$$"
|
||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||
if [[ -f "$live" ]]; then cp -f "$live" "$FALLBACK_STATE_FILE"; else : > "$FALLBACK_STATE_FILE"; fi
|
||||
warn "DRY RUN — state writes redirected to $FALLBACK_STATE_FILE (live state untouched)"
|
||||
fi
|
||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||||
@@ -759,8 +775,16 @@ run_handback() {
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||
|| warn "DRY RUN — would rsync: $job"
|
||||
# if/else, not A && B || C. In the shorthand a REAL run whose rsync exits
|
||||
# non-zero falls through to the || branch and logs "DRY RUN — would rsync",
|
||||
# so a failed Tier writeback reported itself as a preview and the real
|
||||
# failure went unsaid. The Tier 1 block below always had this right.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||
|| error "Tier $tier writeback FAILED: $job"
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||||
|
||||
@@ -325,6 +325,27 @@ else
|
||||
warn "No state file found — assuming NORMAL (first run)"
|
||||
fi
|
||||
|
||||
# fallback.sh must actually be RUNNING, not merely enabled
|
||||
#
|
||||
# Every phase after this one waits for the daemon to change state. FALLBACK_ENABLED=true says
|
||||
# it is allowed to run; it does not say array_started.sh launched it, or that it is still alive.
|
||||
# Without this the test passes pre-flight, drops a real iptables rule on the partner, waits
|
||||
# FALLBACK_TEST_BLOCK_WAIT for a transition nothing is there to make, and fails Phase 3 blaming
|
||||
# fallback detection. Only the EXIT trap gets connectivity back.
|
||||
#
|
||||
# In --dry-run nothing is blocked and nothing is waited on, so a dead daemon is worth saying but
|
||||
# not worth aborting for — the walkthrough still shows the operator the shape of the run.
|
||||
if pgrep -f "Fallback/fallback\.sh" >/dev/null 2>&1; then
|
||||
log "fallback.sh daemon is running"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "fallback.sh is NOT running — a real test would abort here"
|
||||
else
|
||||
error "fallback.sh is not running — nothing would detect the outage this test creates"
|
||||
error "Start it with array_started.sh, or run with --dry-run to walk the phases"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Tier 1 containers configured
|
||||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||||
|
||||
Reference in New Issue
Block a user