A fallback dry run wrote real reboot-surviving state, and a failed tier writeback reported itself as a dry run
This commit is contained in:
+26
-2
@@ -368,6 +368,22 @@ state_set() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
state_init() {
|
state_init() {
|
||||||
|
# A dry run must not leave the host believing it failed over. state_set() writes
|
||||||
|
# unconditionally, and this file survives reboots and is what the real daemon — and the
|
||||||
|
# Monitor and Fallback cards — read to decide what is happening. A --dry-run walk through
|
||||||
|
# FAILOVER would have written state=FALLBACK, the tier flags and the strike counter into it
|
||||||
|
# for real, and nothing would have put them back.
|
||||||
|
#
|
||||||
|
# Copied rather than merely redirected, so the preview still starts from the live state and
|
||||||
|
# can advance through tiers exactly as a real run would. The copy lands in the RAM cache and
|
||||||
|
# dies with the reboot.
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
local live="$FALLBACK_STATE_FILE"
|
||||||
|
FALLBACK_STATE_FILE="${VV_CACHE_ROOT:-/tmp/varaverk}/fallback_state.dryrun.$$"
|
||||||
|
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||||
|
if [[ -f "$live" ]]; then cp -f "$live" "$FALLBACK_STATE_FILE"; else : > "$FALLBACK_STATE_FILE"; fi
|
||||||
|
warn "DRY RUN — state writes redirected to $FALLBACK_STATE_FILE (live state untouched)"
|
||||||
|
fi
|
||||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||||||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||||||
@@ -759,8 +775,16 @@ run_handback() {
|
|||||||
for job in "${jobs[@]}"; do
|
for job in "${jobs[@]}"; do
|
||||||
[[ -z "$job" ]] && continue
|
[[ -z "$job" ]] && continue
|
||||||
log "Syncing: $job"
|
log "Syncing: $job"
|
||||||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
# if/else, not A && B || C. In the shorthand a REAL run whose rsync exits
|
||||||
|| warn "DRY RUN — would rsync: $job"
|
# non-zero falls through to the || branch and logs "DRY RUN — would rsync",
|
||||||
|
# so a failed Tier writeback reported itself as a preview and the real
|
||||||
|
# failure went unsaid. The Tier 1 block below always had this right.
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||||
|
|| error "Tier $tier writeback FAILED: $job"
|
||||||
|
else
|
||||||
|
warn "DRY RUN — would rsync: $job"
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||||||
|
|||||||
@@ -325,6 +325,27 @@ else
|
|||||||
warn "No state file found — assuming NORMAL (first run)"
|
warn "No state file found — assuming NORMAL (first run)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# fallback.sh must actually be RUNNING, not merely enabled
|
||||||
|
#
|
||||||
|
# Every phase after this one waits for the daemon to change state. FALLBACK_ENABLED=true says
|
||||||
|
# it is allowed to run; it does not say array_started.sh launched it, or that it is still alive.
|
||||||
|
# Without this the test passes pre-flight, drops a real iptables rule on the partner, waits
|
||||||
|
# FALLBACK_TEST_BLOCK_WAIT for a transition nothing is there to make, and fails Phase 3 blaming
|
||||||
|
# fallback detection. Only the EXIT trap gets connectivity back.
|
||||||
|
#
|
||||||
|
# In --dry-run nothing is blocked and nothing is waited on, so a dead daemon is worth saying but
|
||||||
|
# not worth aborting for — the walkthrough still shows the operator the shape of the run.
|
||||||
|
if pgrep -f "Fallback/fallback\.sh" >/dev/null 2>&1; then
|
||||||
|
log "fallback.sh daemon is running"
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "fallback.sh is NOT running — a real test would abort here"
|
||||||
|
else
|
||||||
|
error "fallback.sh is not running — nothing would detect the outage this test creates"
|
||||||
|
error "Start it with array_started.sh, or run with --dry-run to walk the phases"
|
||||||
|
phase_fail "Pre-flight"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# Tier 1 containers configured
|
# Tier 1 containers configured
|
||||||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||||||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||||||
|
|||||||
Reference in New Issue
Block a user