Give the test harness a stop that cannot strand its iptables rule, and clean up the dry-run state copy

This commit is contained in:
Gmer4Lfe
2026-08-22 00:20:40 -04:00
parent 41c0598220
commit f8119f3786
2 changed files with 67 additions and 1 deletions
+10 -1
View File
@@ -928,7 +928,16 @@ echo " $ICON_NET Remote IP: $REMOTE_SERVER"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
FALLBACK_RUNNING=true
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
# The dry-run state copy is per-PID and would otherwise accumulate one file per preview run.
# EXIT as well as the signals, because a dry run is usually ended with Ctrl-C or --stop but can
# also just fall out of the loop.
dryrun_state_cleanup() {
[[ "$DRY_RUN" == true && "$FALLBACK_STATE_FILE" == *".dryrun."* ]] && rm -f "$FALLBACK_STATE_FILE"
return 0
}
trap 'dryrun_state_cleanup' EXIT
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; dryrun_state_cleanup; exit 0' \
SIGTERM SIGINT
while [[ "$FALLBACK_RUNNING" == true ]]; do
+57
View File
@@ -118,6 +118,11 @@
# fallback_test.sh --log
# Verbose output on every check in every phase.
#
# fallback_test.sh --stop
# Stop a running test. SIGTERM only — never SIGKILL, because only this script's EXIT
# trap removes the iptables DROP rule it installed. Also sweeps a rule stranded by an
# earlier SIGKILL or power cut.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -126,6 +131,58 @@ source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
#
# SIGTERM ONLY, and deliberately no SIGKILL escalation — the opposite of fallback.sh --stop.
# A running test holds an iptables DROP rule against the partner, and the only thing that removes
# it is this script's own EXIT trap. SIGKILL does not run traps, so force-killing a test strands
# the rule: the partner stays invisible, fallback.sh reads that as a permanent outage and holds
# FALLBACK indefinitely. A test that will not die is a worse outcome than a test still running,
# so this reports the stranded rule and the command to clear it rather than causing one.
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
LOCKFILE="${LOCK_DIR}/fallback_test.lock"
if [[ ! -f "$LOCKFILE" ]]; then
log "No fallback_test.sh lock found — not running"
exit 0
fi
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
target_pid="${lock_content%%:*}"
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
warn "Stale lock — fallback_test.sh not running (PID ${target_pid:-unknown} gone) — clearing"
rm -f "$LOCKFILE"
# A stale lock is exactly the SIGKILL/power-cut case, so the rule may still be in place.
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
warn "Stranded iptables DROP rule found for $REMOTE_SERVER — removing"
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
&& warn "Stranded rule removed — remote connectivity restored ✅" \
|| error "Could not remove stranded rule — run: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
fi
exit 0
fi
warn "Stopping fallback_test.sh (PID $target_pid) — SIGTERM so its trap clears the iptables rule..."
kill -TERM "$target_pid" 2>/dev/null || true
waited=0
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 30 ]]; do
sleep 1
(( waited++ )) || true
done
if kill -0 "$target_pid" 2>/dev/null; then
error "fallback_test.sh (PID $target_pid) did not exit within 30s"
error "NOT force-killing — SIGKILL would strand the iptables DROP rule on $REMOTE_SERVER"
error "Wait, or clear manually: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
exit 1
fi
warn "Stopped: fallback_test.sh (PID $target_pid) ✅"
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
error "iptables DROP rule for $REMOTE_SERVER survived the stop — removing"
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
&& warn "Rule removed ✅" || error "Could not remove — run it by hand"
else
log "No iptables DROP rule remains for $REMOTE_SERVER"
fi
exit 0
fi
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
DOCKER_TIMEOUT=15