Give the test harness a stop that cannot strand its iptables rule, and clean up the dry-run state copy
This commit is contained in:
+10
-1
@@ -928,7 +928,16 @@ echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
FALLBACK_RUNNING=true
|
||||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
||||
|
||||
# The dry-run state copy is per-PID and would otherwise accumulate one file per preview run.
|
||||
# EXIT as well as the signals, because a dry run is usually ended with Ctrl-C or --stop but can
|
||||
# also just fall out of the loop.
|
||||
dryrun_state_cleanup() {
|
||||
[[ "$DRY_RUN" == true && "$FALLBACK_STATE_FILE" == *".dryrun."* ]] && rm -f "$FALLBACK_STATE_FILE"
|
||||
return 0
|
||||
}
|
||||
trap 'dryrun_state_cleanup' EXIT
|
||||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; dryrun_state_cleanup; exit 0' \
|
||||
SIGTERM SIGINT
|
||||
|
||||
while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
|
||||
@@ -118,6 +118,11 @@
|
||||
# fallback_test.sh --log
|
||||
# Verbose output on every check in every phase.
|
||||
#
|
||||
# fallback_test.sh --stop
|
||||
# Stop a running test. SIGTERM only — never SIGKILL, because only this script's EXIT
|
||||
# trap removes the iptables DROP rule it installed. Also sweeps a rule stranded by an
|
||||
# earlier SIGKILL or power cut.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -126,6 +131,58 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||||
#
|
||||
# SIGTERM ONLY, and deliberately no SIGKILL escalation — the opposite of fallback.sh --stop.
|
||||
# A running test holds an iptables DROP rule against the partner, and the only thing that removes
|
||||
# it is this script's own EXIT trap. SIGKILL does not run traps, so force-killing a test strands
|
||||
# the rule: the partner stays invisible, fallback.sh reads that as a permanent outage and holds
|
||||
# FALLBACK indefinitely. A test that will not die is a worse outcome than a test still running,
|
||||
# so this reports the stranded rule and the command to clear it rather than causing one.
|
||||
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||||
LOCKFILE="${LOCK_DIR}/fallback_test.lock"
|
||||
if [[ ! -f "$LOCKFILE" ]]; then
|
||||
log "No fallback_test.sh lock found — not running"
|
||||
exit 0
|
||||
fi
|
||||
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||||
target_pid="${lock_content%%:*}"
|
||||
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||||
warn "Stale lock — fallback_test.sh not running (PID ${target_pid:-unknown} gone) — clearing"
|
||||
rm -f "$LOCKFILE"
|
||||
# A stale lock is exactly the SIGKILL/power-cut case, so the rule may still be in place.
|
||||
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||
warn "Stranded iptables DROP rule found for $REMOTE_SERVER — removing"
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||
&& warn "Stranded rule removed — remote connectivity restored ✅" \
|
||||
|| error "Could not remove stranded rule — run: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
warn "Stopping fallback_test.sh (PID $target_pid) — SIGTERM so its trap clears the iptables rule..."
|
||||
kill -TERM "$target_pid" 2>/dev/null || true
|
||||
waited=0
|
||||
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 30 ]]; do
|
||||
sleep 1
|
||||
(( waited++ )) || true
|
||||
done
|
||||
if kill -0 "$target_pid" 2>/dev/null; then
|
||||
error "fallback_test.sh (PID $target_pid) did not exit within 30s"
|
||||
error "NOT force-killing — SIGKILL would strand the iptables DROP rule on $REMOTE_SERVER"
|
||||
error "Wait, or clear manually: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||
exit 1
|
||||
fi
|
||||
warn "Stopped: fallback_test.sh (PID $target_pid) ✅"
|
||||
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||
error "iptables DROP rule for $REMOTE_SERVER survived the stop — removing"
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||
&& warn "Rule removed ✅" || error "Could not remove — run it by hand"
|
||||
else
|
||||
log "No iptables DROP rule remains for $REMOTE_SERVER ✅"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
|
||||
Reference in New Issue
Block a user