Give the test harness a stop that cannot strand its iptables rule, and clean up the dry-run state copy
This commit is contained in:
+10
-1
@@ -928,7 +928,16 @@ echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
|||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
FALLBACK_RUNNING=true
|
FALLBACK_RUNNING=true
|
||||||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
|
||||||
|
# The dry-run state copy is per-PID and would otherwise accumulate one file per preview run.
|
||||||
|
# EXIT as well as the signals, because a dry run is usually ended with Ctrl-C or --stop but can
|
||||||
|
# also just fall out of the loop.
|
||||||
|
dryrun_state_cleanup() {
|
||||||
|
[[ "$DRY_RUN" == true && "$FALLBACK_STATE_FILE" == *".dryrun."* ]] && rm -f "$FALLBACK_STATE_FILE"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
trap 'dryrun_state_cleanup' EXIT
|
||||||
|
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; dryrun_state_cleanup; exit 0' \
|
||||||
SIGTERM SIGINT
|
SIGTERM SIGINT
|
||||||
|
|
||||||
while [[ "$FALLBACK_RUNNING" == true ]]; do
|
while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||||
|
|||||||
@@ -118,6 +118,11 @@
|
|||||||
# fallback_test.sh --log
|
# fallback_test.sh --log
|
||||||
# Verbose output on every check in every phase.
|
# Verbose output on every check in every phase.
|
||||||
#
|
#
|
||||||
|
# fallback_test.sh --stop
|
||||||
|
# Stop a running test. SIGTERM only — never SIGKILL, because only this script's EXIT
|
||||||
|
# trap removes the iptables DROP rule it installed. Also sweeps a rule stranded by an
|
||||||
|
# earlier SIGKILL or power cut.
|
||||||
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
@@ -126,6 +131,58 @@ source "$SCRIPT_DIR/../load_config.sh"
|
|||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
|
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||||||
|
#
|
||||||
|
# SIGTERM ONLY, and deliberately no SIGKILL escalation — the opposite of fallback.sh --stop.
|
||||||
|
# A running test holds an iptables DROP rule against the partner, and the only thing that removes
|
||||||
|
# it is this script's own EXIT trap. SIGKILL does not run traps, so force-killing a test strands
|
||||||
|
# the rule: the partner stays invisible, fallback.sh reads that as a permanent outage and holds
|
||||||
|
# FALLBACK indefinitely. A test that will not die is a worse outcome than a test still running,
|
||||||
|
# so this reports the stranded rule and the command to clear it rather than causing one.
|
||||||
|
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||||||
|
LOCKFILE="${LOCK_DIR}/fallback_test.lock"
|
||||||
|
if [[ ! -f "$LOCKFILE" ]]; then
|
||||||
|
log "No fallback_test.sh lock found — not running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||||||
|
target_pid="${lock_content%%:*}"
|
||||||
|
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||||||
|
warn "Stale lock — fallback_test.sh not running (PID ${target_pid:-unknown} gone) — clearing"
|
||||||
|
rm -f "$LOCKFILE"
|
||||||
|
# A stale lock is exactly the SIGKILL/power-cut case, so the rule may still be in place.
|
||||||
|
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||||
|
warn "Stranded iptables DROP rule found for $REMOTE_SERVER — removing"
|
||||||
|
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||||
|
&& warn "Stranded rule removed — remote connectivity restored ✅" \
|
||||||
|
|| error "Could not remove stranded rule — run: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
warn "Stopping fallback_test.sh (PID $target_pid) — SIGTERM so its trap clears the iptables rule..."
|
||||||
|
kill -TERM "$target_pid" 2>/dev/null || true
|
||||||
|
waited=0
|
||||||
|
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 30 ]]; do
|
||||||
|
sleep 1
|
||||||
|
(( waited++ )) || true
|
||||||
|
done
|
||||||
|
if kill -0 "$target_pid" 2>/dev/null; then
|
||||||
|
error "fallback_test.sh (PID $target_pid) did not exit within 30s"
|
||||||
|
error "NOT force-killing — SIGKILL would strand the iptables DROP rule on $REMOTE_SERVER"
|
||||||
|
error "Wait, or clear manually: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
warn "Stopped: fallback_test.sh (PID $target_pid) ✅"
|
||||||
|
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||||
|
error "iptables DROP rule for $REMOTE_SERVER survived the stop — removing"
|
||||||
|
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||||
|
&& warn "Rule removed ✅" || error "Could not remove — run it by hand"
|
||||||
|
else
|
||||||
|
log "No iptables DROP rule remains for $REMOTE_SERVER ✅"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
||||||
DOCKER_TIMEOUT=15
|
DOCKER_TIMEOUT=15
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user