Push INACTIVE to the mirror before revoking the key that push needs, and stop two steps reporting success they did not have
This commit is contained in:
@@ -902,8 +902,15 @@ check_both_healthy() {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Returns non-zero when the mirror did NOT leave with current state, so the caller can say so.
|
||||||
|
# The whole point of this step is the guarantee in the summary — "mirror leaves with current
|
||||||
|
# Critical-Data" — and it used to print that unconditionally at the end of the function. An
|
||||||
|
# offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs"
|
||||||
|
# immediately followed by "Final sync complete — mirror has current state ✅", and the summary
|
||||||
|
# scored Step 2 as a pass. The mirror left with whatever it happened to have.
|
||||||
do_final_sync() {
|
do_final_sync() {
|
||||||
log "Running final critical sync..."
|
log "Running final critical sync..."
|
||||||
|
local _synced=0 _failed=0
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||||||
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||||||
@@ -916,17 +923,30 @@ do_final_sync() {
|
|||||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
||||||
--profile="$_profile" --log
|
--profile="$_profile" --log
|
||||||
fi
|
fi
|
||||||
|
# rsync.sh exits 0 both on a real sync and on a clean Tier-1 gate exit, so the
|
||||||
|
# gate is checked here rather than inferred from its status.
|
||||||
|
if [[ "${RSYNC_ENABLED:-true}" == true ]]; then (( _synced++ )); else (( _failed++ )); fi
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
||||||
|
_failed=1
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
# ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution",
|
# ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution",
|
||||||
# so this line aborted Step 2 of every offboard --dry-run with a shell error instead of
|
# so this line aborted Step 2 of every offboard --dry-run with a shell error instead of
|
||||||
# printing. A count of an unset array is already 0, which is the only default needed.
|
# printing. A count of an unset array is already 0, which is the only default needed.
|
||||||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)"
|
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$_failed" -gt 0 ]]; then
|
||||||
|
warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had"
|
||||||
|
[[ "${RSYNC_ENABLED:-true}" != true ]] && \
|
||||||
|
warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent"
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
warn "Final sync complete — mirror has current state ✅"
|
warn "Final sync complete — mirror has current state ✅"
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
# Safe master.conf modification with error handling — appends the key if not already present,
|
# Safe master.conf modification with error handling — appends the key if not already present,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
# OWNER PATH (10 steps)
|
# OWNER PATH (11 steps)
|
||||||
# Step 1: Stop rsync — halt any running sync before state changes
|
# Step 1: Stop rsync — halt any running sync before state changes
|
||||||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||||||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||||||
@@ -23,9 +23,11 @@
|
|||||||
# Step 6: Restart own stack — bring up owner's own parked containers
|
# Step 6: Restart own stack — bring up owner's own parked containers
|
||||||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||||||
# Step 9: Revocation — Emby admin, SSH keys
|
# Step 9: Emby revocation — remove the mirror's Emby admin while Emby is still reachable
|
||||||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||||
# Tailscale — grace deadline recorded, device removed after it expires
|
# Tailscale — grace deadline recorded, device removed after it expires
|
||||||
|
# Step 11: SSH revocation — keys, both directions. Genuinely last: it is the step that
|
||||||
|
# removes the access every step above depends on
|
||||||
#
|
#
|
||||||
# MIRROR PATH (8 steps)
|
# MIRROR PATH (8 steps)
|
||||||
# Step 1: Stop rsync — halt any running sync
|
# Step 1: Stop rsync — halt any running sync
|
||||||
@@ -169,6 +171,7 @@ STEP_OWN_STACK_OK=true
|
|||||||
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
|
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
|
||||||
STEP_MIRROR_STACK_OK=true # or "skipped"
|
STEP_MIRROR_STACK_OK=true # or "skipped"
|
||||||
STEP_STATE_WRITE_OK=true
|
STEP_STATE_WRITE_OK=true
|
||||||
|
STEP_STATE_PUSH_OK=false # INACTIVE actually delivered to the mirror, or "skipped"
|
||||||
FILTERED_ARGS=()
|
FILTERED_ARGS=()
|
||||||
|
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
@@ -565,22 +568,28 @@ else
|
|||||||
STEP_MIRROR_STACK_OK=skipped
|
STEP_MIRROR_STACK_OK=skipped
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
# ── Step 9: Emby revocation ───────────────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
echo "━━━ $ICON_SHIELD Step 9/11 — Emby Revocation ━━━"
|
||||||
|
|
||||||
# Emby admin — before SSH key revocation while Emby still reachable
|
# Before SSH key revocation, while Emby is still reachable
|
||||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||||
|
|
||||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
|
||||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
|
||||||
|
|
||||||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||||||
# State is written after container cleanup and SSH revocation so that:
|
# Ahead of SSH revocation, which is now Step 11.
|
||||||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
#
|
||||||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
# "State Written Both Ends" is the guarantee this step exists for — neither side left believing
|
||||||
|
# the partnership is active. It could not deliver it: revocation used to run here in Step 9 and
|
||||||
|
# then this push authenticated with the key it had just destroyed, so every offboard ended
|
||||||
|
# "Could not push state file to remote — will propagate on next sync". There is no next sync;
|
||||||
|
# the gates are closed and the keys are gone. The mirror was left reading ACTIVE for ever.
|
||||||
|
#
|
||||||
|
# The original ordering note said state must follow revocation so a crash between steps 5–9
|
||||||
|
# re-runs from scratch rather than early-exiting on INACTIVE. Writing it one step later than the
|
||||||
|
# cleanup preserves that — the cleanup is still done before any state is recorded — while
|
||||||
|
# putting the push back inside the window where it can actually reach the mirror.
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
echo "━━━ $ICON_GEAR Step 10/11 — Write State ━━━"
|
||||||
|
|
||||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||||
|
|
||||||
@@ -596,12 +605,27 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
STEP_STATE_WRITE_OK=false
|
STEP_STATE_WRITE_OK=false
|
||||||
fi
|
fi
|
||||||
add_to_blocklist "$MIRROR" "$REASON"
|
add_to_blocklist "$MIRROR" "$REASON"
|
||||||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
if push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||||
|
STEP_STATE_PUSH_OK=true
|
||||||
|
else
|
||||||
|
warn "Could not push INACTIVE to $MIRROR — it will keep reading ACTIVE until told otherwise"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
STEP_STATE_PUSH_OK=skipped
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||||||
|
STEP_STATE_PUSH_OK=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Step 11: SSH key revocation ───────────────────────────────────────────────────────────────
|
||||||
|
# Genuinely last. Every step above needs working remote access — this is the one that takes it
|
||||||
|
# away, so it goes after the final thing that has to reach the mirror.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_SHIELD Step 11/11 — SSH Key Revocation ━━━"
|
||||||
|
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||||
|
|
||||||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
@@ -690,8 +714,10 @@ echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")"
|
|||||||
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
|
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
|
||||||
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
|
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
|
||||||
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
|
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
|
||||||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
echo " Step 9 — Emby revoked: $(_ok "${STEP_EMBY_OK:-true}")"
|
||||||
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
|
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
|
||||||
|
echo " Step 10 — Pushed to mirror: $( [[ "$STEP_STATE_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_STATE_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still reads ACTIVE"; } )"
|
||||||
|
echo " Step 11 — Keys revoked: $(_revoke_status)"
|
||||||
echo ""
|
echo ""
|
||||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
||||||
|
|||||||
@@ -566,3 +566,13 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
"Rsync Stop" "warning"
|
"Rsync Stop" "warning"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Explicit, because callers branch on it. Without this the script's status is whatever the last
|
||||||
|
# thing to run happened to return — a notify, or a test that evaluated false — and
|
||||||
|
# partnership_offboard.sh does `rsync_stop.sh || STEP_STOP_OK=false`. That is how an offboard
|
||||||
|
# printed "Rsync stopped ✅" in the log and "Step 1 — Stop rsync: ❌" in its own summary.
|
||||||
|
#
|
||||||
|
# Only a container that failed to come back is a failure: killing rsync is the job, and finding
|
||||||
|
# nothing to kill is a success, not an error.
|
||||||
|
[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && exit 1
|
||||||
|
exit 0
|
||||||
Reference in New Issue
Block a user