Push INACTIVE to the mirror before revoking the key that push needs, and stop two steps reporting success they did not have

This commit is contained in:
Gmer4Lfe
2026-08-17 04:58:15 -04:00
parent a46317de18
commit 6a15a9d99a
3 changed files with 72 additions and 16 deletions
+20
View File
@@ -902,8 +902,15 @@ check_both_healthy() {
return 0
}
# Returns non-zero when the mirror did NOT leave with current state, so the caller can say so.
# The whole point of this step is the guarantee in the summary — "mirror leaves with current
# Critical-Data" — and it used to print that unconditionally at the end of the function. An
# offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs"
# immediately followed by "Final sync complete — mirror has current state ✅", and the summary
# scored Step 2 as a pass. The mirror left with whatever it happened to have.
do_final_sync() {
log "Running final critical sync..."
local _synced=0 _failed=0
if [[ "$DRY_RUN" == false ]]; then
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
@@ -916,17 +923,30 @@ do_final_sync() {
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
--profile="$_profile" --log
fi
# rsync.sh exits 0 both on a real sync and on a clean Tier-1 gate exit, so the
# gate is checked here rather than inferred from its status.
if [[ "${RSYNC_ENABLED:-true}" == true ]]; then (( _synced++ )); else (( _failed++ )); fi
done
else
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
_failed=1
fi
else
# ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution",
# so this line aborted Step 2 of every offboard --dry-run with a shell error instead of
# printing. A count of an unset array is already 0, which is the only default needed.
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)"
return 0
fi
if [[ "$_failed" -gt 0 ]]; then
warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had"
[[ "${RSYNC_ENABLED:-true}" != true ]] && \
warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent"
return 1
fi
warn "Final sync complete — mirror has current state ✅"
return 0
}
# Safe master.conf modification with error handling — appends the key if not already present,
+41 -15
View File
@@ -13,7 +13,7 @@
# OPERATIONAL MODEL
# ==============================================================================================
#
# OWNER PATH (10 steps)
# OWNER PATH (11 steps)
# Step 1: Stop rsync — halt any running sync before state changes
# Step 2: Final sync — mirror leaves with current Critical-Data state
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
@@ -23,9 +23,11 @@
# Step 6: Restart own stack — bring up owner's own parked containers
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
# Step 8: Restart mirror — bring up mirror's own parked containers
# Step 9: Revocation — Emby admin, SSH keys
# Step 9: Emby revocation — remove the mirror's Emby admin while Emby is still reachable
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
# Tailscale — grace deadline recorded, device removed after it expires
# Step 11: SSH revocation — keys, both directions. Genuinely last: it is the step that
# removes the access every step above depends on
#
# MIRROR PATH (8 steps)
# Step 1: Stop rsync — halt any running sync
@@ -169,6 +171,7 @@ STEP_OWN_STACK_OK=true
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
STEP_MIRROR_STACK_OK=true # or "skipped"
STEP_STATE_WRITE_OK=true
STEP_STATE_PUSH_OK=false # INACTIVE actually delivered to the mirror, or "skipped"
FILTERED_ARGS=()
for arg in "$@"; do
@@ -565,22 +568,28 @@ else
STEP_MIRROR_STACK_OK=skipped
fi
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
# ── Step 9: Emby revocation ───────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
echo "━━━ $ICON_SHIELD Step 9/11 Emby Revocation ━━━"
# Emby admin — before SSH key revocation while Emby still reachable
# Before SSH key revocation, while Emby is still reachable
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
# SSH key revocation — mutual, both directions; must run while Tailscale still active
do_ssh_key_revocation "${MIRROR_IP:-}"
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
# State is written after container cleanup and SSH revocation so that:
# • Re-running after a crash between steps 59 restarts from scratch (no early-exit on INACTIVE)
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
# Ahead of SSH revocation, which is now Step 11.
#
# "State Written Both Ends" is the guarantee this step exists for — neither side left believing
# the partnership is active. It could not deliver it: revocation used to run here in Step 9 and
# then this push authenticated with the key it had just destroyed, so every offboard ended
# "Could not push state file to remote — will propagate on next sync". There is no next sync;
# the gates are closed and the keys are gone. The mirror was left reading ACTIVE for ever.
#
# The original ordering note said state must follow revocation so a crash between steps 59
# re-runs from scratch rather than early-exiting on INACTIVE. Writing it one step later than the
# cleanup preserves that — the cleanup is still done before any state is recorded — while
# putting the push back inside the window where it can actually reach the mirror.
echo ""
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
echo "━━━ $ICON_GEAR Step 10/11 — Write State ━━━"
NOW=$(date '+%Y-%m-%d %H:%M:%S')
@@ -596,12 +605,27 @@ if [[ "$DRY_RUN" == false ]]; then
STEP_STATE_WRITE_OK=false
fi
add_to_blocklist "$MIRROR" "$REASON"
[[ "$MIRROR_REACHABLE" == true ]] && \
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
if [[ "$MIRROR_REACHABLE" == true ]]; then
if push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
STEP_STATE_PUSH_OK=true
else
warn "Could not push INACTIVE to $MIRROR — it will keep reading ACTIVE until told otherwise"
fi
else
STEP_STATE_PUSH_OK=skipped
fi
else
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
STEP_STATE_PUSH_OK=true
fi
# ── Step 11: SSH key revocation ───────────────────────────────────────────────────────────────
# Genuinely last. Every step above needs working remote access — this is the one that takes it
# away, so it goes after the final thing that has to reach the mirror.
echo ""
echo "━━━ $ICON_SHIELD Step 11/11 — SSH Key Revocation ━━━"
do_ssh_key_revocation "${MIRROR_IP:-}"
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
echo ""
@@ -690,8 +714,10 @@ echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")"
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
echo " Step 9 — Keys revoked: $(_revoke_status)"
echo " Step 9 — Emby revoked: $(_ok "${STEP_EMBY_OK:-true}")"
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
echo " Step 10 — Pushed to mirror: $( [[ "$STEP_STATE_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_STATE_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still reads ACTIVE"; } )"
echo " Step 11 — Keys revoked: $(_revoke_status)"
echo ""
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then