From 6a15a9d99ab74729811c00958acd07113c5bf819 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 17 Aug 2026 04:58:15 -0400 Subject: [PATCH] Push INACTIVE to the mirror before revoking the key that push needs, and stop two steps reporting success they did not have --- Partnership/partnership_manager.sh | 20 +++++++++++ Partnership/partnership_offboard.sh | 56 +++++++++++++++++++++-------- System_Essentials/rsync_stop.sh | 12 ++++++- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index f5a13fc..aa859e4 100755 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -902,8 +902,15 @@ check_both_healthy() { return 0 } +# Returns non-zero when the mirror did NOT leave with current state, so the caller can say so. +# The whole point of this step is the guarantee in the summary — "mirror leaves with current +# Critical-Data" — and it used to print that unconditionally at the end of the function. An +# offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs" +# immediately followed by "Final sync complete — mirror has current state ✅", and the summary +# scored Step 2 as a pass. The mirror left with whatever it happened to have. do_final_sync() { log "Running final critical sync..." + local _synced=0 _failed=0 if [[ "$DRY_RUN" == false ]]; then if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then for _share in "${CRITICAL_SYNC_SHARES[@]}"; do @@ -916,17 +923,30 @@ do_final_sync() { bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \ --profile="$_profile" --log fi + # rsync.sh exits 0 both on a real sync and on a clean Tier-1 gate exit, so the + # gate is checked here rather than inferred from its status. + if [[ "${RSYNC_ENABLED:-true}" == true ]]; then (( _synced++ )); else (( _failed++ )); fi done else warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)" + _failed=1 fi else # ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution", # so this line aborted Step 2 of every offboard --dry-run with a shell error instead of # printing. A count of an unset array is already 0, which is the only default needed. warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)" + return 0 + fi + + if [[ "$_failed" -gt 0 ]]; then + warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had" + [[ "${RSYNC_ENABLED:-true}" != true ]] && \ + warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent" + return 1 fi warn "Final sync complete — mirror has current state ✅" + return 0 } # Safe master.conf modification with error handling — appends the key if not already present, diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh index e4d0a69..cfeed73 100755 --- a/Partnership/partnership_offboard.sh +++ b/Partnership/partnership_offboard.sh @@ -13,7 +13,7 @@ # OPERATIONAL MODEL # ============================================================================================== # -# OWNER PATH (10 steps) +# OWNER PATH (11 steps) # Step 1: Stop rsync — halt any running sync before state changes # Step 2: Final sync — mirror leaves with current Critical-Data state # Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost @@ -23,9 +23,11 @@ # Step 6: Restart own stack — bring up owner's own parked containers # Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror # Step 8: Restart mirror — bring up mirror's own parked containers -# Step 9: Revocation — Emby admin, SSH keys +# Step 9: Emby revocation — remove the mirror's Emby admin while Emby is still reachable # Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted # Tailscale — grace deadline recorded, device removed after it expires +# Step 11: SSH revocation — keys, both directions. Genuinely last: it is the step that +# removes the access every step above depends on # # MIRROR PATH (8 steps) # Step 1: Stop rsync — halt any running sync @@ -169,6 +171,7 @@ STEP_OWN_STACK_OK=true STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable STEP_MIRROR_STACK_OK=true # or "skipped" STEP_STATE_WRITE_OK=true +STEP_STATE_PUSH_OK=false # INACTIVE actually delivered to the mirror, or "skipped" FILTERED_ARGS=() for arg in "$@"; do @@ -565,22 +568,28 @@ else STEP_MIRROR_STACK_OK=skipped fi -# ── Step 9: Revocation (Emby + SSH) ────────────────────────────────────────────────────────── +# ── Step 9: Emby revocation ─────────────────────────────────────────────────────────────────── echo "" -echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━" +echo "━━━ $ICON_SHIELD Step 9/11 — Emby Revocation ━━━" -# Emby admin — before SSH key revocation while Emby still reachable +# Before SSH key revocation, while Emby is still reachable [[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" -# SSH key revocation — mutual, both directions; must run while Tailscale still active -do_ssh_key_revocation "${MIRROR_IP:-}" - # ── Step 10: Write state, push to mirror, blocklist ─────────────────────────────────────────── -# State is written after container cleanup and SSH revocation so that: -# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE) -# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard +# Ahead of SSH revocation, which is now Step 11. +# +# "State Written Both Ends" is the guarantee this step exists for — neither side left believing +# the partnership is active. It could not deliver it: revocation used to run here in Step 9 and +# then this push authenticated with the key it had just destroyed, so every offboard ended +# "Could not push state file to remote — will propagate on next sync". There is no next sync; +# the gates are closed and the keys are gone. The mirror was left reading ACTIVE for ever. +# +# The original ordering note said state must follow revocation so a crash between steps 5–9 +# re-runs from scratch rather than early-exiting on INACTIVE. Writing it one step later than the +# cleanup preserves that — the cleanup is still done before any state is recorded — while +# putting the push back inside the window where it can actually reach the mirror. echo "" -echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━" +echo "━━━ $ICON_GEAR Step 10/11 — Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') @@ -596,12 +605,27 @@ if [[ "$DRY_RUN" == false ]]; then STEP_STATE_WRITE_OK=false fi add_to_blocklist "$MIRROR" "$REASON" - [[ "$MIRROR_REACHABLE" == true ]] && \ - push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" + if [[ "$MIRROR_REACHABLE" == true ]]; then + if push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then + STEP_STATE_PUSH_OK=true + else + warn "Could not push INACTIVE to $MIRROR — it will keep reading ACTIVE until told otherwise" + fi + else + STEP_STATE_PUSH_OK=skipped + fi else warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote" + STEP_STATE_PUSH_OK=true fi +# ── Step 11: SSH key revocation ─────────────────────────────────────────────────────────────── +# Genuinely last. Every step above needs working remote access — this is the one that takes it +# away, so it goes after the final thing that has to reach the mirror. +echo "" +echo "━━━ $ICON_SHIELD Step 11/11 — SSH Key Revocation ━━━" +do_ssh_key_revocation "${MIRROR_IP:-}" + # Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then echo "" @@ -690,8 +714,10 @@ echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")" echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )" echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )" echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )" -echo " Step 9 — Keys revoked: $(_revoke_status)" +echo " Step 9 — Emby revoked: $(_ok "${STEP_EMBY_OK:-true}")" echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )" +echo " Step 10 — Pushed to mirror: $( [[ "$STEP_STATE_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_STATE_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still reads ACTIVE"; } )" +echo " Step 11 — Keys revoked: $(_revoke_status)" echo "" echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then diff --git a/System_Essentials/rsync_stop.sh b/System_Essentials/rsync_stop.sh index a2e573d..7a8635b 100755 --- a/System_Essentials/rsync_stop.sh +++ b/System_Essentials/rsync_stop.sh @@ -565,4 +565,14 @@ if [[ "$DRY_RUN" == false ]]; then notify "Rsync stopped on $(hostname) ($MY_ID) — mode: $MODE${CONTAINERS_RESTARTED:+ — recovered: ${CONTAINERS_RESTARTED[*]}}" \ "Rsync Stop" "warning" fi -fi \ No newline at end of file +fi + +# Explicit, because callers branch on it. Without this the script's status is whatever the last +# thing to run happened to return — a notify, or a test that evaluated false — and +# partnership_offboard.sh does `rsync_stop.sh || STEP_STOP_OK=false`. That is how an offboard +# printed "Rsync stopped ✅" in the log and "Step 1 — Stop rsync: ❌" in its own summary. +# +# Only a container that failed to come back is a failure: killing rsync is the job, and finding +# nothing to kill is a success, not an error. +[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file