#!/bin/bash # ============================================================================================== # ============================= Partnership Offboard =========================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Cleanly ends a partnership. Role is detected automatically — run on either server. # Owner path runs the full sequence including remote cleanup and final sync. # Mirror path handles the local side and signals the owner to complete its own cleanup. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # OWNER PATH (11 steps) # Step 1: Stop rsync — halt any running sync before state changes # Step 2: Final sync — mirror leaves with current Critical-Data state # Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost # Step 4: Disarm sync gates — RSYNC/CRITICAL_RSYNC/CONF_SYNC/ARR_SYNC=false in master.conf, # the exact inverse of onboard Step 9c # Step 5: Local cleanup — remove fallback coverage containers + appdata # Step 6: Restart own stack — bring up owner's own parked containers # Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror # Step 8: Restart mirror — bring up mirror's own parked containers # Step 9: Emby revocation — remove the mirror's Emby admin while Emby is still reachable # Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted # Tailscale — grace deadline recorded, device removed after it expires # Step 11: SSH revocation — keys, both directions. Genuinely last: it is the step that # removes the access every step above depends on # # MIRROR PATH (8 steps) # Step 1: Stop rsync — halt any running sync # Step 2: Reconfigure WebUIs — local auth WebUIs → localhost # Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack) # Step 4: Fallback cleanup — remove fallback coverage containers # Step 5: Disarm sync gates — same four gates as the owner path # Step 6: Revoke Emby admin — remove own admin account from local Emby instance # Step 7: Restart own stack — bring up own parked containers # Step 8: Finalise — write INACTIVE, clear phase flags, signal owner, THEN revoke # keys. Revocation is last because the signal needs the key # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Sync Before Severing # The final sync (owner step 2) runs before any state change, so the mirror leaves with # current Critical-Data rather than a snapshot from whenever the last scheduled sync # happened. Once keys are revoked there is no second chance to move data. # # Stop the Sync First # Step 1 on both paths halts rsync before anything else. A sync running through a # partnership teardown would be writing to a partner that is having its access removed # underneath it. # # Revoke Last, Not First # SSH keys and Emby admin are revoked at the end. Every earlier step needs working remote # access — revoking up front would strand the remaining cleanup on the far side and leave # the mirror holding containers nobody can remove. # # Both Sides Land Somewhere Valid # Each path restarts the host's own parked containers before finishing. Offboarding must # leave two working standalone servers, not one working server and one stripped of the # coverage it was relying on. # # Role Detected, Not Declared # Owner and mirror run different sequences, and the role is derived rather than passed in. # A human choosing the wrong path would run the owner's remote-cleanup steps against a # server that never deployed anything. # # Blocklist Is the Enforcement # Writing INACTIVE state is not enough on its own — a stale cron or a script mid-flight # could still attempt a sync. The mirror is blocklisted so rsync.sh refuses it outright, # independently of whatever any config still says. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Root Enforcement # Container removal, conf edits and SSH key revocation all require root. # # Lock Acquisition # acquire_lock "strict" — an offboard is not resumable partway, so a second instance is # refused rather than queued behind the first. # # Host Detection # detect_hosts() resolves MY_ID / REMOTE_ID, which the role detection builds on. # # Docker Presence Check # Verified before any container removal is attempted. # # Ordered Teardown # The step sequence is the safeguard — sync, then reconfigure, then remove, then restart, # then revoke, then record. Reordering breaks the guarantees above. # # Own Stack Restored # Parked containers are brought back up on both sides before the run completes. # # State Written Both Ends # INACTIVE is written locally and pushed to the mirror, so neither side is left believing # a partnership is still active. # # Partner Blocklisted # The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on — # stale access cannot survive the offboard. # # Tailscale Grace Window Is a Deadline, Not a Sleep # Device removal happens after state is written, not before, so the final state push # cannot be cut off by removing its own transport. The grace period itself is recorded to # STATE_DIR/tailscale_removal_due.db and the offboard returns. It used to sleep # PARTNERSHIP_GRACE_HOURS inline — six hours by default — holding the lock and its job record # open the whole time, reporting "running", and blocking any re-onboard behind it. # # Dry Run Support # --dry-run walks the full sequence reporting each step without executing any. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_PARTNERSHIP_AUTH_STACK # Auth container XMLs to push during onboard — used on offboard to identify what # to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed # from the mirror on both owner-initiated and mirror-initiated offboard. # # HOST*_PARTNERSHIP_ARR_STACK # Arr container XMLs — same cleanup logic as auth stack. # # HOST*_PARTNERSHIP_SERVICES_STACK # Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) — same cleanup logic. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # Partnership/partnership_offboard.sh # Full offboard — role detected automatically # # Partnership/partnership_offboard.sh --dry-run # Preview all steps without executing # # Partnership/partnership_offboard.sh --log # Verbose per-step output # # Partnership/partnership_offboard.sh --reason= # Tag the offboard reason in state file and blocklist (default: manual) # Called by partnership_manager.sh --offboard (reason passed through) # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPTS_ROOT="$SCRIPT_DIR/.." SSH_TIMEOUT=15 source "$SCRIPTS_ROOT/load_config.sh" source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" # ── Parse flags ─────────────────────────────────────────────────────────────────────────────── REASON="manual" STEP_DISABLE_RSYNC_OK=true # both paths report it; only the mirror path re-initialised it TAILSCALE_REMOVED=false # set only when remove_tailscale_device actually succeeds # Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from # MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded. # An offboard that failed to remove a single container still reported a clean teardown. STEP_LOCAL_CLEANUP_OK=true STEP_OWN_STACK_OK=true STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable STEP_MIRROR_STACK_OK=true # or "skipped" STEP_STATE_WRITE_OK=true STEP_STATE_PUSH_OK=false # INACTIVE actually delivered to the mirror, or "skipped" STEP_SETUP_PUSH_OK=false # cleared phase flags delivered to the mirror, or "skipped" FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --reason=*) REASON="${arg#--reason=}" ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ── Source partnership_manager.sh for shared helpers ────────────────────────────────────────── # PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed. PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== [[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts partnership_resolve_roles LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db" OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db" MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db" OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db" acquire_lock "strict" trap _pm_trap_restart_stopped EXIT # Check already offboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree" exit 0 fi fi START=$(date +%s) echo "" echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " This: $MY_ID ($LOCAL_SERVER_NAME)" echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )" echo " Reason: $REASON" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" # ============================================================================================== # ── HELPER: disarm every sync gate a completed onboard armed ───────────────────────────────── # # The exact inverse of partnership_onboard.sh Step 9c, which sets RSYNC_ENABLED, # CONF_SYNC_ENABLED and ARR_SYNC_ENABLED to true on a completed onboard. Offboard used to turn # off CRITICAL_RSYNC_ENABLED and nothing else, which left the far more consequential Tier 1 # RSYNC_ENABLED switched on with no partner to sync to. # # That asymmetry read as safe and was not. Turning off one Tier 2 gate looks like "sync is off" # in the summary, while INTERMEDIATE, DAILY, WEEKLY and FALLBACK rsync all stayed live behind an # open Tier 1 — and CONF_SYNC_ENABLED kept a 4-hourly job reaching for a partner that had just # been removed, failing and notifying each time. # # CRITICAL_RSYNC_ENABLED stays in the list. It is a Tier 2 gate and closing Tier 1 already stops # it, but leaving it true would misreport the state to anyone reading the conf rather than the # tier logic. # # FALLBACK_ENABLED is deliberately NOT here. Onboard does not arm it, so offboard has no business # disarming it — it is the operator's switch, and the summary says so rather than moving it. # ============================================================================================== _VV_SYNC_GATES=(RSYNC_ENABLED CRITICAL_RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED) # ── Stop any running rsync, and say honestly whether it worked ──────────────────────────────── # # Both offboard paths called "$SCRIPTS_ROOT/Rsync/rsync_stop.sh". That file has never existed — # rsync_stop.sh lives in System_Essentials/. With stderr sent to /dev/null the "No such file" # went unseen, and the line below it printed "Rsync stopped ✅" unconditionally, so every # offboard ever run reported stopping an rsync it had not touched. On the owner path the failed # exit also set STEP_STOP_OK=false, which is why the summary said ❌ two lines under a ✅. # # One helper, one path, and the outcome is the return value. stop_rsync_now() { local script="$SCRIPTS_ROOT/System_Essentials/rsync_stop.sh" if [[ ! -f "$script" ]]; then warn "rsync_stop.sh not found at $script" return 1 fi bash "$script" --rsync-only } _disarm_sync_gates() { local gate rc=0 conf="$SCRIPTS_ROOT/Configurations/master.conf" for gate in "${_VV_SYNC_GATES[@]}"; do # set_conf_bool, not update_master_conf: the latter rewrites the whole line and would # strip the trailing comment that explains what each tier gates. Same helper onboard # arms with, so arming and disarming are one operation in two directions. set_conf_bool "$gate" "false" "$conf" || rc=1 done return "$rc" } # ============================================================================================== # ── HELPER: revoke own admin account from local Emby instance ──────────────────────────────── # # Mirror-initiated path only. Called before start_own_stack so Emby is still running. # Uses local EMBY_API_KEY and the mirror's own short name as the username to delete. # ============================================================================================== revoke_local_emby_admin() { local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" local emby_url="http://127.0.0.1:${emby_port}" echo "" echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━" if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" return 0 fi if [[ -z "${EMBY_API_KEY:-}" ]]; then warn "EMBY_API_KEY not set — skipping local Emby admin revocation" return 1 fi # The account to revoke is this server's own short name (the mirror user's account) local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would delete Emby admin '$username' at $emby_url" return 0 fi log "Looking up Emby user '$username' at $emby_url..." local users_json user_id users_json=$(curl -sf --max-time 15 \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users" 2>/dev/null) user_id=$(echo "$users_json" | \ grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \ grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1) if [[ -z "$user_id" ]]; then warn "Emby user '$username' not found at $emby_url — may already be removed" return 0 fi local del_code del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ -X DELETE \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users/${user_id}" 2>/dev/null) if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then echo "Emby admin '$username' removed ✅" else warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually" fi } # ============================================================================================== # ── MIRROR PATH ─────────────────────────────────────────────────────────────────────────────── # ============================================================================================== if [[ "$AM_MIRROR" == true ]]; then warn "$MIRROR_ID ($MIRROR) is initiating offboard" warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 fi OWNER_IP=$(resolve_tailscale_ip "$OWNER") OWNER_REACHABLE=false [[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true STEP_STOP_RSYNC_OK=true STEP_WEBUI_OK=true STEP_STACK_CLEANUP_OK=true STEP_FALLBACK_CLEANUP_OK=true STEP_DISABLE_RSYNC_OK=true STEP_EMBY_OK=true SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false # ── Step 1: Stop rsync ──────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━" if [[ "$DRY_RUN" == false ]]; then if stop_rsync_now; then echo "Rsync stopped ✅" else warn "Could not stop rsync — a transfer may still be running into $OWNER" STEP_STOP_RSYNC_OK=false fi else warn "DRY RUN — would stop rsync" fi # ── Step 2: Reconfigure local WebUIs → localhost ────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━" reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false # ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ───────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━" if [[ "$OWNER_REACHABLE" == true ]]; then cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false else warn "Owner unreachable — cannot read deployed stack list" warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable" STEP_STACK_CLEANUP_OK=false fi # ── Step 4: Remove fallback coverage containers ─────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━" cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false # ── Step 5: Disarm the sync gates ───────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_GEAR Step 5/8 — Disarm Sync Gates ━━━" if [[ "$DRY_RUN" == false ]]; then _disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false else warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}" fi # ── Step 6: Revoke Emby admin locally ───────────────────────────────────────────────────── revoke_local_emby_admin || STEP_EMBY_OK=false # ── Step 7: Restart own stack ───────────────────────────────────────────────────────────── start_own_stack # ── Step 8: SSH key revocation, write state, signal owner ──────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━" # State first, keys last — same ordering the owner path needed. Revocation used to run here, # before the push below, so the mirror destroyed the key and then tried to tell the owner it # had left using that key. The owner never heard, and the notify promised it would "finalise # on next check" — a check that now had no way in. NOW=$(date '+%Y-%m-%d %H:%M:%S') if [[ "$DRY_RUN" == false ]]; then write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" echo "Local state: INACTIVE ✅" add_to_blocklist "$OWNER" "$REASON" # Inverse of onboard's write_onboard_phase. MIRROR_ID is this host on this path, and the # flags are named for the mirror on both sides, so the same names clear here. _setup_db="$(platform_setup_db_path)" for _flag in "${MIRROR_ID}_PHASE1_DONE" "${MIRROR_ID}_PHASE2_DONE" "${MIRROR_ID}_KEY_READY"; do clear_state_var "$_setup_db" "$_flag" done echo "Onboard phase flags cleared ✅" unset _setup_db _flag else warn "DRY RUN — would write INACTIVE state, clear phase flags and blocklist $OWNER" fi if [[ "$OWNER_REACHABLE" == true ]]; then platform_push_setup_state 2>/dev/null || warn "Could not push cleared setup state to $OWNER" push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY" notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ "Partnership" "normal" else warn "$OWNER unreachable — state written locally, owner will see it when reachable" fi # Last, for the reason above: everything before it needs the key. do_ssh_key_revocation "${OWNER_IP:-}" # ── Summary ─────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━" echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)" echo " Owner: $OWNER_ID ($OWNER)" echo " Reason: $REASON" echo " Duration: $(format_duration $(( END - START )))" echo "" _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } _revoke_status() { if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "both directions ✅" elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "local only ✅ — remote failed (revoke manually on $OWNER)" else echo "⚠️ failed — check warnings above" fi } echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")" echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")" echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")" echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")" echo " Step 5 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)" echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")" echo " Step 7 — Own stack: started" echo " Step 8 — Keys revoked: $(_revoke_status)" echo "" echo " State: INACTIVE ✅" echo " Blocklist: $OWNER blocked ✅" echo " Owner: will finalise + final sync on next --check" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ warn "$ICON_DONE DONE — mirror separation complete ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── OWNER PATH ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== warn "Offboarding $MIRROR_ID ($MIRROR) from partnership" warn "Final sync will run — mirror leaves with current state" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 echo "Proceeding..." fi WEBUI_FAILURES=0 STEP_STOP_OK=true STEP_SYNC_OK=true SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false # ── Step 1: Stop rsync ──────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━" if [[ "$DRY_RUN" == false ]]; then if stop_rsync_now; then echo "Rsync stopped ✅" else warn "Could not stop rsync — a transfer may still be running into $MIRROR" warn " Everything below changes state while data is still moving, which is the one" warn " ordering this step exists to prevent. Check: ps -ef | grep rsync" STEP_STOP_OK=false fi else warn "DRY RUN — would stop rsync" fi # ── Step 2: Final sync ──────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━" do_final_sync || STEP_SYNC_OK=false # ── Step 3: Reconfigure mirror WebUIs → localhost ───────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━" MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") MIRROR_REACHABLE=false [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true if [[ "$MIRROR_REACHABLE" == true ]]; then for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "localhost" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done else warn "$MIRROR unreachable — WebUI reconfiguration skipped" warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check" (( WEBUI_FAILURES++ )) fi # ── Step 4: Disarm the sync gates ───────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_GEAR Step 4/10 — Disarm Sync Gates ━━━" if [[ "$DRY_RUN" == false ]]; then _disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false else warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}" fi # ── Step 5: Local container cleanup ─────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━" cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false # ── Step 6: Restart own stack ───────────────────────────────────────────────────────────────── start_own_stack || STEP_OWN_STACK_OK=false # ── Step 7: Remote container cleanup ────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then # Remove auth/arr stack containers deployed during onboard (by config array) cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false # Remove fallback coverage containers (by *-owner_short naming pattern) cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false else warn "$MIRROR unreachable — remote container cleanup skipped" warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually" STEP_REMOTE_CLEANUP_OK=skipped fi # ── Step 8: Restart mirror's own stack ──────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false else STEP_MIRROR_STACK_OK=skipped fi # ── Step 9: Emby revocation ─────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Step 9/11 — Emby Revocation ━━━" # Before SSH key revocation, while Emby is still reachable [[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" # ── Step 10: Write state, push to mirror, blocklist ─────────────────────────────────────────── # Ahead of SSH revocation, which is now Step 11. # # "State Written Both Ends" is the guarantee this step exists for — neither side left believing # the partnership is active. It could not deliver it: revocation used to run here in Step 9 and # then this push authenticated with the key it had just destroyed, so every offboard ended # "Could not push state file to remote — will propagate on next sync". There is no next sync; # the gates are closed and the keys are gone. The mirror was left reading ACTIVE for ever. # # The original ordering note said state must follow revocation so a crash between steps 5–9 # re-runs from scratch rather than early-exiting on INACTIVE. Writing it one step later than the # cleanup preserves that — the cleanup is still done before any state is recorded — while # putting the push back inside the window where it can actually reach the mirror. echo "" echo "━━━ $ICON_GEAR Step 10/11 — Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') if [[ "$DRY_RUN" == false ]]; then # Checked, because this is the record every other host and every later --check reads. A # failed write here leaves both sides believing the partnership is still active while the # summary says INACTIVE — the one line in the teardown that must not be assumed. if write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then echo "Local state: INACTIVE ✅" else error "Failed to write local state file — $MIRROR may still look ACTIVE here" STEP_STATE_WRITE_OK=false fi add_to_blocklist "$MIRROR" "$REASON" # The inverse of onboard's write_onboard_phase, which had none. Without this a completed # offboard leaves the state file reading INACTIVE beside HOST*_PHASE1_DONE / _PHASE2_DONE # still set — and the setup checklist and partnership card read the flags, not the state # file, so a torn-down partnership went on presenting itself as fully onboarded. # # KEY_READY goes too: it means "a key is generated and waiting to be installed", which stops # being true the moment Step 11 revokes both sides. _setup_db="$(platform_setup_db_path)" for _flag in "${MIRROR_ID}_PHASE1_DONE" "${MIRROR_ID}_PHASE2_DONE" "${MIRROR_ID}_KEY_READY"; do clear_state_var "$_setup_db" "$_flag" done echo "Onboard phase flags cleared ✅" unset _setup_db _flag if [[ "$MIRROR_REACHABLE" == true ]]; then # Two separate pushes, and the summary used to report only the second. A run where the # setup.db push failed and the state-file push succeeded printed "Could not push state # file to remote" in the body and "Pushed to mirror: ✅" in the summary — describing # different files with the same words. They are tracked apart now, because they fail # apart: the mirror can be told the partnership is INACTIVE while keeping the phase # flags that make its own UI still claim a finished onboard. # # Pushed after the flags are cleared, so the mirror receives the cleared file rather than # the version that still claimed a finished onboard. if platform_push_setup_state 2>/dev/null; then STEP_SETUP_PUSH_OK=true else warn "Could not push cleared phase flags to $MIRROR — its wizard and partnership" warn " card will keep showing a completed onboard until it is reinstalled or pulled" fi if push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then STEP_STATE_PUSH_OK=true else warn "Could not push INACTIVE to $MIRROR — it will keep reading ACTIVE until told otherwise" fi else STEP_STATE_PUSH_OK=skipped STEP_SETUP_PUSH_OK=skipped fi else warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote" STEP_STATE_PUSH_OK=true STEP_SETUP_PUSH_OK=true fi # ── Step 11: SSH key revocation ─────────────────────────────────────────────────────────────── # Genuinely last. Every step above needs working remote access — this is the one that takes it # away, so it goes after the final thing that has to reach the mirror. echo "" echo "━━━ $ICON_SHIELD Step 11/11 — SSH Key Revocation ━━━" do_ssh_key_revocation "${MIRROR_IP:-}" # Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then echo "" echo "━━━ $ICON_NET Tailscale Separation ━━━" # The grace period is recorded as a deadline, not slept through. # # This used to `sleep $((PARTNERSHIP_GRACE_HOURS * 3600))` inline — six hours by default — # holding the offboard's lock and its job record open the whole time, showing "running" to # every status reader, and blocking any re-onboard behind the lock. Worse, the sleep ran # even when removal was going to be a no-op: TAILSCALE_API_KEY and TAILSCALE_TAILNET are # both empty here, so the six hours bought nothing at all. # # The offboard's own work is finished by this point. Writing the deadline lets the teardown # complete now and leaves the removal to whoever reads the file — and makes the wait # visible and cancellable instead of buried in a sleeping process. _grace_h="${PARTNERSHIP_GRACE_HOURS:-6}" if [[ "$_grace_h" -gt 0 ]] && [[ "$MIRROR_REACHABLE" == true ]]; then _due=$(( $(date +%s) + _grace_h * 3600 )) if [[ "$DRY_RUN" == false ]]; then printf 'host=%s\ndue=%s\ndue_human=%s\nreason=%s\n' \ "$MIRROR" "$_due" "$(date -d "@$_due" '+%Y-%m-%d %H:%M:%S')" "$REASON" \ > "${STATE_DIR}/tailscale_removal_due.db" fi warn "Grace period: $MIRROR stays on the tailnet until $(date -d "@$_due" '+%Y-%m-%d %H:%M') — recorded, not slept" # No CLI entry point removes it yet, and there is deliberately no invented one here: # remove_tailscale_device() is a partnership_manager.sh function with no --mode of its # own, and it no-ops without credentials regardless. Say what is true. if [[ -z "${TAILSCALE_API_KEY:-}" || -z "${TAILSCALE_TAILNET:-}" ]]; then warn "Automatic removal is not possible — TAILSCALE_API_KEY/TAILSCALE_TAILNET are unset; remove it in the Tailscale admin console" else warn "Removal after that is not yet automated — remove it in the Tailscale admin console" fi TAILSCALE_REMOVED=deferred unset _grace_h _due else # No grace configured, or the mirror is already unreachable — remove now. # # Outcome recorded, not assumed. remove_tailscale_device returns 1 when TAILSCALE_API_KEY # or TAILSCALE_TAILNET is unset — it warns "skipping Tailscale removal" and the summary # went on to report "removed ✅" anyway, so an offboard that left the device on the # tailnet said it had taken it off. Neither key is configured here, so that was every run. if remove_tailscale_device "$MIRROR"; then TAILSCALE_REMOVED=true fi fi fi # Backup handover notification if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_DISK Backup Handover ━━━" echo "Backups available for $MIRROR:" for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do [[ -z "$path" ]] && continue echo " $path" done notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \ "Partnership" "warning" fi # ── Summary ─────────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━" echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)" echo " Mirror: $MIRROR_ID ($MIRROR)" echo " Reason: $REASON" echo " Duration: $(format_duration $(( END - START )))" echo "" _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } _revoke_status() { if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "both directions ✅" elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "local only ✅ — remote failed (revoke manually on $MIRROR)" else echo "⚠️ failed — check warnings above" fi } echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")" echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")" echo " Step 3 — WebUI failures: $WEBUI_FAILURES" echo " Step 4 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)" echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")" echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )" echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )" echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )" echo " Step 9 — Emby revoked: $(_ok "${STEP_EMBY_OK:-true}")" echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )" echo " Step 10 — INACTIVE pushed: $( [[ "$STEP_STATE_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_STATE_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still reads ACTIVE"; } )" echo " Step 10 — Phase flags pushed: $( [[ "$STEP_SETUP_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_SETUP_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still shows a finished onboard"; } )" echo " Step 11 — Keys revoked: $(_revoke_status)" echo "" echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then case "$TAILSCALE_REMOVED" in true) echo " Tailscale: $MIRROR removed ✅" ;; deferred) echo " Tailscale: $MIRROR kept until the grace period expires — see ${STATE_DIR}/tailscale_removal_due.db" ;; *) echo " Tailscale: $MIRROR NOT removed ⚠ — still on the tailnet (needs TAILSCALE_API_KEY + TAILSCALE_TAILNET)" ;; esac fi # Named because it is the one partnership switch neither onboard nor offboard moves, so it # survives an offboard still true and there is nothing else that would ever mention it. [[ "${FALLBACK_ENABLED:-false}" == true ]] && \ echo " FALLBACK_ENABLED is still true — nothing left to fail over to, turn it off yourself" echo "" echo " $MIRROR leaves with:" # Each line reads the step that produced it. "✓ Current auth config (final sync)" was a # literal, so an offboard whose Step 2 had just warned "Final sync did NOT complete" still # closed by telling the operator the mirror held current auth data. That is the one claim here # somebody might act on — it is the difference between a partner that can stand alone and one # carrying a stale copy of the auth stack. if [[ "$STEP_SYNC_OK" == true ]]; then echo " ✓ Current auth config (final sync)" else echo " ✗ Auth config NOT synced — it keeps whatever it already had (see Step 2)" fi if [[ "${WEBUI_FAILURES:-0}" -eq 0 ]]; then echo " ✓ Auth WebUIs → localhost" else echo " ✗ ${WEBUI_FAILURES} auth WebUI(s) still point at $LOCAL_SERVER_NAME — fix by hand there" fi echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups" echo "" # The verdict is derived, never asserted. "DONE — clean separation complete ✅" printed # unconditionally, under a summary that had already shown two failed steps. if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$STEP_STOP_OK" == true && "$STEP_SYNC_OK" == true \ && "$STEP_STATE_PUSH_OK" != false && "$STEP_SETUP_PUSH_OK" != false \ && "${WEBUI_FAILURES:-0}" -eq 0 ]]; then echo "$ICON_DONE DONE — clean separation complete ✅" else warn "$ICON_DONE Offboard finished with unresolved steps — the partnership is ended, but" warn " the ❌ lines above did not happen. Re-read them before re-onboarding." fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0