#!/bin/bash # ============================================================================================== # ============================= Partnership Offboard =========================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Cleanly ends a partnership. Role is detected automatically — run on either server. # Owner path runs the full sequence including remote cleanup and final sync. # Mirror path handles the local side and signals the owner to complete its own cleanup. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # OWNER PATH (10 steps) # Step 1: Stop rsync — halt any running sync before state changes # Step 2: Final sync — mirror leaves with current Critical-Data state # Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost # Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf # Step 5: Local cleanup — remove fallback coverage containers + appdata # Step 6: Restart own stack — bring up owner's own parked containers # Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror # Step 8: Restart mirror — bring up mirror's own parked containers # Step 9: Revocation — Emby admin, SSH keys # Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted # Tailscale — grace window then device removal (after state written) # # MIRROR PATH (8 steps) # Step 1: Stop rsync — halt any running sync # Step 2: Reconfigure WebUIs — local auth WebUIs → localhost # Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack) # Step 4: Fallback cleanup — remove fallback coverage containers # Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf # Step 6: Revoke Emby admin — remove own admin account from local Emby instance # Step 7: Restart own stack — bring up own parked containers # Step 8: SSH revocation — revoke keys both directions, write state, signal owner # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_PARTNERSHIP_AUTH_STACK # Auth container XMLs to push during onboard — used on offboard to identify what # to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed # from the mirror on both owner-initiated and mirror-initiated offboard. # # HOST*_PARTNERSHIP_ARR_STACK # Arr container XMLs — same cleanup logic as auth stack. # # HOST*_PARTNERSHIP_SERVICES_STACK # Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) — same cleanup logic. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # Partnership/partnership_offboard.sh # Full offboard — role detected automatically # # Partnership/partnership_offboard.sh --dry-run # Preview all steps without executing # # Partnership/partnership_offboard.sh --log # Verbose per-step output # # Partnership/partnership_offboard.sh --reason= # Tag the offboard reason in state file and blocklist (default: manual) # Called by partnership_manager.sh --offboard (reason passed through) # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPTS_ROOT="$SCRIPT_DIR/.." SSH_TIMEOUT=15 source "$SCRIPTS_ROOT/load_config.sh" source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" # ── Parse flags ─────────────────────────────────────────────────────────────────────────────── REASON="manual" FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --reason=*) REASON="${arg#--reason=}" ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ── Source partnership_manager.sh for shared helpers ────────────────────────────────────────── # PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed. PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== [[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) OWNER="${!OWNER_ID}" MIRROR="${!MIRROR_ID}" MIRROR_SSH_KEY="$SSH_KEY" OWNER_SSH_KEY="$SSH_KEY" AM_OWNER=false AM_MIRROR=false [[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true [[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db" OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db" MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db" OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db" acquire_lock "strict" trap _pm_trap_restart_stopped EXIT # Check already offboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree" exit 0 fi fi START=$(date +%s) echo "" echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " This: $MY_ID ($LOCAL_SERVER_NAME)" echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )" echo " Reason: $REASON" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" # ============================================================================================== # ── HELPER: revoke own admin account from local Emby instance ──────────────────────────────── # # Mirror-initiated path only. Called before start_own_stack so Emby is still running. # Uses local EMBY_API_KEY and the mirror's own short name as the username to delete. # ============================================================================================== revoke_local_emby_admin() { local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" local emby_url="http://127.0.0.1:${emby_port}" echo "" echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━" if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" return 0 fi if [[ -z "${EMBY_API_KEY:-}" ]]; then warn "EMBY_API_KEY not set — skipping local Emby admin revocation" return 1 fi # The account to revoke is this server's own short name (the mirror user's account) local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would delete Emby admin '$username' at $emby_url" return 0 fi log "Looking up Emby user '$username' at $emby_url..." local users_json user_id users_json=$(curl -sf --max-time 15 \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users" 2>/dev/null) user_id=$(echo "$users_json" | \ grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \ grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1) if [[ -z "$user_id" ]]; then warn "Emby user '$username' not found at $emby_url — may already be removed" return 0 fi local del_code del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ -X DELETE \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users/${user_id}" 2>/dev/null) if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then echo "Emby admin '$username' removed ✅" else warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually" fi } # ============================================================================================== # ── MIRROR PATH ─────────────────────────────────────────────────────────────────────────────── # ============================================================================================== if [[ "$AM_MIRROR" == true ]]; then warn "$MIRROR_ID ($MIRROR) is initiating offboard" warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 fi OWNER_IP=$(resolve_tailscale_ip "$OWNER") OWNER_REACHABLE=false [[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true STEP_STOP_RSYNC_OK=true STEP_WEBUI_OK=true STEP_STACK_CLEANUP_OK=true STEP_FALLBACK_CLEANUP_OK=true STEP_DISABLE_RSYNC_OK=true STEP_EMBY_OK=true SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false # ── Step 1: Stop rsync ──────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━" if [[ "$DRY_RUN" == false ]]; then bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true echo "Rsync stopped ✅" else warn "DRY RUN — would stop rsync" fi # ── Step 2: Reconfigure local WebUIs → localhost ────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━" reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false # ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ───────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━" if [[ "$OWNER_REACHABLE" == true ]]; then cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false else warn "Owner unreachable — cannot read deployed stack list" warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable" STEP_STACK_CLEANUP_OK=false fi # ── Step 4: Remove fallback coverage containers ─────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━" cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false # ── Step 5: Disable critical sync ───────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━" if [[ "$DRY_RUN" == false ]]; then update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \ warn "CRITICAL_RSYNC_ENABLED=false ✅" || \ { warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; } else warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" fi # ── Step 6: Revoke Emby admin locally ───────────────────────────────────────────────────── revoke_local_emby_admin || STEP_EMBY_OK=false # ── Step 7: Restart own stack ───────────────────────────────────────────────────────────── start_own_stack # ── Step 8: SSH key revocation, write state, signal owner ──────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━" do_ssh_key_revocation "${OWNER_IP:-}" NOW=$(date '+%Y-%m-%d %H:%M:%S') if [[ "$DRY_RUN" == false ]]; then write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" echo "Local state: INACTIVE ✅" add_to_blocklist "$OWNER" "$REASON" else warn "DRY RUN — would write INACTIVE state and blocklist $OWNER" fi if [[ "$OWNER_REACHABLE" == true ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY" notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ "Partnership" "normal" else warn "$OWNER unreachable — state written locally, owner will see it when reachable" fi # ── Summary ─────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━" echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)" echo " Owner: $OWNER_ID ($OWNER)" echo " Reason: $REASON" echo " Duration: $(format_duration $(( END - START )))" echo "" _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } _revoke_status() { if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "both directions ✅" elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "local only ✅ — remote failed (revoke manually on $OWNER)" else echo "⚠️ failed — check warnings above" fi } echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")" echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")" echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")" echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")" echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")" echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")" echo " Step 7 — Own stack: started" echo " Step 8 — Keys revoked: $(_revoke_status)" echo "" echo " State: INACTIVE ✅" echo " Blocklist: $OWNER blocked ✅" echo " Owner: will finalise + final sync on next --check" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ warn "$ICON_DONE DONE — mirror separation complete ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── OWNER PATH ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== warn "Offboarding $MIRROR_ID ($MIRROR) from partnership" warn "Final sync will run — mirror leaves with current state" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 echo "Proceeding..." fi WEBUI_FAILURES=0 STEP_STOP_OK=true STEP_SYNC_OK=true SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false # ── Step 1: Stop rsync ──────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━" if [[ "$DRY_RUN" == false ]]; then bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false echo "Rsync stopped ✅" else warn "DRY RUN — would stop rsync" fi # ── Step 2: Final sync ──────────────────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━" do_final_sync || STEP_SYNC_OK=false # ── Step 3: Reconfigure mirror WebUIs → localhost ───────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━" MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") MIRROR_REACHABLE=false [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true if [[ "$MIRROR_REACHABLE" == true ]]; then for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "localhost" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done else warn "$MIRROR unreachable — WebUI reconfiguration skipped" warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check" (( WEBUI_FAILURES++ )) fi # ── Step 4: Disable critical sync ───────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━" if [[ "$DRY_RUN" == false ]]; then update_master_conf "CRITICAL_RSYNC_ENABLED" "false" warn "CRITICAL_RSYNC_ENABLED=false ✅" else warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" fi # ── Step 5: Local container cleanup ─────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━" cleanup_partner_containers # ── Step 6: Restart own stack ───────────────────────────────────────────────────────────────── start_own_stack # ── Step 7: Remote container cleanup ────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then # Remove auth/arr stack containers deployed during onboard (by config array) cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" # Remove fallback coverage containers (by *-owner_short naming pattern) cleanup_owner_containers_on_mirror "$MIRROR_IP" else warn "$MIRROR unreachable — remote container cleanup skipped" warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually" fi # ── Step 8: Restart mirror's own stack ──────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━" [[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP" # ── Step 9: Revocation (Emby + SSH) ────────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━" # Emby admin — before SSH key revocation while Emby still reachable [[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" # SSH key revocation — mutual, both directions; must run while Tailscale still active do_ssh_key_revocation "${MIRROR_IP:-}" # ── Step 10: Write state, push to mirror, blocklist ─────────────────────────────────────────── # State is written after container cleanup and SSH revocation so that: # • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE) # • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard echo "" echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') if [[ "$DRY_RUN" == false ]]; then write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" echo "Local state: INACTIVE ✅" add_to_blocklist "$MIRROR" "$REASON" [[ "$MIRROR_REACHABLE" == true ]] && \ push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" else warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote" fi # Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then echo "" echo "━━━ $ICON_NET Tailscale Separation ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..." if [[ "$DRY_RUN" == false ]]; then trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT sleep "$grace_seconds" trap - SIGTERM SIGINT fi fi remove_tailscale_device "$MIRROR" fi # Backup handover notification if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_DISK Backup Handover ━━━" echo "Backups available for $MIRROR:" for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do [[ -z "$path" ]] && continue echo " $path" done notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \ "Partnership" "warning" fi # ── Summary ─────────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━" echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)" echo " Mirror: $MIRROR_ID ($MIRROR)" echo " Reason: $REASON" echo " Duration: $(format_duration $(( END - START )))" echo "" _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } _revoke_status() { if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "both directions ✅" elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "local only ✅ — remote failed (revoke manually on $MIRROR)" else echo "⚠️ failed — check warnings above" fi } echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")" echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")" echo " Step 3 — WebUI failures: $WEBUI_FAILURES" echo " Step 4 — Disable sync: ✅" echo " Step 5 — Local cleanup: ✅" echo " Step 6 — Own stack: started" echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )" echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )" echo " Step 9 — Keys revoked: $(_revoke_status)" echo " Step 10 — State: INACTIVE ✅" echo "" echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ echo " Tailscale: $MIRROR removed ✅" echo "" echo " $MIRROR leaves with:" echo " ✓ Current auth config (final sync)" echo " ✓ Auth WebUIs → localhost" echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ warn "$ICON_DONE DONE — clean separation complete ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0