Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
621 lines
29 KiB
Bash
Executable File
621 lines
29 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= Partnership Offboard ===========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Cleanly ends a partnership. Role is detected automatically — run on either server.
|
||
# Owner path runs the full sequence including remote cleanup and final sync.
|
||
# Mirror path handles the local side and signals the owner to complete its own cleanup.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# OWNER PATH (10 steps)
|
||
# Step 1: Stop rsync — halt any running sync before state changes
|
||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||
# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
||
# Step 6: Restart own stack — bring up owner's own parked containers
|
||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||
# Step 9: Revocation — Emby admin, SSH keys
|
||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||
# Tailscale — grace window then device removal (after state written)
|
||
#
|
||
# MIRROR PATH (8 steps)
|
||
# Step 1: Stop rsync — halt any running sync
|
||
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
||
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
||
# Step 4: Fallback cleanup — remove fallback coverage containers
|
||
# Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
||
# Step 7: Restart own stack — bring up own parked containers
|
||
# Step 8: SSH revocation — revoke keys both directions, write state, signal owner
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Sync Before Severing
|
||
# The final sync (owner step 2) runs before any state change, so the mirror leaves with
|
||
# current Critical-Data rather than a snapshot from whenever the last scheduled sync
|
||
# happened. Once keys are revoked there is no second chance to move data.
|
||
#
|
||
# Stop the Sync First
|
||
# Step 1 on both paths halts rsync before anything else. A sync running through a
|
||
# partnership teardown would be writing to a partner that is having its access removed
|
||
# underneath it.
|
||
#
|
||
# Revoke Last, Not First
|
||
# SSH keys and Emby admin are revoked at the end. Every earlier step needs working remote
|
||
# access — revoking up front would strand the remaining cleanup on the far side and leave
|
||
# the mirror holding containers nobody can remove.
|
||
#
|
||
# Both Sides Land Somewhere Valid
|
||
# Each path restarts the host's own parked containers before finishing. Offboarding must
|
||
# leave two working standalone servers, not one working server and one stripped of the
|
||
# coverage it was relying on.
|
||
#
|
||
# Role Detected, Not Declared
|
||
# Owner and mirror run different sequences, and the role is derived rather than passed in.
|
||
# A human choosing the wrong path would run the owner's remote-cleanup steps against a
|
||
# server that never deployed anything.
|
||
#
|
||
# Blocklist Is the Enforcement
|
||
# Writing INACTIVE state is not enough on its own — a stale cron or a script mid-flight
|
||
# could still attempt a sync. The mirror is blocklisted so rsync.sh refuses it outright,
|
||
# independently of whatever any config still says.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Enforcement
|
||
# Container removal, conf edits and SSH key revocation all require root.
|
||
#
|
||
# Lock Acquisition
|
||
# acquire_lock "strict" — an offboard is not resumable partway, so a second instance is
|
||
# refused rather than queued behind the first.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() resolves MY_ID / REMOTE_ID, which the role detection builds on.
|
||
#
|
||
# Docker Presence Check
|
||
# Verified before any container removal is attempted.
|
||
#
|
||
# Ordered Teardown
|
||
# The step sequence is the safeguard — sync, then reconfigure, then remove, then restart,
|
||
# then revoke, then record. Reordering breaks the guarantees above.
|
||
#
|
||
# Own Stack Restored
|
||
# Parked containers are brought back up on both sides before the run completes.
|
||
#
|
||
# State Written Both Ends
|
||
# INACTIVE is written locally and pushed to the mirror, so neither side is left believing
|
||
# a partnership is still active.
|
||
#
|
||
# Partner Blocklisted
|
||
# The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on —
|
||
# stale access cannot survive the offboard.
|
||
#
|
||
# Tailscale Grace Window
|
||
# Device removal happens after state is written, not before, so the final state push
|
||
# cannot be cut off by removing its own transport.
|
||
#
|
||
# Dry Run Support
|
||
# --dry-run walks the full sequence reporting each step without executing any.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_PARTNERSHIP_AUTH_STACK
|
||
# Auth container XMLs to push during onboard — used on offboard to identify what
|
||
# to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed
|
||
# from the mirror on both owner-initiated and mirror-initiated offboard.
|
||
#
|
||
# HOST*_PARTNERSHIP_ARR_STACK
|
||
# Arr container XMLs — same cleanup logic as auth stack.
|
||
#
|
||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||
# Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) — same cleanup logic.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# Partnership/partnership_offboard.sh
|
||
# Full offboard — role detected automatically
|
||
#
|
||
# Partnership/partnership_offboard.sh --dry-run
|
||
# Preview all steps without executing
|
||
#
|
||
# Partnership/partnership_offboard.sh --log
|
||
# Verbose per-step output
|
||
#
|
||
# Partnership/partnership_offboard.sh --reason=<string>
|
||
# Tag the offboard reason in state file and blocklist (default: manual)
|
||
# Called by partnership_manager.sh --offboard (reason passed through)
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||
SSH_TIMEOUT=15
|
||
|
||
source "$SCRIPTS_ROOT/load_config.sh"
|
||
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
||
|
||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||
REASON="manual"
|
||
FILTERED_ARGS=()
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--reason=*) REASON="${arg#--reason=}" ;;
|
||
*) FILTERED_ARGS+=("$arg") ;;
|
||
esac
|
||
done
|
||
|
||
parse_args "${FILTERED_ARGS[@]}"
|
||
|
||
# ── Source partnership_manager.sh for shared helpers ──────────────────────────────────────────
|
||
# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed.
|
||
PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
|
||
partnership_resolve_roles
|
||
|
||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||
OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db"
|
||
MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db"
|
||
OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db"
|
||
|
||
acquire_lock "strict"
|
||
|
||
trap _pm_trap_restart_stopped EXIT
|
||
|
||
# Check already offboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||
warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
START=$(date +%s)
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
||
echo " Reason: $REASON"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
||
#
|
||
# Mirror-initiated path only. Called before start_own_stack so Emby is still running.
|
||
# Uses local EMBY_API_KEY and the mirror's own short name as the username to delete.
|
||
# ==============================================================================================
|
||
revoke_local_emby_admin() {
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://127.0.0.1:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping local Emby admin revocation"
|
||
return 1
|
||
fi
|
||
|
||
# The account to revoke is this server's own short name (the mirror user's account)
|
||
local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}"
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
log "Looking up Emby user '$username' at $emby_url..."
|
||
|
||
local users_json user_id
|
||
users_json=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
user_id=$(echo "$users_json" | \
|
||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
local del_code
|
||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X DELETE \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||
|
||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||
echo "Emby admin '$username' removed ✅"
|
||
else
|
||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||
warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
fi
|
||
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||
OWNER_REACHABLE=false
|
||
[[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true
|
||
|
||
STEP_STOP_RSYNC_OK=true
|
||
STEP_WEBUI_OK=true
|
||
STEP_STACK_CLEANUP_OK=true
|
||
STEP_FALLBACK_CLEANUP_OK=true
|
||
STEP_DISABLE_RSYNC_OK=true
|
||
STEP_EMBY_OK=true
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
|
||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||
echo "Rsync stopped ✅"
|
||
else
|
||
warn "DRY RUN — would stop rsync"
|
||
fi
|
||
|
||
# ── Step 2: Reconfigure local WebUIs → localhost ──────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━"
|
||
|
||
reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false
|
||
|
||
# ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ─────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━"
|
||
|
||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||
cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false
|
||
else
|
||
warn "Owner unreachable — cannot read deployed stack list"
|
||
warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable"
|
||
STEP_STACK_CLEANUP_OK=false
|
||
fi
|
||
|
||
# ── Step 4: Remove fallback coverage containers ───────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━"
|
||
|
||
cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false
|
||
|
||
# ── Step 5: Disable critical sync ─────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \
|
||
warn "CRITICAL_RSYNC_ENABLED=false ✅" || \
|
||
{ warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; }
|
||
else
|
||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||
fi
|
||
|
||
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
||
revoke_local_emby_admin || STEP_EMBY_OK=false
|
||
|
||
# ── Step 7: Restart own stack ─────────────────────────────────────────────────────────────
|
||
start_own_stack
|
||
|
||
# ── Step 8: SSH key revocation, write state, signal owner ────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
||
|
||
do_ssh_key_revocation "${OWNER_IP:-}"
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
echo "Local state: INACTIVE ✅"
|
||
add_to_blocklist "$OWNER" "$REASON"
|
||
else
|
||
warn "DRY RUN — would write INACTIVE state and blocklist $OWNER"
|
||
fi
|
||
|
||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
||
END=$(date +%s)
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━"
|
||
echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Owner: $OWNER_ID ($OWNER)"
|
||
echo " Reason: $REASON"
|
||
echo " Duration: $(format_duration $(( END - START )))"
|
||
echo ""
|
||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||
_revoke_status() {
|
||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "both directions ✅"
|
||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "local only ✅ — remote failed (revoke manually on $OWNER)"
|
||
else
|
||
echo "⚠️ failed — check warnings above"
|
||
fi
|
||
}
|
||
|
||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")"
|
||
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
||
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
||
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
||
echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")"
|
||
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
||
echo " Step 7 — Own stack: started"
|
||
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
||
echo ""
|
||
echo " State: INACTIVE ✅"
|
||
echo " Blocklist: $OWNER blocked ✅"
|
||
echo " Owner: will finalise + final sync on next --check"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||
warn "$ICON_DONE DONE — mirror separation complete ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||
warn "Final sync will run — mirror leaves with current state"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
echo "Proceeding..."
|
||
fi
|
||
|
||
WEBUI_FAILURES=0
|
||
STEP_STOP_OK=true
|
||
STEP_SYNC_OK=true
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
|
||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
|
||
echo "Rsync stopped ✅"
|
||
else
|
||
warn "DRY RUN — would stop rsync"
|
||
fi
|
||
|
||
# ── Step 2: Final sync ────────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━"
|
||
|
||
do_final_sync || STEP_SYNC_OK=false
|
||
|
||
# ── Step 3: Reconfigure mirror WebUIs → localhost ─────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━"
|
||
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||
MIRROR_REACHABLE=false
|
||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "localhost" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
else
|
||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||
(( WEBUI_FAILURES++ ))
|
||
fi
|
||
|
||
# ── Step 4: Disable critical sync ─────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||
else
|
||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||
fi
|
||
|
||
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
||
|
||
cleanup_partner_containers
|
||
|
||
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
||
start_own_stack
|
||
|
||
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━"
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||
else
|
||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
||
fi
|
||
|
||
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
||
|
||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
||
|
||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
||
|
||
# Emby admin — before SSH key revocation while Emby still reachable
|
||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||
|
||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||
|
||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||
# State is written after container cleanup and SSH revocation so that:
|
||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
echo "Local state: INACTIVE ✅"
|
||
add_to_blocklist "$MIRROR" "$REASON"
|
||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
else
|
||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||
fi
|
||
|
||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||
sleep "$grace_seconds"
|
||
trap - SIGTERM SIGINT
|
||
fi
|
||
fi
|
||
remove_tailscale_device "$MIRROR"
|
||
fi
|
||
|
||
# Backup handover notification
|
||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||
echo "Backups available for $MIRROR:"
|
||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||
[[ -z "$path" ]] && continue
|
||
echo " $path"
|
||
done
|
||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||
"Partnership" "warning"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||
END=$(date +%s)
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━"
|
||
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||
echo " Reason: $REASON"
|
||
echo " Duration: $(format_duration $(( END - START )))"
|
||
echo ""
|
||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||
_revoke_status() {
|
||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "both directions ✅"
|
||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "local only ✅ — remote failed (revoke manually on $MIRROR)"
|
||
else
|
||
echo "⚠️ failed — check warnings above"
|
||
fi
|
||
}
|
||
|
||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
||
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||
echo " Step 4 — Disable sync: ✅"
|
||
echo " Step 5 — Local cleanup: ✅"
|
||
echo " Step 6 — Own stack: started"
|
||
echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
||
echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
||
echo " Step 10 — State: INACTIVE ✅"
|
||
echo ""
|
||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||
echo " Tailscale: $MIRROR removed ✅"
|
||
echo ""
|
||
echo " $MIRROR leaves with:"
|
||
echo " ✓ Current auth config (final sync)"
|
||
echo " ✓ Auth WebUIs → localhost"
|
||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|