718 lines
36 KiB
Bash
Executable File
718 lines
36 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= Partnership Offboard ===========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Cleanly ends a partnership. Role is detected automatically — run on either server.
|
||
# Owner path runs the full sequence including remote cleanup and final sync.
|
||
# Mirror path handles the local side and signals the owner to complete its own cleanup.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# OWNER PATH (10 steps)
|
||
# Step 1: Stop rsync — halt any running sync before state changes
|
||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||
# Step 4: Disarm sync gates — RSYNC/CRITICAL_RSYNC/CONF_SYNC/ARR_SYNC=false in master.conf,
|
||
# the exact inverse of onboard Step 9c
|
||
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
||
# Step 6: Restart own stack — bring up owner's own parked containers
|
||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||
# Step 9: Revocation — Emby admin, SSH keys
|
||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||
# Tailscale — grace deadline recorded, device removed after it expires
|
||
#
|
||
# MIRROR PATH (8 steps)
|
||
# Step 1: Stop rsync — halt any running sync
|
||
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
||
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
||
# Step 4: Fallback cleanup — remove fallback coverage containers
|
||
# Step 5: Disarm sync gates — same four gates as the owner path
|
||
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
||
# Step 7: Restart own stack — bring up own parked containers
|
||
# Step 8: SSH revocation — revoke keys both directions, write state, signal owner
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Sync Before Severing
|
||
# The final sync (owner step 2) runs before any state change, so the mirror leaves with
|
||
# current Critical-Data rather than a snapshot from whenever the last scheduled sync
|
||
# happened. Once keys are revoked there is no second chance to move data.
|
||
#
|
||
# Stop the Sync First
|
||
# Step 1 on both paths halts rsync before anything else. A sync running through a
|
||
# partnership teardown would be writing to a partner that is having its access removed
|
||
# underneath it.
|
||
#
|
||
# Revoke Last, Not First
|
||
# SSH keys and Emby admin are revoked at the end. Every earlier step needs working remote
|
||
# access — revoking up front would strand the remaining cleanup on the far side and leave
|
||
# the mirror holding containers nobody can remove.
|
||
#
|
||
# Both Sides Land Somewhere Valid
|
||
# Each path restarts the host's own parked containers before finishing. Offboarding must
|
||
# leave two working standalone servers, not one working server and one stripped of the
|
||
# coverage it was relying on.
|
||
#
|
||
# Role Detected, Not Declared
|
||
# Owner and mirror run different sequences, and the role is derived rather than passed in.
|
||
# A human choosing the wrong path would run the owner's remote-cleanup steps against a
|
||
# server that never deployed anything.
|
||
#
|
||
# Blocklist Is the Enforcement
|
||
# Writing INACTIVE state is not enough on its own — a stale cron or a script mid-flight
|
||
# could still attempt a sync. The mirror is blocklisted so rsync.sh refuses it outright,
|
||
# independently of whatever any config still says.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Enforcement
|
||
# Container removal, conf edits and SSH key revocation all require root.
|
||
#
|
||
# Lock Acquisition
|
||
# acquire_lock "strict" — an offboard is not resumable partway, so a second instance is
|
||
# refused rather than queued behind the first.
|
||
#
|
||
# Host Detection
|
||
# detect_hosts() resolves MY_ID / REMOTE_ID, which the role detection builds on.
|
||
#
|
||
# Docker Presence Check
|
||
# Verified before any container removal is attempted.
|
||
#
|
||
# Ordered Teardown
|
||
# The step sequence is the safeguard — sync, then reconfigure, then remove, then restart,
|
||
# then revoke, then record. Reordering breaks the guarantees above.
|
||
#
|
||
# Own Stack Restored
|
||
# Parked containers are brought back up on both sides before the run completes.
|
||
#
|
||
# State Written Both Ends
|
||
# INACTIVE is written locally and pushed to the mirror, so neither side is left believing
|
||
# a partnership is still active.
|
||
#
|
||
# Partner Blocklisted
|
||
# The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on —
|
||
# stale access cannot survive the offboard.
|
||
#
|
||
# Tailscale Grace Window Is a Deadline, Not a Sleep
|
||
# Device removal happens after state is written, not before, so the final state push
|
||
# cannot be cut off by removing its own transport. The grace period itself is recorded to
|
||
# STATE_DIR/tailscale_removal_due.db and the offboard returns. It used to sleep
|
||
# PARTNERSHIP_GRACE_HOURS inline — six hours by default — holding the lock and its job record
|
||
# open the whole time, reporting "running", and blocking any re-onboard behind it.
|
||
#
|
||
# Dry Run Support
|
||
# --dry-run walks the full sequence reporting each step without executing any.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_PARTNERSHIP_AUTH_STACK
|
||
# Auth container XMLs to push during onboard — used on offboard to identify what
|
||
# to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed
|
||
# from the mirror on both owner-initiated and mirror-initiated offboard.
|
||
#
|
||
# HOST*_PARTNERSHIP_ARR_STACK
|
||
# Arr container XMLs — same cleanup logic as auth stack.
|
||
#
|
||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||
# Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) — same cleanup logic.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# Partnership/partnership_offboard.sh
|
||
# Full offboard — role detected automatically
|
||
#
|
||
# Partnership/partnership_offboard.sh --dry-run
|
||
# Preview all steps without executing
|
||
#
|
||
# Partnership/partnership_offboard.sh --log
|
||
# Verbose per-step output
|
||
#
|
||
# Partnership/partnership_offboard.sh --reason=<string>
|
||
# Tag the offboard reason in state file and blocklist (default: manual)
|
||
# Called by partnership_manager.sh --offboard (reason passed through)
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||
SSH_TIMEOUT=15
|
||
|
||
source "$SCRIPTS_ROOT/load_config.sh"
|
||
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
||
|
||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||
REASON="manual"
|
||
STEP_DISABLE_RSYNC_OK=true # both paths report it; only the mirror path re-initialised it
|
||
TAILSCALE_REMOVED=false # set only when remove_tailscale_device actually succeeds
|
||
|
||
# Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from
|
||
# MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded.
|
||
# An offboard that failed to remove a single container still reported a clean teardown.
|
||
STEP_LOCAL_CLEANUP_OK=true
|
||
STEP_OWN_STACK_OK=true
|
||
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
|
||
STEP_MIRROR_STACK_OK=true # or "skipped"
|
||
STEP_STATE_WRITE_OK=true
|
||
FILTERED_ARGS=()
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--reason=*) REASON="${arg#--reason=}" ;;
|
||
*) FILTERED_ARGS+=("$arg") ;;
|
||
esac
|
||
done
|
||
|
||
parse_args "${FILTERED_ARGS[@]}"
|
||
|
||
# ── Source partnership_manager.sh for shared helpers ──────────────────────────────────────────
|
||
# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed.
|
||
PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
|
||
partnership_resolve_roles
|
||
|
||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||
OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db"
|
||
MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db"
|
||
OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db"
|
||
|
||
acquire_lock "strict"
|
||
|
||
trap _pm_trap_restart_stopped EXIT
|
||
|
||
# Check already offboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||
warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
START=$(date +%s)
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
||
echo " Reason: $REASON"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER: disarm every sync gate a completed onboard armed ─────────────────────────────────
|
||
#
|
||
# The exact inverse of partnership_onboard.sh Step 9c, which sets RSYNC_ENABLED,
|
||
# CONF_SYNC_ENABLED and ARR_SYNC_ENABLED to true on a completed onboard. Offboard used to turn
|
||
# off CRITICAL_RSYNC_ENABLED and nothing else, which left the far more consequential Tier 1
|
||
# RSYNC_ENABLED switched on with no partner to sync to.
|
||
#
|
||
# That asymmetry read as safe and was not. Turning off one Tier 2 gate looks like "sync is off"
|
||
# in the summary, while INTERMEDIATE, DAILY, WEEKLY and FALLBACK rsync all stayed live behind an
|
||
# open Tier 1 — and CONF_SYNC_ENABLED kept a 4-hourly job reaching for a partner that had just
|
||
# been removed, failing and notifying each time.
|
||
#
|
||
# CRITICAL_RSYNC_ENABLED stays in the list. It is a Tier 2 gate and closing Tier 1 already stops
|
||
# it, but leaving it true would misreport the state to anyone reading the conf rather than the
|
||
# tier logic.
|
||
#
|
||
# FALLBACK_ENABLED is deliberately NOT here. Onboard does not arm it, so offboard has no business
|
||
# disarming it — it is the operator's switch, and the summary says so rather than moving it.
|
||
# ==============================================================================================
|
||
_VV_SYNC_GATES=(RSYNC_ENABLED CRITICAL_RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED)
|
||
|
||
_disarm_sync_gates() {
|
||
local gate rc=0 conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||
for gate in "${_VV_SYNC_GATES[@]}"; do
|
||
# set_conf_bool, not update_master_conf: the latter rewrites the whole line and would
|
||
# strip the trailing comment that explains what each tier gates. Same helper onboard
|
||
# arms with, so arming and disarming are one operation in two directions.
|
||
set_conf_bool "$gate" "false" "$conf" || rc=1
|
||
done
|
||
return "$rc"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
||
#
|
||
# Mirror-initiated path only. Called before start_own_stack so Emby is still running.
|
||
# Uses local EMBY_API_KEY and the mirror's own short name as the username to delete.
|
||
# ==============================================================================================
|
||
revoke_local_emby_admin() {
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://127.0.0.1:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping local Emby admin revocation"
|
||
return 1
|
||
fi
|
||
|
||
# The account to revoke is this server's own short name (the mirror user's account)
|
||
local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}"
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
log "Looking up Emby user '$username' at $emby_url..."
|
||
|
||
local users_json user_id
|
||
users_json=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
user_id=$(echo "$users_json" | \
|
||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
local del_code
|
||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X DELETE \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||
|
||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||
echo "Emby admin '$username' removed ✅"
|
||
else
|
||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||
warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
fi
|
||
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||
OWNER_REACHABLE=false
|
||
[[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true
|
||
|
||
STEP_STOP_RSYNC_OK=true
|
||
STEP_WEBUI_OK=true
|
||
STEP_STACK_CLEANUP_OK=true
|
||
STEP_FALLBACK_CLEANUP_OK=true
|
||
STEP_DISABLE_RSYNC_OK=true
|
||
STEP_EMBY_OK=true
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
|
||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||
echo "Rsync stopped ✅"
|
||
else
|
||
warn "DRY RUN — would stop rsync"
|
||
fi
|
||
|
||
# ── Step 2: Reconfigure local WebUIs → localhost ──────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━"
|
||
|
||
reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false
|
||
|
||
# ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ─────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━"
|
||
|
||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||
cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false
|
||
else
|
||
warn "Owner unreachable — cannot read deployed stack list"
|
||
warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable"
|
||
STEP_STACK_CLEANUP_OK=false
|
||
fi
|
||
|
||
# ── Step 4: Remove fallback coverage containers ───────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━"
|
||
|
||
cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false
|
||
|
||
# ── Step 5: Disarm the sync gates ─────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Step 5/8 — Disarm Sync Gates ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
_disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false
|
||
else
|
||
warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}"
|
||
fi
|
||
|
||
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
||
revoke_local_emby_admin || STEP_EMBY_OK=false
|
||
|
||
# ── Step 7: Restart own stack ─────────────────────────────────────────────────────────────
|
||
start_own_stack
|
||
|
||
# ── Step 8: SSH key revocation, write state, signal owner ────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
||
|
||
do_ssh_key_revocation "${OWNER_IP:-}"
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
echo "Local state: INACTIVE ✅"
|
||
add_to_blocklist "$OWNER" "$REASON"
|
||
else
|
||
warn "DRY RUN — would write INACTIVE state and blocklist $OWNER"
|
||
fi
|
||
|
||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
||
END=$(date +%s)
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━"
|
||
echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Owner: $OWNER_ID ($OWNER)"
|
||
echo " Reason: $REASON"
|
||
echo " Duration: $(format_duration $(( END - START )))"
|
||
echo ""
|
||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||
_revoke_status() {
|
||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "both directions ✅"
|
||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "local only ✅ — remote failed (revoke manually on $OWNER)"
|
||
else
|
||
echo "⚠️ failed — check warnings above"
|
||
fi
|
||
}
|
||
|
||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")"
|
||
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
||
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
||
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
||
echo " Step 5 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
|
||
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
||
echo " Step 7 — Own stack: started"
|
||
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
||
echo ""
|
||
echo " State: INACTIVE ✅"
|
||
echo " Blocklist: $OWNER blocked ✅"
|
||
echo " Owner: will finalise + final sync on next --check"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||
warn "$ICON_DONE DONE — mirror separation complete ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||
warn "Final sync will run — mirror leaves with current state"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
echo "Proceeding..."
|
||
fi
|
||
|
||
WEBUI_FAILURES=0
|
||
STEP_STOP_OK=true
|
||
STEP_SYNC_OK=true
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
|
||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
|
||
echo "Rsync stopped ✅"
|
||
else
|
||
warn "DRY RUN — would stop rsync"
|
||
fi
|
||
|
||
# ── Step 2: Final sync ────────────────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━"
|
||
|
||
do_final_sync || STEP_SYNC_OK=false
|
||
|
||
# ── Step 3: Reconfigure mirror WebUIs → localhost ─────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━"
|
||
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||
MIRROR_REACHABLE=false
|
||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "localhost" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
else
|
||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||
(( WEBUI_FAILURES++ ))
|
||
fi
|
||
|
||
# ── Step 4: Disarm the sync gates ─────────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Step 4/10 — Disarm Sync Gates ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
_disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false
|
||
else
|
||
warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}"
|
||
fi
|
||
|
||
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
||
|
||
cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false
|
||
|
||
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
||
start_own_stack || STEP_OWN_STACK_OK=false
|
||
|
||
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━"
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false
|
||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||
cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false
|
||
else
|
||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
||
STEP_REMOTE_CLEANUP_OK=skipped
|
||
fi
|
||
|
||
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false
|
||
else
|
||
STEP_MIRROR_STACK_OK=skipped
|
||
fi
|
||
|
||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
||
|
||
# Emby admin — before SSH key revocation while Emby still reachable
|
||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||
|
||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||
|
||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||
# State is written after container cleanup and SSH revocation so that:
|
||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
# Checked, because this is the record every other host and every later --check reads. A
|
||
# failed write here leaves both sides believing the partnership is still active while the
|
||
# summary says INACTIVE — the one line in the teardown that must not be assumed.
|
||
if write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then
|
||
echo "Local state: INACTIVE ✅"
|
||
else
|
||
error "Failed to write local state file — $MIRROR may still look ACTIVE here"
|
||
STEP_STATE_WRITE_OK=false
|
||
fi
|
||
add_to_blocklist "$MIRROR" "$REASON"
|
||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
else
|
||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||
fi
|
||
|
||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||
# The grace period is recorded as a deadline, not slept through.
|
||
#
|
||
# This used to `sleep $((PARTNERSHIP_GRACE_HOURS * 3600))` inline — six hours by default —
|
||
# holding the offboard's lock and its job record open the whole time, showing "running" to
|
||
# every status reader, and blocking any re-onboard behind the lock. Worse, the sleep ran
|
||
# even when removal was going to be a no-op: TAILSCALE_API_KEY and TAILSCALE_TAILNET are
|
||
# both empty here, so the six hours bought nothing at all.
|
||
#
|
||
# The offboard's own work is finished by this point. Writing the deadline lets the teardown
|
||
# complete now and leaves the removal to whoever reads the file — and makes the wait
|
||
# visible and cancellable instead of buried in a sleeping process.
|
||
_grace_h="${PARTNERSHIP_GRACE_HOURS:-6}"
|
||
if [[ "$_grace_h" -gt 0 ]] && [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
_due=$(( $(date +%s) + _grace_h * 3600 ))
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
printf 'host=%s\ndue=%s\ndue_human=%s\nreason=%s\n' \
|
||
"$MIRROR" "$_due" "$(date -d "@$_due" '+%Y-%m-%d %H:%M:%S')" "$REASON" \
|
||
> "${STATE_DIR}/tailscale_removal_due.db"
|
||
fi
|
||
warn "Grace period: $MIRROR stays on the tailnet until $(date -d "@$_due" '+%Y-%m-%d %H:%M') — recorded, not slept"
|
||
# No CLI entry point removes it yet, and there is deliberately no invented one here:
|
||
# remove_tailscale_device() is a partnership_manager.sh function with no --mode of its
|
||
# own, and it no-ops without credentials regardless. Say what is true.
|
||
if [[ -z "${TAILSCALE_API_KEY:-}" || -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||
warn "Automatic removal is not possible — TAILSCALE_API_KEY/TAILSCALE_TAILNET are unset; remove it in the Tailscale admin console"
|
||
else
|
||
warn "Removal after that is not yet automated — remove it in the Tailscale admin console"
|
||
fi
|
||
TAILSCALE_REMOVED=deferred
|
||
unset _grace_h _due
|
||
else
|
||
# No grace configured, or the mirror is already unreachable — remove now.
|
||
#
|
||
# Outcome recorded, not assumed. remove_tailscale_device returns 1 when TAILSCALE_API_KEY
|
||
# or TAILSCALE_TAILNET is unset — it warns "skipping Tailscale removal" and the summary
|
||
# went on to report "removed ✅" anyway, so an offboard that left the device on the
|
||
# tailnet said it had taken it off. Neither key is configured here, so that was every run.
|
||
if remove_tailscale_device "$MIRROR"; then
|
||
TAILSCALE_REMOVED=true
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# Backup handover notification
|
||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||
echo "Backups available for $MIRROR:"
|
||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||
[[ -z "$path" ]] && continue
|
||
echo " $path"
|
||
done
|
||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||
"Partnership" "warning"
|
||
fi
|
||
|
||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||
END=$(date +%s)
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━"
|
||
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||
echo " Reason: $REASON"
|
||
echo " Duration: $(format_duration $(( END - START )))"
|
||
echo ""
|
||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||
_revoke_status() {
|
||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "both directions ✅"
|
||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "local only ✅ — remote failed (revoke manually on $MIRROR)"
|
||
else
|
||
echo "⚠️ failed — check warnings above"
|
||
fi
|
||
}
|
||
|
||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
||
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||
echo " Step 4 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
|
||
echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")"
|
||
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
|
||
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
|
||
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
|
||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
||
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
|
||
echo ""
|
||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
case "$TAILSCALE_REMOVED" in
|
||
true) echo " Tailscale: $MIRROR removed ✅" ;;
|
||
deferred) echo " Tailscale: $MIRROR kept until the grace period expires — see ${STATE_DIR}/tailscale_removal_due.db" ;;
|
||
*) echo " Tailscale: $MIRROR NOT removed ⚠ — still on the tailnet (needs TAILSCALE_API_KEY + TAILSCALE_TAILNET)" ;;
|
||
esac
|
||
fi
|
||
# Named because it is the one partnership switch neither onboard nor offboard moves, so it
|
||
# survives an offboard still true and there is nothing else that would ever mention it.
|
||
[[ "${FALLBACK_ENABLED:-false}" == true ]] && \
|
||
echo " FALLBACK_ENABLED is still true — nothing left to fail over to, turn it off yourself"
|
||
echo ""
|
||
echo " $MIRROR leaves with:"
|
||
echo " ✓ Current auth config (final sync)"
|
||
echo " ✓ Auth WebUIs → localhost"
|
||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|