#!/bin/bash # ============================================================================================== # ============================= Partnership Transfer ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Transfers ownership from the current owner to the current mirror. After # transfer the roles are swapped: what was the mirror becomes the new owner, # and what was the owner becomes the new mirror. # # No containers are moved — only config and WebUI targets are updated. Both # servers remain in the partnership; the sync direction reverses on the next # fallback.sh / critical_sync_maintenance.sh cycle. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # OWNER ONLY — mirror cannot run this script. # # Step 1: Health Verification — both servers pass N consecutive health checks # Step 2: Pre-transfer Sync — final sync in current direction (owner → mirror) # Step 3: Reconfigure WebUIs — new owner WebUIs → localhost # new mirror WebUIs → new owner IP # Step 4: Flip Ownership — update PARTNERSHIP_OWNER_HOST in master.conf # on both servers # Step 5: Write State — ACTIVE written locally and pushed to new mirror # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Config Changes Hands, Data Does Not # No containers move and no appdata is copied. A transfer rewrites who is authoritative # and where WebUIs point; the sync direction reverses naturally on the next fallback or # critical-sync cycle. Keeping data out of the transfer is what makes it cheap enough to # be reversible. # # Owner Initiates, Always # Only the current owner can run this. The owner holds the authoritative config, so a # mirror-initiated transfer would be writing ownership state it does not own — and if both # sides ran it, neither would be owner. # # Prove Health Before Swapping # Both servers must pass consecutive health checks first. Handing ownership to a partner # that is unhealthy converts a recoverable situation into an outage with the authoritative # side on the weaker host. # # Roles Swap Atomically # Owner and mirror are two ends of one relationship, not independent flags. Any window # where both believe they are owner — or neither does — is worse than the transfer simply # failing, so the swap is written as one transition rather than two updates. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Owner-Only Enforcement # The script reads MY_ID from detect_hosts() and exits immediately if it is # not the current PARTNERSHIP_OWNER_HOST. The mirror cannot run a transfer. # # Explicit Confirmation String # Requires the exact passphrase from PARTNERSHIP_TRANSFER_CONFIRM via # --confirm=. Without a matching string the transfer is cancelled # before any steps execute. Prevents accidental ownership changes. # # Active Partnership Guard # Reads the local state file and exits if the current state is INACTIVE. # A transfer without an active partnership has no defined outcome. # # Dual Health Verification # Both servers must pass PARTNERSHIP_TRANSFER_STRIKES consecutive health # checks before proceeding. A single failure resets the strike counter. # After PARTNERSHIP_TRANSFER_MAX_ATTEMPTS total attempts the transfer aborts. # # Pre-transfer Final Sync # A full sync in the current direction (owner → mirror) runs immediately # before roles flip. Ensures the mirror is current before it becomes the owner. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # PARTNERSHIP_OWNER_HOST # Current owner host ID (e.g. "HOST1"). Updated on both servers after transfer. # # PARTNERSHIP_TRANSFER_CONFIRM # Exact string required to confirm transfer (default: "i-understand-this-transfers-ownership"). # Pass via --confirm=. # # PARTNERSHIP_TRANSFER_STRIKES # Consecutive health checks both servers must pass before transfer proceeds (default: 3). # # PARTNERSHIP_TRANSFER_MAX_ATTEMPTS # Max health check attempts before giving up (default: 20). # # CRITICAL_SYNC_SHARES # Array of "path|profile" or "path" entries for do_final_sync(). # # PARTNERSHIP_AUTH_WEBUIS # Array of "ContainerName|WebUIPort" entries reconfigured during transfer. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership # Full transfer — owner detected automatically. # # Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership --dry-run # Preview all steps without executing. Confirmation check is skipped in dry-run mode. # # Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership --log # Verbose per-step output. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPTS_ROOT="$SCRIPT_DIR/.." SSH_TIMEOUT=15 source "$SCRIPTS_ROOT/load_config.sh" # ── Parse flags ─────────────────────────────────────────────────────────────────────────────── TRANSFER_CONFIRM_INPUT="" FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ── Source partnership_manager.sh for shared helpers ────────────────────────────────────────── # PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed. PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== [[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts partnership_resolve_roles LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db" acquire_lock "strict" # ============================================================================================== # ━━━ Preflight ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_FALLBACK Partnership Transfer — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" if [[ "$AM_MIRROR" == true ]]; then error "Only the owner ($OWNER / $OWNER_ID) can run --transfer" error "Run from $OWNER, or use --offboard and re-onboard with roles swapped" exit 1 fi if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then error "No active partnership — transfer requires an active partnership" error "If roles are already correct, check PARTNERSHIP_OWNER_HOST in master.conf" exit 1 fi fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — OWNERSHIP TRANSFER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " Current owner: $OWNER_ID ($OWNER)" echo " Current mirror: $MIRROR_ID ($MIRROR)" echo "" echo " After transfer:" echo " New owner: $MIRROR_ID ($MIRROR)" echo " New mirror: $OWNER_ID ($OWNER)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # ============================================================================================== # ━━━ Confirmation ━━━ # ============================================================================================== if [[ "$DRY_RUN" == false ]]; then if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then echo "" echo "To proceed, pass exactly:" echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}" echo "" error "Transfer cancelled — confirmation required" exit 1 fi if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then error "Confirmation string does not match — transfer cancelled" exit 1 fi log "Confirmation accepted" else warn "DRY RUN — confirmation check skipped" fi # ============================================================================================== # ━━━ Step 1: Health Verification ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SHIELD Step 1: Health Verification ━━━" log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" STRIKES=0 ATTEMPTS=0 MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}" while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do (( ATTEMPTS++ )) if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable" error "Transfer cancelled — try again when both servers are healthy" exit 1 fi if check_both_healthy; then (( STRIKES++ )) log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10 else warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)" STRIKES=0 sleep 30 fi done warn "Both servers healthy — proceeding ✅" # Resolve IPs after health checks confirm reachability MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") OWNER_IP=$(resolve_tailscale_ip "$OWNER") [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve mirror Tailscale IP"; exit 1; } # Compute post-transfer roles NEW_OWNER_ID="$MIRROR_ID" NEW_MIRROR_ID="$OWNER_ID" NEW_OWNER="$MIRROR" NEW_MIRROR="$OWNER" NEW_OWNER_IP="$MIRROR_IP" # ============================================================================================== # ━━━ Step 2: Pre-transfer Sync ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Step 2: Pre-transfer Sync ━━━" do_final_sync # ============================================================================================== # ━━━ Step 3: Reconfigure WebUIs ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Step 3: Reconfigure WebUIs ━━━" WEBUI_FAILURES=0 # New owner (current mirror, HOST2) WebUIs → localhost — it now manages itself directly log "New owner ($NEW_OWNER) WebUIs → localhost" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "localhost" \ "$SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done # New mirror (us, HOST1) WebUIs → new owner IP — defers to new owner going forward log "New mirror ($NEW_MIRROR) WebUIs → $NEW_OWNER_IP" reconfigure_local_webuis "$NEW_OWNER_IP" WEBUI_RC=$? [[ "$WEBUI_RC" -gt 0 ]] && (( WEBUI_FAILURES += WEBUI_RC )) # ============================================================================================== # ━━━ Step 4: Flip Ownership in master.conf ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Step 4: Flip Ownership ━━━" if [[ "$DRY_RUN" == false ]]; then update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\"" # Push updated master.conf to new owner so both servers agree immediately. # master.conf is shared — host-specific credentials live in host*.conf. _REMOTE_SD=$(resolve_remote_scripts_dir "$MIRROR_IP" "$SSH_KEY" "no") scp -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ -o StrictHostKeyChecking=no \ -o BatchMode=yes \ "$SCRIPTS_ROOT/Configurations/master.conf" \ "root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \ echo "master.conf pushed to $NEW_OWNER ✅" || \ error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually" else warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers" fi # ============================================================================================== # ━━━ Step 5: Write State ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Step 5: Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer" push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$SSH_KEY" # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━" echo " New owner: $NEW_OWNER_ID ($NEW_OWNER — $NEW_OWNER_IP)" echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync direction: $NEW_OWNER → $NEW_MIRROR (next cycle)" echo " Ownership: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ($NEW_OWNER) ✅" echo "fallback.sh and critical_sync_maintenance.sh will adapt on next cycle" echo "No containers were moved — only config and WebUI targets updated" [[ "$WEBUI_FAILURES" -gt 0 ]] && \ warn "$WEBUI_FAILURES WebUI(s) failed — check templates manually" notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \ "Partnership" "normal" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0