339 lines
15 KiB
Bash
Executable File
339 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Partnership Transfer ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Transfers ownership from the current owner to the current mirror. After
|
|
# transfer the roles are swapped: what was the mirror becomes the new owner,
|
|
# and what was the owner becomes the new mirror.
|
|
#
|
|
# No containers are moved — only config and WebUI targets are updated. Both
|
|
# servers remain in the partnership; the sync direction reverses on the next
|
|
# fallback.sh / critical_sync_maintenance.sh cycle.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# OWNER ONLY — mirror cannot run this script.
|
|
#
|
|
# Step 1: Health Verification — both servers pass N consecutive health checks
|
|
# Step 2: Pre-transfer Sync — final sync in current direction (owner → mirror)
|
|
# Step 3: Reconfigure WebUIs — new owner WebUIs → localhost
|
|
# new mirror WebUIs → new owner IP
|
|
# Step 4: Flip Ownership — update PARTNERSHIP_OWNER_HOST in master.conf
|
|
# on both servers
|
|
# Step 5: Write State — ACTIVE written locally and pushed to new mirror
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Owner-Only Enforcement
|
|
# The script reads MY_ID from detect_hosts() and exits immediately if it is
|
|
# not the current PARTNERSHIP_OWNER_HOST. The mirror cannot run a transfer.
|
|
#
|
|
# Explicit Confirmation String
|
|
# Requires the exact passphrase from PARTNERSHIP_TRANSFER_CONFIRM via
|
|
# --confirm=<value>. Without a matching string the transfer is cancelled
|
|
# before any steps execute. Prevents accidental ownership changes.
|
|
#
|
|
# Active Partnership Guard
|
|
# Reads the local state file and exits if the current state is INACTIVE.
|
|
# A transfer without an active partnership has no defined outcome.
|
|
#
|
|
# Dual Health Verification
|
|
# Both servers must pass PARTNERSHIP_TRANSFER_STRIKES consecutive health
|
|
# checks before proceeding. A single failure resets the strike counter.
|
|
# After PARTNERSHIP_TRANSFER_MAX_ATTEMPTS total attempts the transfer aborts.
|
|
#
|
|
# Pre-transfer Final Sync
|
|
# A full sync in the current direction (owner → mirror) runs immediately
|
|
# before roles flip. Ensures the mirror is current before it becomes the owner.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# PARTNERSHIP_OWNER_HOST
|
|
# Current owner host ID (e.g. "HOST1"). Updated on both servers after transfer.
|
|
#
|
|
# PARTNERSHIP_TRANSFER_CONFIRM
|
|
# Exact string required to confirm transfer (default: "i-understand-this-transfers-ownership").
|
|
# Pass via --confirm=<value>.
|
|
#
|
|
# PARTNERSHIP_TRANSFER_STRIKES
|
|
# Consecutive health checks both servers must pass before transfer proceeds (default: 3).
|
|
#
|
|
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS
|
|
# Max health check attempts before giving up (default: 20).
|
|
#
|
|
# CRITICAL_SYNC_SHARES
|
|
# Array of "path|profile" or "path" entries for do_final_sync().
|
|
#
|
|
# PARTNERSHIP_AUTH_WEBUIS
|
|
# Array of "ContainerName|WebUIPort" entries reconfigured during transfer.
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership
|
|
# Full transfer — owner detected automatically.
|
|
#
|
|
# Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership --dry-run
|
|
# Preview all steps without executing. Confirmation check is skipped in dry-run mode.
|
|
#
|
|
# Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership --log
|
|
# Verbose per-step output.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
SSH_TIMEOUT=15
|
|
|
|
source "$SCRIPTS_ROOT/load_config.sh"
|
|
|
|
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
|
TRANSFER_CONFIRM_INPUT=""
|
|
FILTERED_ARGS=()
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ── Source partnership_manager.sh for shared helpers ──────────────────────────────────────────
|
|
# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed.
|
|
PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
|
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
|
OWNER="${!OWNER_ID}"
|
|
MIRROR="${!MIRROR_ID}"
|
|
MIRROR_SSH_KEY="$SSH_KEY"
|
|
OWNER_SSH_KEY="$SSH_KEY"
|
|
|
|
AM_OWNER=false
|
|
AM_MIRROR=false
|
|
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
|
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
|
|
|
LOCAL_STATE_FILE="${STATE_DIR:-/boot/config}/partnership_${LOCAL_SERVER_NAME}.db"
|
|
REMOTE_STATE_FILE="${STATE_DIR:-/boot/config}/partnership_${REMOTE_SERVER_NAME}.db"
|
|
|
|
acquire_lock "strict"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Preflight ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Partnership Transfer — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
|
|
if [[ "$AM_MIRROR" == true ]]; then
|
|
error "Only the owner ($OWNER / $OWNER_ID) can run --transfer"
|
|
error "Run from $OWNER, or use --offboard and re-onboard with roles swapped"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
|
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
|
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
|
error "No active partnership — transfer requires an active partnership"
|
|
error "If roles are already correct, check PARTNERSHIP_OWNER_HOST in master.conf"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " Current owner: $OWNER_ID ($OWNER)"
|
|
echo " Current mirror: $MIRROR_ID ($MIRROR)"
|
|
echo ""
|
|
echo " After transfer:"
|
|
echo " New owner: $MIRROR_ID ($MIRROR)"
|
|
echo " New mirror: $OWNER_ID ($OWNER)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Confirmation ━━━
|
|
# ==============================================================================================
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
|
echo ""
|
|
echo "To proceed, pass exactly:"
|
|
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
|
echo ""
|
|
error "Transfer cancelled — confirmation required"
|
|
exit 1
|
|
fi
|
|
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
|
error "Confirmation string does not match — transfer cancelled"
|
|
exit 1
|
|
fi
|
|
log "Confirmation accepted"
|
|
else
|
|
warn "DRY RUN — confirmation check skipped"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 1: Health Verification ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Step 1: Health Verification ━━━"
|
|
log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
|
|
|
STRIKES=0
|
|
ATTEMPTS=0
|
|
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
|
|
|
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
|
(( ATTEMPTS++ ))
|
|
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
|
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
|
error "Transfer cancelled — try again when both servers are healthy"
|
|
exit 1
|
|
fi
|
|
|
|
if check_both_healthy; then
|
|
(( STRIKES++ ))
|
|
log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
|
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
|
else
|
|
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
|
STRIKES=0
|
|
sleep 30
|
|
fi
|
|
done
|
|
warn "Both servers healthy — proceeding ✅"
|
|
|
|
# Resolve IPs after health checks confirm reachability
|
|
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
|
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve mirror Tailscale IP"; exit 1; }
|
|
|
|
# Compute post-transfer roles
|
|
NEW_OWNER_ID="$MIRROR_ID"
|
|
NEW_MIRROR_ID="$OWNER_ID"
|
|
NEW_OWNER="$MIRROR"
|
|
NEW_MIRROR="$OWNER"
|
|
NEW_OWNER_IP="$MIRROR_IP"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 2: Pre-transfer Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Step 2: Pre-transfer Sync ━━━"
|
|
do_final_sync
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 3: Reconfigure WebUIs ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Step 3: Reconfigure WebUIs ━━━"
|
|
|
|
WEBUI_FAILURES=0
|
|
|
|
# New owner (current mirror, HOST2) WebUIs → localhost — it now manages itself directly
|
|
log "New owner ($NEW_OWNER) WebUIs → localhost"
|
|
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
container="${entry%%|*}"
|
|
port="${entry##*|}"
|
|
reconfigure_webui "$container" "$port" "localhost" \
|
|
"$SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
|
done
|
|
|
|
# New mirror (us, HOST1) WebUIs → new owner IP — defers to new owner going forward
|
|
log "New mirror ($NEW_MIRROR) WebUIs → $NEW_OWNER_IP"
|
|
reconfigure_local_webuis "$NEW_OWNER_IP"
|
|
WEBUI_RC=$?
|
|
[[ "$WEBUI_RC" -gt 0 ]] && (( WEBUI_FAILURES += WEBUI_RC ))
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 4: Flip Ownership in master.conf ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 4: Flip Ownership ━━━"
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
|
|
|
|
# Push updated master.conf to new owner so both servers agree immediately.
|
|
# master.conf is shared — host-specific credentials live in host*.conf.
|
|
_REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
|
|
"root@${MIRROR_IP}" \
|
|
"grep -m1 '^SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'" \
|
|
2>/dev/null | tr -d '[:space:]')
|
|
_REMOTE_SD="${_REMOTE_SD:-/boot/config/plugins/varaverk}"
|
|
scp -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o StrictHostKeyChecking=no \
|
|
"$SCRIPTS_ROOT/Configurations/master.conf" \
|
|
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
|
|
log "master.conf pushed to $NEW_OWNER ✅" || \
|
|
error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually"
|
|
else
|
|
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 5: Write State ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Step 5: Write State ━━━"
|
|
|
|
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
|
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
|
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$SSH_KEY"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
|
echo " New owner: $NEW_OWNER_ID ($NEW_OWNER — $NEW_OWNER_IP)"
|
|
echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR)"
|
|
echo " WebUI failures: $WEBUI_FAILURES"
|
|
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR (next cycle)"
|
|
echo " Ownership: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID"
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
else
|
|
warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ($NEW_OWNER) ✅"
|
|
echo "fallback.sh and critical_sync_maintenance.sh will adapt on next cycle"
|
|
echo "No containers were moved — only config and WebUI targets updated"
|
|
[[ "$WEBUI_FAILURES" -gt 0 ]] && \
|
|
warn "$WEBUI_FAILURES WebUI(s) failed — check templates manually"
|
|
notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \
|
|
"Partnership" "normal"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|