Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
351 lines
16 KiB
Bash
Executable File
351 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Partnership Transfer ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Transfers ownership from the current owner to the current mirror. After
|
|
# transfer the roles are swapped: what was the mirror becomes the new owner,
|
|
# and what was the owner becomes the new mirror.
|
|
#
|
|
# No containers are moved — only config and WebUI targets are updated. Both
|
|
# servers remain in the partnership; the sync direction reverses on the next
|
|
# fallback.sh / critical_sync_maintenance.sh cycle.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# OWNER ONLY — mirror cannot run this script.
|
|
#
|
|
# Step 1: Health Verification — both servers pass N consecutive health checks
|
|
# Step 2: Pre-transfer Sync — final sync in current direction (owner → mirror)
|
|
# Step 3: Reconfigure WebUIs — new owner WebUIs → localhost
|
|
# new mirror WebUIs → new owner IP
|
|
# Step 4: Flip Ownership — update PARTNERSHIP_OWNER_HOST in master.conf
|
|
# on both servers
|
|
# Step 5: Write State — ACTIVE written locally and pushed to new mirror
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Config Changes Hands, Data Does Not
|
|
# No containers move and no appdata is copied. A transfer rewrites who is authoritative
|
|
# and where WebUIs point; the sync direction reverses naturally on the next fallback or
|
|
# critical-sync cycle. Keeping data out of the transfer is what makes it cheap enough to
|
|
# be reversible.
|
|
#
|
|
# Owner Initiates, Always
|
|
# Only the current owner can run this. The owner holds the authoritative config, so a
|
|
# mirror-initiated transfer would be writing ownership state it does not own — and if both
|
|
# sides ran it, neither would be owner.
|
|
#
|
|
# Prove Health Before Swapping
|
|
# Both servers must pass consecutive health checks first. Handing ownership to a partner
|
|
# that is unhealthy converts a recoverable situation into an outage with the authoritative
|
|
# side on the weaker host.
|
|
#
|
|
# Roles Swap Atomically
|
|
# Owner and mirror are two ends of one relationship, not independent flags. Any window
|
|
# where both believe they are owner — or neither does — is worse than the transfer simply
|
|
# failing, so the swap is written as one transition rather than two updates.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Owner-Only Enforcement
|
|
# The script reads MY_ID from detect_hosts() and exits immediately if it is
|
|
# not the current PARTNERSHIP_OWNER_HOST. The mirror cannot run a transfer.
|
|
#
|
|
# Explicit Confirmation String
|
|
# Requires the exact passphrase from PARTNERSHIP_TRANSFER_CONFIRM via
|
|
# --confirm=<value>. Without a matching string the transfer is cancelled
|
|
# before any steps execute. Prevents accidental ownership changes.
|
|
#
|
|
# Active Partnership Guard
|
|
# Reads the local state file and exits if the current state is INACTIVE.
|
|
# A transfer without an active partnership has no defined outcome.
|
|
#
|
|
# Dual Health Verification
|
|
# Both servers must pass PARTNERSHIP_TRANSFER_STRIKES consecutive health
|
|
# checks before proceeding. A single failure resets the strike counter.
|
|
# After PARTNERSHIP_TRANSFER_MAX_ATTEMPTS total attempts the transfer aborts.
|
|
#
|
|
# Pre-transfer Final Sync
|
|
# A full sync in the current direction (owner → mirror) runs immediately
|
|
# before roles flip. Ensures the mirror is current before it becomes the owner.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# PARTNERSHIP_OWNER_HOST
|
|
# Current owner host ID (e.g. "HOST1"). Updated on both servers after transfer.
|
|
#
|
|
# PARTNERSHIP_TRANSFER_CONFIRM
|
|
# Exact string required to confirm transfer (default: "i-understand-this-transfers-ownership").
|
|
# Pass via --confirm=<value>.
|
|
#
|
|
# PARTNERSHIP_TRANSFER_STRIKES
|
|
# Consecutive health checks both servers must pass before transfer proceeds (default: 3).
|
|
#
|
|
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS
|
|
# Max health check attempts before giving up (default: 20).
|
|
#
|
|
# CRITICAL_SYNC_SHARES
|
|
# Array of "path|profile" or "path" entries for do_final_sync().
|
|
#
|
|
# PARTNERSHIP_AUTH_WEBUIS
|
|
# Array of "ContainerName|WebUIPort" entries reconfigured during transfer.
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership
|
|
# Full transfer — owner detected automatically.
|
|
#
|
|
# Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership --dry-run
|
|
# Preview all steps without executing. Confirmation check is skipped in dry-run mode.
|
|
#
|
|
# Partnership/partnership_transfer.sh --confirm=i-understand-this-transfers-ownership --log
|
|
# Verbose per-step output.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
SSH_TIMEOUT=15
|
|
|
|
source "$SCRIPTS_ROOT/load_config.sh"
|
|
|
|
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
|
TRANSFER_CONFIRM_INPUT=""
|
|
FILTERED_ARGS=()
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ── Source partnership_manager.sh for shared helpers ──────────────────────────────────────────
|
|
# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed.
|
|
PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
partnership_resolve_roles
|
|
|
|
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
|
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
|
|
|
acquire_lock "strict"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Preflight ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Partnership Transfer — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
|
|
if [[ "$AM_MIRROR" == true ]]; then
|
|
error "Only the owner ($OWNER / $OWNER_ID) can run --transfer"
|
|
error "Run from $OWNER, or use --offboard and re-onboard with roles swapped"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
|
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
|
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
|
error "No active partnership — transfer requires an active partnership"
|
|
error "If roles are already correct, check PARTNERSHIP_OWNER_HOST in master.conf"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " Current owner: $OWNER_ID ($OWNER)"
|
|
echo " Current mirror: $MIRROR_ID ($MIRROR)"
|
|
echo ""
|
|
echo " After transfer:"
|
|
echo " New owner: $MIRROR_ID ($MIRROR)"
|
|
echo " New mirror: $OWNER_ID ($OWNER)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Confirmation ━━━
|
|
# ==============================================================================================
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
|
echo ""
|
|
echo "To proceed, pass exactly:"
|
|
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
|
echo ""
|
|
error "Transfer cancelled — confirmation required"
|
|
exit 1
|
|
fi
|
|
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
|
error "Confirmation string does not match — transfer cancelled"
|
|
exit 1
|
|
fi
|
|
log "Confirmation accepted"
|
|
else
|
|
warn "DRY RUN — confirmation check skipped"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 1: Health Verification ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Step 1: Health Verification ━━━"
|
|
log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
|
|
|
STRIKES=0
|
|
ATTEMPTS=0
|
|
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
|
|
|
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
|
(( ATTEMPTS++ ))
|
|
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
|
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
|
error "Transfer cancelled — try again when both servers are healthy"
|
|
exit 1
|
|
fi
|
|
|
|
if check_both_healthy; then
|
|
(( STRIKES++ ))
|
|
log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
|
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
|
else
|
|
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
|
STRIKES=0
|
|
sleep 30
|
|
fi
|
|
done
|
|
warn "Both servers healthy — proceeding ✅"
|
|
|
|
# Resolve IPs after health checks confirm reachability
|
|
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
|
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve mirror Tailscale IP"; exit 1; }
|
|
|
|
# Compute post-transfer roles
|
|
NEW_OWNER_ID="$MIRROR_ID"
|
|
NEW_MIRROR_ID="$OWNER_ID"
|
|
NEW_OWNER="$MIRROR"
|
|
NEW_MIRROR="$OWNER"
|
|
NEW_OWNER_IP="$MIRROR_IP"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 2: Pre-transfer Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Step 2: Pre-transfer Sync ━━━"
|
|
do_final_sync
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 3: Reconfigure WebUIs ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Step 3: Reconfigure WebUIs ━━━"
|
|
|
|
WEBUI_FAILURES=0
|
|
|
|
# New owner (current mirror, HOST2) WebUIs → localhost — it now manages itself directly
|
|
log "New owner ($NEW_OWNER) WebUIs → localhost"
|
|
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
container="${entry%%|*}"
|
|
port="${entry##*|}"
|
|
reconfigure_webui "$container" "$port" "localhost" \
|
|
"$SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
|
done
|
|
|
|
# New mirror (us, HOST1) WebUIs → new owner IP — defers to new owner going forward
|
|
log "New mirror ($NEW_MIRROR) WebUIs → $NEW_OWNER_IP"
|
|
reconfigure_local_webuis "$NEW_OWNER_IP"
|
|
WEBUI_RC=$?
|
|
[[ "$WEBUI_RC" -gt 0 ]] && (( WEBUI_FAILURES += WEBUI_RC ))
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 4: Flip Ownership in master.conf ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 4: Flip Ownership ━━━"
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
|
|
|
|
# Push updated master.conf to new owner so both servers agree immediately.
|
|
# master.conf is shared — host-specific credentials live in host*.conf.
|
|
_REMOTE_SD=$(resolve_remote_scripts_dir "$MIRROR_IP" "$SSH_KEY" "no")
|
|
scp -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o StrictHostKeyChecking=no \
|
|
-o BatchMode=yes \
|
|
"$SCRIPTS_ROOT/Configurations/master.conf" \
|
|
"root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
|
|
echo "master.conf pushed to $NEW_OWNER ✅" || \
|
|
error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually"
|
|
else
|
|
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Step 5: Write State ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Step 5: Write State ━━━"
|
|
|
|
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
|
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
|
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$SSH_KEY"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
|
echo " New owner: $NEW_OWNER_ID ($NEW_OWNER — $NEW_OWNER_IP)"
|
|
echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR)"
|
|
echo " WebUI failures: $WEBUI_FAILURES"
|
|
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR (next cycle)"
|
|
echo " Ownership: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID"
|
|
echo ""
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
else
|
|
warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ($NEW_OWNER) ✅"
|
|
echo "fallback.sh and critical_sync_maintenance.sh will adapt on next cycle"
|
|
echo "No containers were moved — only config and WebUI targets updated"
|
|
[[ "$WEBUI_FAILURES" -gt 0 ]] && \
|
|
warn "$WEBUI_FAILURES WebUI(s) failed — check templates manually"
|
|
notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \
|
|
"Partnership" "normal"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|