#!/bin/bash # ----------------------------------------------------------------------------------------------- # ----------------------------- Partnership Manager -------------------------------------------- # ----------------------------------------------------------------------------------------------- # Manages the relationship lifecycle between two unRAID servers. # HOST1 is always the owner (source of truth). HOST2 is always the mirror. # PARTNERSHIP_OWNER_HOST flips to "HOST2" after --transfer. # # Modes: # --onboard ← owner only — set up mirror relationship # Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP # HOST2 clicks NPM → gets HOST1's NPM automatically # # --offboard ← either server — clean separation # Either party can initiate # HOST2 offboard: writes state, reconfigures own WebUIs → localhost # HOST1 sees it on next --check → finalises, runs final sync # HOST1 offboard: final sync, reconfigures HOST2 WebUIs, removes Tailscale # Both leave with current state, clean exit ✅ # # --transfer ← owner only — flip ownership # Requires confirmation string + health strike system # Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in Master.conf # # --check ← called by critical_sync_maintenance.sh every 15min # Reads both state files via SSH # Detects offboard requests → finalises from owner side # Updates last_seen_remote timestamp # Increments offline counter → auto-offboards after threshold # Silent when healthy # # --status ← either server — show current state # # State files on /boot/config (survives reboots, available before array): # /boot/config/partnership_HOST1.db ← HOST1 writes only # /boot/config/partnership_HOST2.db ← HOST2 writes only # Propagated via SSH — no rsync needed # # All configuration in Master.conf under PARTNERSHIP section. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" # Parse flags before parse_args MODE="" TRANSFER_CONFIRM_INPUT="" REMOTE_SEEN=false REMOTE_UNSEEN=false FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --onboard) MODE="onboard" ;; --offboard) MODE="offboard" ;; --transfer) MODE="transfer" ;; --check) MODE="check" ;; --status) MODE="status" ;; --confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;; --remote-seen) REMOTE_SEEN=true ;; --remote-unseen) REMOTE_UNSEEN=true ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- detect_hosts # Derive owner and mirror from PARTNERSHIP_OWNER_HOST if [[ "${PARTNERSHIP_OWNER_HOST:-HOST1}" == "HOST1" ]]; then OWNER="$HOST1" MIRROR="$HOST2" OWNER_SSH_KEY="$HOST1_SSH_KEY" MIRROR_SSH_KEY="$HOST2_SSH_KEY" else OWNER="$HOST2" MIRROR="$HOST1" OWNER_SSH_KEY="$HOST2_SSH_KEY" MIRROR_SSH_KEY="$HOST1_SSH_KEY" fi AM_OWNER=false AM_MIRROR=false [[ "$LOCAL_SERVER_NAME" == "$OWNER" ]] && AM_OWNER=true [[ "$LOCAL_SERVER_NAME" == "$MIRROR" ]] && AM_MIRROR=true # State files LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db" OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db" MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db" # Offline counter file OFFLINE_COUNTER="/boot/config/partnership_offline_days.db" if [[ -z "$MODE" ]]; then error "No mode specified" echo "Usage:" echo " partnership_manage.sh --onboard [--dry-run]" echo " partnership_manage.sh --offboard [--dry-run]" echo " partnership_manage.sh --transfer --confirm=... [--dry-run]" echo " partnership_manage.sh --check --remote-seen|--remote-unseen" echo " partnership_manage.sh --status" exit 1 fi # Role-based access control if [[ "$AM_MIRROR" == true ]]; then case "$MODE" in onboard|transfer) error "Only the owner ($OWNER) can run --$MODE" error "Run from $OWNER or use --offboard to separate cleanly" exit 1 ;; esac fi # Acquire lock for all modes except check (check is called frequently, lock would pile up) [[ "$MODE" != "check" ]] && acquire_lock "strict" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" # ----------------------------------------------------------------------------------------------- # HELPERS # ----------------------------------------------------------------------------------------------- read_state_file() { local file="$1" local key="$2" grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2 } write_state_file() { local file="$1" shift # flock to prevent concurrent writes ( flock -x 200 cat > "$file" << EOF state=${1:-UNKNOWN} owner=${OWNER} mirror=${MIRROR} onboarded=${2:-} offboarded=${3:-} triggered_by=${4:-} reason=${5:-} last_seen_remote=${6:-} updated=$(date '+%Y-%m-%d %H:%M:%S') EOF ) 200>"${file}.lock" } push_state_to_remote() { local local_file="$1" local remote_ip="$2" local ssh_key="$3" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push state file to remote" return 0 fi scp -i "$ssh_key" -o ConnectTimeout=10 \ "$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \ success "State file pushed to remote ✅" || \ warn "Could not push state file to remote — will propagate on next sync" } read_remote_state() { local remote_ip="$1" local ssh_key="$2" local remote_file="$3" ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ "cat '$remote_file' 2>/dev/null" 2>/dev/null } reconfigure_webui() { local container="$1" local port="$2" local target_ip="$3" local ssh_key="$4" local remote_ip="$5" local label="${6:-remote}" info "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/" return 0 fi local template template=$(ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ "grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null) if [[ -z "$template" ]]; then warn "$container template not found on $label — WebUI needs manual reconfiguration" return 1 fi ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ "sed -i 's|.*|http://${target_ip}:${port}/|g' '$template'" \ 2>/dev/null && \ success "$container → http://${target_ip}:${port}/ ✅" || { error "Failed to reconfigure $container WebUI" return 1 } } reconfigure_local_webuis() { local target_ip="$1" info "Reconfiguring local auth WebUIs → ${target_ip}..." local failures=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue local container="${entry%%|*}" local port="${entry##*|}" local template template=$(grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ xargs grep -l "\"$container\"" 2>/dev/null | head -1) if [[ -z "$template" ]]; then warn "$container template not found locally" ((failures++)) continue fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/" continue fi sed -i "s|.*|http://${target_ip}:${port}/|g" \ "$template" 2>/dev/null && \ success "$container → http://${target_ip}:${port}/ ✅" || \ { error "Failed to reconfigure $container"; ((failures++)); } done return $failures } get_tailscale_device_id() { local hostname="$1" curl -sf --max-time 10 \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \ 2>/dev/null | \ grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \ grep -o '"id":"[^"]*"' | \ grep -o '[^"]*"$' | tr -d '"' } remove_tailscale_device() { local hostname="$1" if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal" return 1 fi info "Looking up Tailscale device ID for $hostname..." local device_id device_id=$(get_tailscale_device_id "$hostname") if [[ -z "$device_id" ]]; then warn "Device $hostname not found in Tailscale — may already be removed" return 0 fi info "Removing $hostname (device $device_id) from Tailscale..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $hostname from Tailscale tailnet" return 0 fi curl -sf --max-time 10 -X DELETE \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \ success "$hostname removed from Tailscale ✅" || \ error "Failed to remove $hostname from Tailscale — remove manually" } check_both_healthy() { mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; } local mirror_ip mirror_ip=$(tailscale ip -4 "$MIRROR" 2>/dev/null) [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout=10 root@"$mirror_ip" \ "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || { error "Mirror not healthy" return 1 } return 0 } do_final_sync() { info "Running final Critical-Data sync..." if [[ "$DRY_RUN" == false ]]; then bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ "/mnt/user/appdata-Failover/Critical-Data" --log bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ "/mnt/user/Media_Server/Emby" --profile=emby-failover --log else warn "DRY RUN — would run final Critical-Data and emby-failover sync" fi success "Final sync complete — mirror has current state ✅" } # ----------------------------------------------------------------------------------------------- # ━━━ STATUS ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$MODE" == "status" ]]; then echo "" echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━" OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null || echo "unreachable") MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null || echo "unreachable") echo " Owner: $OWNER ($OWNER_IP)" echo " Mirror: $MIRROR ($MIRROR_IP)" echo " Enabled: ${PARTNERSHIP_ENABLED:-false}" echo " Running as: $LOCAL_SERVER_NAME ($( [[ "$AM_OWNER" == true ]] && echo "owner" || echo "mirror"))" echo "" # Local state if [[ -f "$LOCAL_STATE_FILE" ]]; then LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded") LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded") LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote") echo " Local state: $LOCAL_STATE" [[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED" [[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED" [[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote last seen: $LOCAL_LAST_SEEN" else echo " Local state: no state file found" fi echo "" # Remote state via SSH REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) if [[ -n "$REMOTE_IP" ]]; then REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -n "$REMOTE_CONTENT" ]]; then REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2) REMOTE_OFFBOARDED=$(echo "$REMOTE_CONTENT" | grep "^offboarded=" | cut -d= -f2) echo " Remote state: $REMOTE_STATE" [[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED" [[ -n "$REMOTE_OFFBOARDED" ]] && echo " Offboarded: $REMOTE_OFFBOARDED" # Check agreement if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then echo "" echo " ✅ Both servers agree: $LOCAL_STATE" else echo "" echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE" echo " Run --onboard or --offboard to resolve" fi else echo " Remote state: state file not found on $REMOTE_SERVER_NAME" fi else echo " Remote state: $REMOTE_SERVER_NAME unreachable" fi echo "" echo " Auth WebUIs configured:" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue echo " ${entry%%|*} → port ${entry##*|}" done # Offline counter if [[ -f "$OFFLINE_COUNTER" ]]; then OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) [[ "$OFFLINE_DAYS" -gt 0 ]] && \ echo "" && \ echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD} days" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ----------------------------------------------------------------------------------------------- # ━━━ CHECK (called every 15min by critical_sync_maintenance.sh) ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$MODE" == "check" ]]; then # Update last_seen_remote and offline counter based on rsync outcome if [[ "$REMOTE_SEEN" == true ]]; then OFFLINE_COUNT=0 echo "0" > "$OFFLINE_COUNTER" # Update last_seen_remote in local state file if [[ -f "$LOCAL_STATE_FILE" ]]; then sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \ "$LOCAL_STATE_FILE" 2>/dev/null fi elif [[ "$REMOTE_UNSEEN" == true ]]; then OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 )) echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER" # Check auto-offboard threshold # Convert 15min intervals to days: threshold_days * 96 intervals/day THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 )) if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard" notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days. Both servers will offboard independently." \ "Partnership" "warning" bash "$0" --offboard --reason="auto-offboard-timeout" exit 0 fi log "Partnership check — remote unseen (count: $OFFLINE_COUNT/$THRESHOLD_INTERVALS)" fi # Read remote state file REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) if [[ -z "$REMOTE_IP" ]]; then log "Partnership check — remote unreachable, skipping state check" exit 0 fi REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") [[ -z "$REMOTE_CONTENT" ]] && log "Partnership check — remote state file not found" && exit 0 REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN") # Both agree — healthy if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then log "Partnership check — ACTIVE, both servers agree ✅" exit 0 fi # Remote requested offboard if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then info "Partnership check — $REMOTE_SERVER_NAME requested offboard" if [[ "$AM_OWNER" == true ]]; then info "Owner finalising offboard request from mirror..." # Final sync — mirror leaves with current state do_final_sync # Update local state NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested" # Disable critical rsync if [[ "$DRY_RUN" == false ]]; then sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \ "$SCRIPT_DIR/../Master.conf" 2>/dev/null fi # Tailscale removal after grace period if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 )) info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see notification..." [[ "$DRY_RUN" == false ]] && sleep "$grace_seconds" remove_tailscale_device "$MIRROR" fi notify "Partnership offboard finalised on $(hostname) — $MIRROR requested separation" \ "Partnership" "normal" else # Mirror sees owner is INACTIVE — clean up own side info "Owner has offboarded — cleaning up mirror side..." reconfigure_local_webuis "localhost" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded" notify "Partnership ended on $(hostname) — $OWNER has offboarded. Auth WebUIs reconfigured to localhost." \ "Partnership" "normal" fi exit 0 fi # Both inactive — nothing to do if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then log "Partnership check — INACTIVE on both servers" exit 0 fi log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE" exit 0 fi # ----------------------------------------------------------------------------------------------- # ━━━ ONBOARD ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$MODE" == "onboard" ]]; then echo "" echo "━━━ $ICON_FAILOVER Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # Check already onboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then info "Partnership already ACTIVE — no action needed" info "Use --status for detail or --offboard to separate" exit 0 fi fi info "Owner: $OWNER" info "Mirror: $MIRROR" # Pre-flight echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" resolve_remote_ip check_connectivity check_remote_array || exit 1 check_remote_docker || exit 1 OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null) [[ -z "$OWNER_IP" ]] && error "Cannot resolve $OWNER Tailscale IP" && exit 1 [[ -z "$MIRROR_IP" ]] && error "Cannot resolve $MIRROR Tailscale IP" && exit 1 success "Owner IP: $OWNER_IP" success "Mirror IP: $MIRROR_IP" # Reconfigure mirror WebUIs → owner IP echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$OWNER_IP" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++)) done # Verify WebUI connectivity if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then echo "" echo "━━━ $ICON_HEALTH Verify WebUI Connectivity ━━━" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then success "$container reachable at http://${OWNER_IP}:${port}/ ✅" else warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running" fi done fi # Write state files echo "" echo "━━━ $ICON_GEAR Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" success "Local state: ACTIVE ✅" # Push to remote push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" # Reset offline counter echo "0" > "$OFFLINE_COUNTER" # Summary echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" echo " Owner: $OWNER ($OWNER_IP)" echo " Mirror: $MIRROR ($MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min" echo "" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS DONE" [[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \ notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \ "Partnership" "normal" else echo "$ICON_WARN Status: DONE with $WEBUI_FAILURES WebUI warning(s)" notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \ "Partnership" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ----------------------------------------------------------------------------------------------- # ━━━ OFFBOARD ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$MODE" == "offboard" ]]; then echo "" echo "━━━ $ICON_FAILOVER Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # Check already offboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then info "Partnership already INACTIVE" info "Use --status to verify both servers agree" exit 0 fi fi REASON="${REASON:-manual}" # ── MIRROR-initiated offboard ──────────────────────────────────────────────────────────── if [[ "$AM_MIRROR" == true ]]; then warn "$MIRROR is initiating offboard" warn "This will reconfigure your local auth WebUIs → localhost" warn "HOST1 will finalise the offboard on its next --check cycle" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 fi # Reconfigure own WebUIs → localhost echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━" reconfigure_local_webuis "localhost" # Write own state file NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" success "Local state: INACTIVE ✅" # Try to push state to owner immediately OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) if [[ -n "$OWNER_IP" ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY" notify "Partnership offboard requested by $MIRROR — owner will finalise on next check cycle" \ "Partnership" "normal" else warn "$OWNER unreachable — state written locally, owner will see it when reachable" fi echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" echo " Your WebUIs: reconfigured → localhost ✅" echo " State: INACTIVE written ✅" echo " Owner: will finalise + run final sync on next --check ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ── OWNER-initiated offboard ───────────────────────────────────────────────────────────── warn "Offboarding $MIRROR from partnership" warn "Final sync will run — mirror leaves with current state" warn "$MIRROR auth WebUIs will be reconfigured → localhost" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 echo "Proceeding..." fi resolve_remote_ip # Stop any running rsync first echo "" echo "━━━ $ICON_STOP Stop Running Rsync ━━━" bash "$SCRIPT_DIR/../unRAID_Essentials/rsync_stop.sh" --rsync-only 2>/dev/null || true # Final sync echo "" echo "━━━ $ICON_SYNC Final Sync ━━━" do_final_sync MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null) MIRROR_REACHABLE=false [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true # Reconfigure mirror WebUIs → localhost echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━" WEBUI_FAILURES=0 if [[ "$MIRROR_REACHABLE" == true ]]; then for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "localhost" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++)) done else warn "$MIRROR unreachable — WebUI reconfiguration skipped" warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on next --check" ((WEBUI_FAILURES++)) fi # Disable critical rsync echo "" echo "━━━ $ICON_GEAR Disable Critical Sync ━━━" if [[ "$DRY_RUN" == false ]]; then sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \ "$SCRIPT_DIR/../Master.conf" 2>/dev/null success "CRITICAL_RSYNC_ENABLED=false ✅" else warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" fi # Write state files echo "" echo "━━━ $ICON_GEAR Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" success "Local state: INACTIVE ✅" # Push to mirror if [[ "$MIRROR_REACHABLE" == true ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" fi # Tailscale removal if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then echo "" echo "━━━ $ICON_NET Tailscale Separation ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 )) info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see state..." [[ "$DRY_RUN" == false ]] && sleep "$grace_seconds" fi remove_tailscale_device "$MIRROR" fi # Backup notification if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_DISK Backup Handover ━━━" info "Backups available for $MIRROR:" for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do [[ -z "$path" ]] && continue echo " $path" done info "Available for ${PARTNERSHIP_GRACE_HOURS:-3} hours — Tailscale access removed after this window" notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-3}hr. Tailscale access expires then. Certs valid ~60-90 days. Run tailscale up to rejoin." \ "Partnership" "warning" fi # Summary echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" echo " Owner: $OWNER" echo " Mirror: $MIRROR" echo " Final sync: complete ✅" echo " WebUI failures: $WEBUI_FAILURES" echo " Critical rsync: disabled ✅" echo " State: INACTIVE ✅" [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ echo " Tailscale: $MIRROR removed ✅" echo "" echo " $MIRROR leaves with:" echo " ✓ Current auth config (final sync)" echo " ✓ His own DDNS and updater" echo " ✓ His own Git mirror" echo " ✓ Auth WebUIs → localhost" echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-3}hr to collect backups before Tailscale removed" echo " ✓ Certs valid ~60-90 more days" echo " ✓ Full ecosystem — just stop the sync" if [[ "$DRY_RUN" == true ]]; then echo "" echo "$ICON_WARN Status: DRY RUN — no changes made" else echo "" echo "$ICON_DONE Status: $ICON_SUCCESS DONE — clean separation complete" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ----------------------------------------------------------------------------------------------- # ━━━ TRANSFER ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$MODE" == "transfer" ]]; then echo "" echo "━━━ $ICON_FAILOVER Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — OWNERSHIP TRANSFER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " Current owner: $OWNER" echo " Current mirror: $MIRROR" echo "" echo " After transfer:" echo " New owner: $MIRROR" echo " New mirror: $OWNER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Confirmation string if [[ "$DRY_RUN" == false ]]; then if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then echo "" echo "To proceed type exactly:" echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}" echo "" error "Transfer cancelled — confirmation required" exit 1 fi if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then error "Confirmation string does not match — transfer cancelled" exit 1 fi success "Confirmation accepted" else warn "DRY RUN — confirmation check skipped" fi # Health strike system echo "" echo "━━━ $ICON_SHIELD Health Verification ━━━" info "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" STRIKES=0 ATTEMPTS=0 MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}" while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do ((ATTEMPTS++)) if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable" error "Transfer cancelled — try again when both servers are healthy" exit 1 fi if check_both_healthy; then ((STRIKES++)) success "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10 else warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)" STRIKES=0 sleep 30 fi done success "Both servers healthy — proceeding" NEW_OWNER="$MIRROR" NEW_MIRROR="$OWNER" NEW_OWNER_IP=$(tailscale ip -4 "$NEW_OWNER" 2>/dev/null) NEW_MIRROR_IP=$(tailscale ip -4 "$NEW_MIRROR" 2>/dev/null) [[ -z "$NEW_OWNER_IP" ]] && error "Cannot resolve new owner Tailscale IP" && exit 1 # Final sync in current direction echo "" echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━" do_final_sync # Reconfigure new mirror WebUIs → new owner echo "" echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━" info "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)" NEW_MIRROR_SSH="${HOST1_SSH_KEY}" [[ "$NEW_MIRROR" == "$HOST2" ]] && NEW_MIRROR_SSH="${HOST2_SSH_KEY}" NEW_OWNER_SSH="${HOST1_SSH_KEY}" [[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_SSH="${HOST2_SSH_KEY}" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \ "$NEW_MIRROR_SSH" "$NEW_MIRROR_IP" "$NEW_MIRROR" || ((WEBUI_FAILURES++)) done # New owner WebUIs → localhost (now manages directly) info "New owner ($NEW_OWNER) WebUIs → localhost" reconfigure_local_webuis "localhost" # Flip PARTNERSHIP_OWNER_HOST in Master.conf on both servers echo "" echo "━━━ $ICON_GEAR Update Ownership ━━━" NEW_OWNER_HOST="HOST1" [[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_HOST="HOST2" if [[ "$DRY_RUN" == false ]]; then sed -i "s/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/" \ "$SCRIPT_DIR/../Master.conf" 2>/dev/null success "Local Master.conf updated: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST ✅" # Update remote ssh -i "$NEW_MIRROR_SSH" -o ConnectTimeout=10 root@"$NEW_MIRROR_IP" \ "sed -i 's/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/' \ '$SCRIPT_DIR/../Master.conf'" 2>/dev/null && \ success "Remote Master.conf updated ✅" || \ error "Failed to update remote Master.conf — update manually" else warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST on both servers" fi # Write state files NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer" push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH" # Summary echo "" echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━" echo " New owner: $NEW_OWNER ($NEW_OWNER_IP)" echo " New mirror: $NEW_MIRROR ($NEW_MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync direction: $NEW_OWNER → $NEW_MIRROR" echo " Master.conf: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" else echo "$ICON_DONE Status: $ICON_SUCCESS DONE — ownership transferred" notify "Partnership ownership transferred — new owner: $NEW_OWNER new mirror: $NEW_MIRROR" \ "Partnership" "normal" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi