#!/bin/bash # ============================================================================================== # ================================= Partnership Manager ======================================== # ============================================================================================== # Manages the relationship lifecycle between two unRAID servers. # HOST1 is always the owner (source of truth). HOST2 is always the mirror. # PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer. # # ── MODES ───────────────────────────────────────────────────────────────────────────────────── # --onboard ← owner only — establish mirror relationship # Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP # HOST2 clicks NPM → gets HOST1's NPM automatically # # --offboard ← either server — clean separation # Mirror-initiated: writes INACTIVE state, reconfigures own WebUIs → localhost # Owner finalises on next --check: final sync, Tailscale removal # Owner-initiated: final sync, reconfigures mirror WebUIs, Tailscale removal # Both leave with current state, clean exit ✅ # # --transfer ← owner only — flip ownership between servers # Requires confirmation string + consecutive health strikes # Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf # # --check ← called by critical_sync_maintenance.sh every 15min # Reads both state files via SSH # Detects offboard requests → finalises from owner side # Updates last_seen_remote timestamp # Increments offline counter → auto-offboards after threshold # Silent when healthy ← never noisy on clean runs # # --status ← either server — show current state, both sides # # ── STATE FILES ─────────────────────────────────────────────────────────────────────────────── # On /boot/config — survives reboots, available before array starts: # /boot/config/partnership_HOST1.db ← HOST1 writes only # /boot/config/partnership_HOST2.db ← HOST2 writes only # Propagated via SSH — no rsync needed # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # Root check — all operations require root # validate_unraid_cmd — notify validated before use # Role enforcement — mirror cannot run owner-only modes # AM_OWNER / AM_MIRROR — all routing via these flags, not hostname strings # version parity — onboard checks both servers match unRAID version # remote docker daemon — onboard verifies remote daemon responsive # SSH_TIMEOUT — all SSH calls timeout-protected # flock on state writes — prevents concurrent state file corruption # SIGTERM trap — grace period sleep interruptible # Silent by default — only warns/errors produce output (--check is always silent healthy) # # ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── # HOST*_PARTNERSHIP_AUTH_WEBUIS — containers reconfigured on onboard/offboard # HOST*_PARTNERSHIP_MIRROR_BACKUPS — paths available after offboard # All aliased by detect_hosts() — script uses PARTNERSHIP_AUTH_WEBUIS etc. # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # PARTNERSHIP_ENABLED — global enable gate # PARTNERSHIP_OWNER_HOST — "HOST1" or "HOST2" — flips on --transfer # PARTNERSHIP_REMOVE_TAILSCALE — remove mirror from tailnet on offboard # PARTNERSHIP_GRACE_HOURS — hours before Tailscale removal after offboard # PARTNERSHIP_OFFLINE_THRESHOLD — days unreachable before auto-offboard # PARTNERSHIP_TRANSFER_CONFIRM — exact string required for --transfer # PARTNERSHIP_TRANSFER_STRIKES — consecutive health checks required # PARTNERSHIP_TRANSFER_MAX_ATTEMPTS — max attempts before giving up # PARTNERSHIP_ONBOARD_VERIFY — verify WebUI connectivity after onboard # PARTNERSHIP_ONBOARD_NOTIFY — notify both servers on onboard completion # PARTNERSHIP_SYNC_INTERVAL — informational — actual schedule in cron # TAILSCALE_API_KEY / TAILSCALE_TAILNET — required when PARTNERSHIP_REMOVE_TAILSCALE=true # # ── BLOCKLIST ───────────────────────────────────────────────────────────────────────────────── # After offboard, the former partner's hostname is written to: # /boot/config/partnership_blocklist.db (format: hostname|timestamp|reason) # # --onboard is hard-blocked if the remote is on the blocklist — exits with error. # --check silently skips remote state reads for blocklisted hosts (no noise every 15min). # --unblock removes an entry to permit re-onboarding. # --status shows the full blocklist. # # The blocklist persists until explicitly cleared — surviving reboots, array restarts, # and Tailscale reconnections. Tailscale removal is a separate step at the network layer; # the blocklist is the application-layer guard. # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # partnership_manage.sh --onboard # partnership_manage.sh --offboard # partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership # partnership_manage.sh --check --remote-seen|--remote-unseen # partnership_manage.sh --status # partnership_manage.sh --unblock # Any mode supports --dry-run and --log # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" SSH_TIMEOUT=15 BLOCKLIST_FILE="/boot/config/partnership_blocklist.db" # ── Parse mode flags before parse_args ──────────────────────────────────────────────────────── MODE="" TRANSFER_CONFIRM_INPUT="" REMOTE_SEEN=false REMOTE_UNSEEN=false REASON="manual" UNBLOCK_HOST="" FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --onboard) MODE="onboard" ;; --offboard) MODE="offboard" ;; --transfer) MODE="transfer" ;; --check) MODE="check" ;; --status) MODE="status" ;; --unblock) MODE="unblock" ;; --confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;; --remote-seen) REMOTE_SEEN=true ;; --remote-unseen) REMOTE_UNSEEN=true ;; --reason=*) REASON="${arg#--reason=}" ;; *) if [[ "$MODE" == "unblock" ]] && [[ -z "$UNBLOCK_HOST" ]] && [[ "$arg" != --* ]]; then UNBLOCK_HOST="$arg" else FILTERED_ARGS+=("$arg") fi ;; esac done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" # detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS detect_hosts # ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ─────────────────────────────────────── OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1" MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) OWNER="${!OWNER_ID}" # hostname string MIRROR="${!MIRROR_ID}" OWNER_SSH_KEY_VAR="${OWNER_ID}_SSH_KEY" MIRROR_SSH_KEY_VAR="${MIRROR_ID}_SSH_KEY" OWNER_SSH_KEY="${!OWNER_SSH_KEY_VAR}" MIRROR_SSH_KEY="${!MIRROR_SSH_KEY_VAR}" AM_OWNER=false AM_MIRROR=false [[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true [[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true # State files LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db" OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db" MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db" OFFLINE_COUNTER="/boot/config/partnership_offline_days.db" if [[ -z "$MODE" ]]; then error "No mode specified" echo "Usage:" echo " partnership_manage.sh --onboard" echo " partnership_manage.sh --offboard" echo " partnership_manage.sh --transfer --confirm=..." echo " partnership_manage.sh --check --remote-seen|--remote-unseen" echo " partnership_manage.sh --status" echo " partnership_manage.sh --unblock " exit 1 fi # Role-based access control if [[ "$AM_MIRROR" == true ]]; then case "$MODE" in onboard|transfer) error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE" error "Run from $OWNER or use --offboard to separate cleanly" exit 1 ;; esac fi # Lock for all modes except --check (check is called frequently, lock would pile up) [[ "$MODE" != "check" ]] && acquire_lock "strict" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== read_state_file() { local file="$1" key="$2" grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2 } write_state_file() { local file="$1" shift # flock prevents concurrent writes to the same state file ( flock -x 200 cat > "$file" << EOF state=${1:-UNKNOWN} owner=${OWNER} mirror=${MIRROR} onboarded=${2:-} offboarded=${3:-} triggered_by=${4:-} reason=${5:-} last_seen_remote=${6:-} updated=$(date '+%Y-%m-%d %H:%M:%S') EOF ) 200>"${file}.lock" } push_state_to_remote() { local local_file="$1" remote_ip="$2" ssh_key="$3" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push state file to remote" return 0 fi timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ "$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \ log "State file pushed to remote ✅" || \ warn "Could not push state file to remote — will propagate on next sync" } read_remote_state() { local remote_ip="$1" ssh_key="$2" remote_file="$3" timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ "cat '$remote_file' 2>/dev/null" 2>/dev/null } # Reconfigure a container's WebUI on the remote server reconfigure_webui() { local container="$1" port="$2" target_ip="$3" local ssh_key="$4" remote_ip="$5" label="${6:-remote}" log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/" return 0 fi local template template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ "grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null) if [[ -z "$template" ]]; then warn "$container template not found on $label — WebUI needs manual reconfiguration" return 1 fi timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \ "sed -i 's|.*|http://${target_ip}:${port}/|g' '$template'" \ 2>/dev/null && \ log "$container → http://${target_ip}:${port}/ ✅" || { error "Failed to reconfigure $container WebUI on $label" return 1 } } # Reconfigure local auth WebUIs to target IP reconfigure_local_webuis() { local target_ip="$1" log "Reconfiguring local auth WebUIs → ${target_ip}..." local failures=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue local container="${entry%%|*}" local port="${entry##*|}" local template template=$(grep -rl '' \ /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ xargs grep -l "\"$container\"" 2>/dev/null | head -1) if [[ -z "$template" ]]; then warn "$container template not found locally" (( failures++ )) continue fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/" continue fi sed -i "s|.*|http://${target_ip}:${port}/|g" \ "$template" 2>/dev/null && \ log "$container → http://${target_ip}:${port}/ ✅" || \ { error "Failed to reconfigure $container"; (( failures++ )); } done return $failures } get_tailscale_device_id() { local hostname="$1" curl -sf --max-time 10 \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \ 2>/dev/null | \ grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \ grep -o '"id":"[^"]*"' | \ grep -o '[^"]*"$' | tr -d '"' } remove_tailscale_device() { local hostname="$1" if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal" return 1 fi log "Looking up Tailscale device ID for $hostname..." local device_id device_id=$(get_tailscale_device_id "$hostname") if [[ -z "$device_id" ]]; then warn "$hostname not found in Tailscale — may already be removed" return 0 fi log "Removing $hostname (device $device_id) from Tailscale..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $hostname from Tailscale tailnet" return 0 fi curl -sf --max-time 10 -X DELETE \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \ warn "$hostname removed from Tailscale ✅" || \ error "Failed to remove $hostname from Tailscale — remove manually" } # ── Blocklist helpers ───────────────────────────────────────────────────────────────────────── is_blocklisted() { local hostname="$1" [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${hostname}|" "$BLOCKLIST_FILE" 2>/dev/null } add_to_blocklist() { local hostname="$1" reason="${2:-offboard}" local now now=$(date '+%Y-%m-%d %H:%M:%S') if ! is_blocklisted "$hostname"; then echo "${hostname}|${now}|${reason}" >> "$BLOCKLIST_FILE" log "Blocklisted: $hostname (reason: $reason)" else log "$hostname already on blocklist" fi } remove_from_blocklist() { local hostname="$1" if [[ ! -f "$BLOCKLIST_FILE" ]]; then log "Blocklist empty — nothing to remove" return 0 fi sed -i "/^${hostname}|/d" "$BLOCKLIST_FILE" 2>/dev/null log "Removed from blocklist: $hostname" } # Revoke SSH access on both sides — call after all other SSH operations complete. # Matches by key comment (format: keyname@hostname — set by ssh_setup.sh at keygen time). # SSH_REVOKE_REMOTE_OK / SSH_REVOKE_LOCAL_OK set in caller scope for summary display. do_ssh_key_revocation() { local remote_ip="$1" local pub_key_file="${SSH_KEY}.pub" echo "" echo "━━━ $ICON_SHIELD SSH Key Revocation ━━━" # Step 1: remove our pubkey from remote's authorized_keys while SSH still works SSH_REVOKE_REMOTE_OK=false if [[ -f "$pub_key_file" ]]; then local our_comment our_comment=$(awk '{print $3}' "$pub_key_file" 2>/dev/null) if [[ -n "$our_comment" ]]; then log "Revoking our pubkey ($our_comment) from $REMOTE_SERVER_NAME..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove our pubkey from $REMOTE_SERVER_NAME authorized_keys" SSH_REVOKE_REMOTE_OK=true elif [[ -n "$remote_ip" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "grep -v '${our_comment}' /root/.ssh/authorized_keys \ > /root/.ssh/authorized_keys.tmp 2>/dev/null \ && mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \ && echo removed" 2>/dev/null | grep -q removed; then log "Our pubkey revoked from $REMOTE_SERVER_NAME ✅" SSH_REVOKE_REMOTE_OK=true else warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:" warn " grep -v '@${LOCAL_SERVER_NAME}' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys" fi else warn "Could not read pubkey comment from $pub_key_file — skipping remote revocation" fi else warn "Pubkey not found at $pub_key_file — skipping remote revocation" fi # Step 2: remove remote's pubkey from our local authorized_keys # Remote key comment ends with @REMOTE_SERVER_NAME — unique, no special chars SSH_REVOKE_LOCAL_OK=false log "Revoking $REMOTE_SERVER_NAME pubkey from local authorized_keys..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $REMOTE_SERVER_NAME pubkey from local authorized_keys" SSH_REVOKE_LOCAL_OK=true elif [[ -f /root/.ssh/authorized_keys ]]; then if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \ > /root/.ssh/authorized_keys.tmp 2>/dev/null && \ mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then log "$REMOTE_SERVER_NAME pubkey revoked locally ✅" SSH_REVOKE_LOCAL_OK=true else warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually" fi else log "$REMOTE_SERVER_NAME pubkey not found in local authorized_keys — already removed" SSH_REVOKE_LOCAL_OK=true fi else log "/root/.ssh/authorized_keys not found — nothing to remove locally" SSH_REVOKE_LOCAL_OK=true fi } # ── FolderView3 integration ─────────────────────────────────────────────────────────────────── FOLDERVIEW3_DIR="/usr/local/emhttp/plugins/folder.view3" FOLDERVIEW3_JSON="/boot/config/plugins/folder.view3/docker.json" # Derive short partner name: strip unraid- prefix (case-insensitive) if present derive_partner_folder_name() { local hostname="$1" local short="${hostname,,}" [[ "$short" == unraid-* ]] && short="${short:7}" # Capitalise first char for readability: jayred365 → Jayred365-Failover echo "${short^}-Fallback" } folderview3_ensure_plugin() { if [[ -d "$FOLDERVIEW3_DIR" ]]; then log "FolderView3 plugin present ✅" return 0 fi if [[ -z "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then warn "FolderView3 plugin not installed and PARTNERSHIP_FOLDERVIEW3_URL is empty" warn "Install manually from Community Applications or set PARTNERSHIP_FOLDERVIEW3_URL in master.conf" return 1 fi warn "FolderView3 not found — installing from CA..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run: plugin install $PARTNERSHIP_FOLDERVIEW3_URL" return 0 fi plugin install "$PARTNERSHIP_FOLDERVIEW3_URL" 2>/dev/null && \ log "FolderView3 installed ✅" || { warn "FolderView3 install failed — folder will not be created" return 1 } } folderview3_create_partner_folder() { local partner_name="$1" shift local containers=("$@") log "FolderView3: creating folder '$partner_name' with ${#containers[@]} container(s)..." folderview3_ensure_plugin || return 1 if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would create FolderView3 folder: $partner_name" for c in "${containers[@]}"; do [[ -n "$c" ]] && warn " DRY RUN — container: $c" done return 0 fi # Initialise JSON if missing if [[ ! -f "$FOLDERVIEW3_JSON" ]]; then mkdir -p "$(dirname "$FOLDERVIEW3_JSON")" echo '{}' > "$FOLDERVIEW3_JSON" fi # Check folder doesn't already exist local existing existing=$(jq -r --arg name "$partner_name" \ 'to_entries[] | select(.value.name == $name) | .key' \ "$FOLDERVIEW3_JSON" 2>/dev/null) if [[ -n "$existing" ]]; then log "FolderView3: folder '$partner_name' already exists (id: $existing) — skipping" return 0 fi # Build containers JSON array local containers_json containers_json=$(printf '%s\n' "${containers[@]}" | \ grep -v '^$' | jq -R . | jq -s .) # Generate a stable random ID from timestamp+name local folder_id folder_id=$(echo "${partner_name}$(date +%s%N)" | md5sum | head -c 12) jq --arg id "$folder_id" --arg name "$partner_name" \ --argjson containers "$containers_json" \ '.[$id] = {"name": $name, "containers": $containers, "containerImages": {}}' \ "$FOLDERVIEW3_JSON" > "${FOLDERVIEW3_JSON}.tmp" && \ mv "${FOLDERVIEW3_JSON}.tmp" "$FOLDERVIEW3_JSON" && \ log "FolderView3: folder '$partner_name' created with ${#containers[@]} container(s) ✅" || { warn "FolderView3: failed to write JSON — check $FOLDERVIEW3_JSON" return 1 } } folderview3_remove_partner_folder() { local partner_name="$1" log "FolderView3: removing folder '$partner_name' and stopping its containers..." if [[ ! -f "$FOLDERVIEW3_JSON" ]]; then log "FolderView3: JSON not found — nothing to remove" return 0 fi if ! command -v jq >/dev/null 2>&1; then warn "jq not found — cannot manage FolderView3 JSON" return 1 fi # Get containers from this folder local containers_json containers_json=$(jq -r --arg name "$partner_name" \ 'to_entries[] | select(.value.name == $name) | .value.containers[]' \ "$FOLDERVIEW3_JSON" 2>/dev/null) if [[ -z "$containers_json" ]]; then log "FolderView3: folder '$partner_name' not found — nothing to remove" return 0 fi # Stop and remove each container in the folder local stopped=0 removed=0 failed=0 while IFS= read -r container; do [[ -z "$container" ]] && continue if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would stop + rm: $container" continue fi # Stop if running if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then if timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1; then log "$container stopped ✅" (( stopped++ )) else warn "$container stop failed" (( failed++ )) fi if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then log "$container removed ✅" (( removed++ )) else warn "$container rm failed (may not exist)" fi else log "$container not found locally — skipping" fi done <<< "$containers_json" # Remove folder entry from JSON if [[ "$DRY_RUN" == false ]]; then jq --arg name "$partner_name" \ 'with_entries(select(.value.name != $name))' \ "$FOLDERVIEW3_JSON" > "${FOLDERVIEW3_JSON}.tmp" && \ mv "${FOLDERVIEW3_JSON}.tmp" "$FOLDERVIEW3_JSON" && \ log "FolderView3: folder '$partner_name' removed ✅" || \ warn "FolderView3: failed to remove folder from JSON" else warn "DRY RUN — would remove folder '$partner_name' from $FOLDERVIEW3_JSON" fi [[ "$DRY_RUN" == false ]] && \ log "FolderView3 cleanup: $stopped stopped, $removed removed, $failed failed" return 0 } # Gather all partner failover containers for this server (all tiers) gather_partner_fallback_containers() { local out_var="$1" eval "${out_var}=()" local tier var for tier in TIER1 TIER2 TIER3 TIER4; do var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_${tier}" # Check if var is set and is an array if declare -p "$var" >/dev/null 2>&1; then local -a _tmp_arr eval "_tmp_arr=(\"\${${var}[@]}\")" for c in "${_tmp_arr[@]}"; do [[ -n "$c" ]] && eval "${out_var}+=(\"\$c\")" done fi done } check_both_healthy() { mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; } local mirror_ip mirror_ip=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null) [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \ "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || { error "Mirror $MIRROR not healthy" return 1 } return 0 } do_final_sync() { log "Running final Critical-Data and Emby sync..." if [[ "$DRY_RUN" == false ]]; then # Use CRITICAL_SYNC_SHARES from HOST* conf if available # Falls back to known paths — these are the critical ones bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ "/mnt/user/appdata-Fallback/Critical-Data" \ --profile=critical-fallback --log bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ "/mnt/user/Media_Server/Emby" \ --profile=emby-fallback --log else warn "DRY RUN — would run final Critical-Data and Emby sync" fi warn "Final sync complete — mirror has current state ✅" } # Safe master.conf modification with error handling update_master_conf() { local key="$1" value="$2" local conf="$SCRIPT_DIR/../master.conf" if [[ ! -f "$conf" ]]; then error "master.conf not found at $conf" return 1 fi if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then log "master.conf updated: ${key}=${value}" return 0 else error "Failed to update master.conf: ${key}=${value}" return 1 fi } # ============================================================================================== # ━━━ Unblock ━━━ # ============================================================================================== if [[ "$MODE" == "unblock" ]]; then if [[ -z "$UNBLOCK_HOST" ]]; then error "Usage: partnership_manager.sh --unblock " error "Example: partnership_manager.sh --unblock unRAID-Jayred365" if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then echo "" echo "Currently blocklisted:" while IFS='|' read -r host ts reason; do echo " $host (blocked $ts — $reason)" done < "$BLOCKLIST_FILE" else echo "Blocklist is empty." fi exit 1 fi if ! is_blocklisted "$UNBLOCK_HOST"; then warn "$UNBLOCK_HOST is not on the blocklist" exit 0 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $UNBLOCK_HOST from blocklist" exit 0 fi remove_from_blocklist "$UNBLOCK_HOST" warn "$UNBLOCK_HOST unblocked — re-onboarding is now permitted ✅" exit 0 fi # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$MODE" == "status" ]]; then echo "" echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━" OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null || echo "unreachable") MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null || echo "unreachable") echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)" echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)" echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " Enabled: ${PARTNERSHIP_ENABLED:-false}" echo "" # Local state if [[ -f "$LOCAL_STATE_FILE" ]]; then LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded") LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded") LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote") echo " Local state: $LOCAL_STATE" [[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED" [[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED" [[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN" else echo " Local state: no state file found" fi echo "" # Remote state REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) if [[ -n "$REMOTE_IP" ]]; then REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -n "$REMOTE_CONTENT" ]]; then REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2) echo " Remote state: $REMOTE_STATE" [[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED" if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then echo "" echo " ✅ Both servers agree: $LOCAL_STATE" else echo "" echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE" fi else echo " Remote state: not found on $REMOTE_SERVER_NAME" fi else echo " Remote state: $REMOTE_SERVER_NAME unreachable" fi echo "" echo " Auth WebUIs:" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue echo " ${entry%%|*} → port ${entry##*|}" done if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then echo "" FOLDER_STATUS_NAME=$(derive_partner_folder_name "$REMOTE_SERVER_NAME") if [[ -f "$FOLDERVIEW3_JSON" ]]; then FOLDER_EXISTS=$(jq -r --arg name "$FOLDER_STATUS_NAME" \ 'to_entries[] | select(.value.name == $name) | .key' \ "$FOLDERVIEW3_JSON" 2>/dev/null) if [[ -n "$FOLDER_EXISTS" ]]; then FOLDER_CONTAINERS=$(jq -r --arg name "$FOLDER_STATUS_NAME" \ '[to_entries[] | select(.value.name == $name) | .value.containers[]] | join(", ")' \ "$FOLDERVIEW3_JSON" 2>/dev/null) echo " FolderView3: $FOLDER_STATUS_NAME ✅" echo " Containers: $FOLDER_CONTAINERS" else echo " FolderView3: $FOLDER_STATUS_NAME — not found in JSON" fi else echo " FolderView3: config not found ($FOLDERVIEW3_JSON)" fi fi if [[ -f "$OFFLINE_COUNTER" ]]; then OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) [[ "$OFFLINE_DAYS" -gt 0 ]] && \ echo "" && \ echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}" fi echo "" if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then echo " Blocklist:" while IFS='|' read -r host ts reason; do echo " ⛔ $host (blocked $ts — $reason)" done < "$BLOCKLIST_FILE" else echo " Blocklist: empty" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Check ━━━ # Called every 15min by critical_sync_maintenance.sh — must be silent when healthy # ============================================================================================== if [[ "$MODE" == "check" ]]; then # Update last_seen_remote and offline counter based on rsync outcome if [[ "$REMOTE_SEEN" == true ]]; then echo "0" > "$OFFLINE_COUNTER" if [[ -f "$LOCAL_STATE_FILE" ]]; then sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \ "$LOCAL_STATE_FILE" 2>/dev/null fi elif [[ "$REMOTE_UNSEEN" == true ]]; then OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 )) echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER" # Auto-offboard threshold: threshold_days × 96 intervals/day (every 15min) THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 )) if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard" notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \ "Partnership" "warning" bash "$0" --offboard --reason=auto-offboard-timeout exit 0 fi log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)" fi # Skip blocklisted partners — they can't auto-reconnect; only --onboard re-establishes if is_blocklisted "$REMOTE_SERVER_NAME"; then log "Partnership check — $REMOTE_SERVER_NAME is blocklisted, skipping" exit 0 fi # Read remote state file REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) if [[ -z "$REMOTE_IP" ]]; then log "Partnership check — remote unreachable, skipping state check" exit 0 fi REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -z "$REMOTE_CONTENT" ]]; then # IP resolved but SSH returned nothing — could be auth failure, not just missing file if [[ -f "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" ]]; then bash "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" --validate 2>/dev/null || true fi log "Partnership check — remote state file not found" exit 0 fi REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN") # Both agree and active — healthy, silent if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then log "Partnership check — ACTIVE, both servers agree ✅" exit 0 fi # Remote requested offboard if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then warn "Partnership check — $REMOTE_SERVER_NAME requested offboard" if [[ "$AM_OWNER" == true ]]; then warn "Owner finalising offboard request from mirror..." do_final_sync NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested" if [[ "$DRY_RUN" == false ]]; then update_master_conf "CRITICAL_RSYNC_ENABLED" "false" fi # SSH key revocation — mutual, both directions # REMOTE_IP already resolved above — this is the last SSH operation SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false do_ssh_key_revocation "$REMOTE_IP" if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace period..." # Grace sleep — interruptible via SIGTERM trap if [[ "$DRY_RUN" == false ]]; then trap 'warn "Partnership check interrupted during grace sleep"; exit 0' SIGTERM SIGINT sleep "$grace_seconds" trap - SIGTERM SIGINT fi remove_tailscale_device "$MIRROR" fi notify "Partnership offboard finalised — $MIRROR requested separation" \ "Partnership" "normal" else # Mirror sees owner is INACTIVE — clean up own side warn "Owner has offboarded — cleaning up mirror side..." reconfigure_local_webuis "localhost" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded" notify "Partnership ended — $OWNER offboarded. Auth WebUIs → localhost." \ "Partnership" "normal" fi exit 0 fi # Both inactive — nothing to do if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then log "Partnership check — INACTIVE on both servers" exit 0 fi log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE" exit 0 fi # ============================================================================================== # ━━━ Onboard ━━━ # ============================================================================================== if [[ "$MODE" == "onboard" ]]; then echo "" echo "━━━ $ICON_FALLBACK Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # Check already onboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then warn "Partnership already ACTIVE — use --status for detail or --offboard to separate" exit 0 fi fi # Warn if re-onboarding a previously blocked partner — onboard is deliberate so it proceeds if is_blocklisted "$MIRROR"; then warn "$MIRROR is on the blocklist from a previous offboard" warn "Proceeding — blocklist will be cleared on successful onboard" fi log "Owner: $OWNER_ID ($OWNER)" log "Mirror: $MIRROR_ID ($MIRROR)" # Pre-flight echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" resolve_remote_ip check_connectivity # Version parity — both servers must agree on unRAID version check_unraid_version_parity || exit 1 # Remote array and Docker daemon check_remote_array || exit 1 check_remote_docker_daemon || exit 1 OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null) MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null) [[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; } [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; } log "Owner IP: $OWNER_IP" log "Mirror IP: $MIRROR_IP" # Reconfigure mirror WebUIs → owner IP echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$OWNER_IP" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done # Verify WebUI connectivity if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then echo "" echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then log "$container reachable at http://${OWNER_IP}:${port}/ ✅" else warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running" fi done fi # Write state files echo "" echo "━━━ $ICON_GEAR Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" log "Local state: ACTIVE ✅" [[ "$DRY_RUN" == false ]] && remove_from_blocklist "$MIRROR" push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" echo "0" > "$OFFLINE_COUNTER" # FolderView3 — create partner folder with this server's failover containers for remote if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then echo "" echo "━━━ $ICON_CONTAINERS FolderView3 Integration ━━━" PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") declare -a PARTNER_CONTAINERS=() gather_partner_fallback_containers PARTNER_CONTAINERS if [[ ${#PARTNER_CONTAINERS[@]} -gt 0 ]]; then folderview3_create_partner_folder "$PARTNER_FOLDER_NAME" "${PARTNER_CONTAINERS[@]}" else log "No FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER* containers configured — skipping folder creation" fi fi # Summary echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" echo " Owner: $OWNER_ID ($OWNER_IP)" echo " Mirror: $MIRROR_ID ($MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅" [[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \ notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \ "Partnership" "normal" else warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually" notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \ "Partnership" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Offboard ━━━ # ============================================================================================== if [[ "$MODE" == "offboard" ]]; then echo "" echo "━━━ $ICON_FALLBACK Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # Check already offboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then warn "Partnership already INACTIVE — use --status to verify both servers agree" exit 0 fi fi # ── Mirror-initiated offboard ───────────────────────────────────────────────────────────── if [[ "$AM_MIRROR" == true ]]; then warn "$MIRROR_ID ($MIRROR) is initiating offboard" warn "Local auth WebUIs will be reconfigured → localhost" warn "$OWNER_ID ($OWNER) will finalise on its next --check cycle" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 fi echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━" reconfigure_local_webuis "localhost" # FolderView3 — remove partner folder and clean containers if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then echo "" echo "━━━ $ICON_CONTAINERS FolderView3 Cleanup ━━━" PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER") folderview3_remove_partner_folder "$PARTNER_FOLDER_NAME" fi NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" log "Local state: INACTIVE ✅" [[ "$DRY_RUN" == false ]] && add_to_blocklist "$OWNER" "$REASON" OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null) if [[ -n "$OWNER_IP" ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY" notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \ "Partnership" "normal" else warn "$OWNER unreachable — state written locally, owner will see it when reachable" fi echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" echo " Your WebUIs: reconfigured → localhost ✅" echo " State: INACTIVE ✅" echo " Blocklist: $OWNER blocked — re-onboard to permit access again ✅" echo " Owner: will finalise + final sync on next --check ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ── Owner-initiated offboard ────────────────────────────────────────────────────────────── warn "Offboarding $MIRROR_ID ($MIRROR) from partnership" warn "Final sync will run — mirror leaves with current state" if [[ "$DRY_RUN" == false ]]; then echo "" echo "You have 10 seconds to cancel (Ctrl+C)..." sleep 10 echo "Proceeding..." fi resolve_remote_ip # Stop any running rsync first echo "" echo "━━━ $ICON_STOP Stop Running Rsync ━━━" bash "$SCRIPT_DIR/../Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true # Final sync echo "" echo "━━━ $ICON_SYNC Final Sync ━━━" do_final_sync MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null) MIRROR_REACHABLE=false [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true # Reconfigure mirror WebUIs → localhost echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━" WEBUI_FAILURES=0 if [[ "$MIRROR_REACHABLE" == true ]]; then for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "localhost" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done else warn "$MIRROR unreachable — WebUI reconfiguration skipped" warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check" (( WEBUI_FAILURES++ )) fi # Disable critical rsync echo "" echo "━━━ $ICON_GEAR Disable Critical Sync ━━━" if [[ "$DRY_RUN" == false ]]; then update_master_conf "CRITICAL_RSYNC_ENABLED" "false" warn "CRITICAL_RSYNC_ENABLED=false ✅" else warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" fi # Write state files echo "" echo "━━━ $ICON_GEAR Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" \ "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" log "Local state: INACTIVE ✅" [[ "$DRY_RUN" == false ]] && add_to_blocklist "$MIRROR" "$REASON" if [[ "$MIRROR_REACHABLE" == true ]]; then push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" fi # FolderView3 — remove partner folder and clean containers on this (owner) side if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then echo "" echo "━━━ $ICON_CONTAINERS FolderView3 Cleanup ━━━" PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") folderview3_remove_partner_folder "$PARTNER_FOLDER_NAME" fi # SSH key revocation — mutual, both directions # Must run before Tailscale removal (SSH needs network) and after state is pushed SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false do_ssh_key_revocation "${MIRROR_IP:-}" # Tailscale removal if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then echo "" echo "━━━ $ICON_NET Tailscale Separation ━━━" if [[ "$MIRROR_REACHABLE" == true ]]; then grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..." if [[ "$DRY_RUN" == false ]]; then trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT sleep "$grace_seconds" trap - SIGTERM SIGINT fi fi remove_tailscale_device "$MIRROR" fi # Backup notification if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_DISK Backup Handover ━━━" log "Backups available for $MIRROR:" for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do [[ -z "$path" ]] && continue echo " $path" done notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \ "Partnership" "warning" fi # Summary echo "" echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" echo " Owner: $OWNER_ID ($OWNER)" echo " Mirror: $MIRROR_ID ($MIRROR)" echo " Final sync: complete ✅" echo " WebUI failures: $WEBUI_FAILURES" echo " Critical rsync: disabled ✅" echo " State: INACTIVE ✅" echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" _revoke_status() { if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true ]] && [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "both directions ✅" elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then echo "local only ✅ — remote failed (revoke manually on $MIRROR)" else echo "⚠️ failed — check warnings above" fi } echo " Keys revoked: $(_revoke_status)" [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \ echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅" [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ echo " Tailscale: $MIRROR removed ✅" echo "" echo " $MIRROR leaves with:" echo " ✓ Current auth config (final sync)" echo " ✓ Auth WebUIs → localhost" echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups" echo " ✓ Full ecosystem — just stop the sync" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ warn "$ICON_DONE DONE — clean separation complete ✅" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Transfer ━━━ # ============================================================================================== if [[ "$MODE" == "transfer" ]]; then echo "" echo "━━━ $ICON_FALLBACK Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — OWNERSHIP TRANSFER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " Current owner: $OWNER_ID ($OWNER)" echo " Current mirror: $MIRROR_ID ($MIRROR)" echo "" echo " After transfer:" echo " New owner: $MIRROR_ID ($MIRROR)" echo " New mirror: $OWNER_ID ($OWNER)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Confirmation check if [[ "$DRY_RUN" == false ]]; then if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then echo "" echo "To proceed pass exactly:" echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}" echo "" error "Transfer cancelled — confirmation required" exit 1 fi if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then error "Confirmation string does not match — transfer cancelled" exit 1 fi log "Confirmation accepted" else warn "DRY RUN — confirmation check skipped" fi # Health strike system echo "" echo "━━━ $ICON_SHIELD Health Verification ━━━" log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" STRIKES=0 ATTEMPTS=0 MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}" while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do (( ATTEMPTS++ )) if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable" error "Transfer cancelled — try again when both servers are healthy" exit 1 fi if check_both_healthy; then (( STRIKES++ )) log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10 else warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)" STRIKES=0 sleep 30 fi done warn "Both servers healthy — proceeding with transfer" NEW_OWNER_ID="$MIRROR_ID" NEW_MIRROR_ID="$OWNER_ID" NEW_OWNER="$MIRROR" NEW_MIRROR="$OWNER" NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY" NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY" NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}" NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}" NEW_OWNER_IP=$(tailscale ip -4 "${NEW_OWNER,,}" 2>/dev/null) NEW_MIRROR_IP=$(tailscale ip -4 "${NEW_MIRROR,,}" 2>/dev/null) [[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; } [[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; } # Final sync in current direction echo "" echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━" do_final_sync # Reconfigure new mirror WebUIs → new owner echo "" echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━" log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \ "$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ )) done # New owner WebUIs → localhost (now manages directly) log "New owner ($NEW_OWNER) WebUIs → localhost" reconfigure_local_webuis "localhost" # Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers echo "" echo "━━━ $ICON_GEAR Update Ownership ━━━" if [[ "$DRY_RUN" == false ]]; then update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\"" # Update remote master.conf timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \ "sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \ '$SCRIPT_DIR/../master.conf'" 2>/dev/null && \ log "Remote master.conf updated ✅" || \ error "Failed to update remote master.conf — update manually" else warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers" fi # Write state files NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer" push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY" # Summary echo "" echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━" echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)" echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync direction: $NEW_OWNER → $NEW_MIRROR" echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅" notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \ "Partnership" "normal" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi