Gmer4Lfe
28d434b76c
All FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER* references updated to
FALLBACK_${REMOTE_ID}_TIER* across fallback_test.sh, partnership_manager.sh,
docker_update.sh, mesh_monitor.sh, and monitor.php. mesh_monitor.sh drops
the inner covering-host loop — tier data now lives in the covered host's own
conf so no cross-host scan is needed. monitor.php reads from the covered
host's conf file rather than the local host's.
1470 lines
59 KiB
Bash
Executable File
1470 lines
59 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= Partnership Manager ============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Manages the full lifecycle of a two-server partnership — onboard, daily health
|
||
# monitoring, offboard, and ownership transfer. Called by partnership_onboard.sh
|
||
# during initial setup, and by critical_sync_maintenance.sh every 30 minutes for
|
||
# the --check mode. All other modes are run manually.
|
||
#
|
||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer.
|
||
# AM_OWNER / AM_MIRROR flags (set by detect_hosts) control all routing —
|
||
# no hostname string comparisons anywhere in this script.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# --onboard (owner only)
|
||
# Reconfigures mirror auth WebUIs → owner's Tailscale IP
|
||
# Mirror operator clicks NPM → gets owner's NPM via Tailscale automatically
|
||
# Writes ACTIVE state on both servers
|
||
#
|
||
# --offboard (either server)
|
||
# Mirror-initiated: reconfigures own WebUIs → localhost, writes INACTIVE state
|
||
# Owner finalises on next --check: final sync, Tailscale removal
|
||
# Owner-initiated: final sync, reconfigures mirror WebUIs → localhost, removes
|
||
# mirror containers + appdata, SSH key revocation, Tailscale removal
|
||
# Both leave with clean state ✅
|
||
#
|
||
# --transfer (owner only)
|
||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf
|
||
# Requires confirmation string + consecutive health check passes
|
||
#
|
||
# --check (called every 30min by critical_sync_maintenance.sh)
|
||
# --remote-seen: rsync succeeded → reset offline counter, read remote state
|
||
# --remote-unseen: rsync failed → increment offline counter → auto-offboard at threshold
|
||
# Silent when healthy — never noisy on clean runs
|
||
#
|
||
# --status (either server)
|
||
# Show state files from both servers, blocklist, SSH key status
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Deferred offboard
|
||
# Either server can offboard without the other being reachable. The initiating
|
||
# server writes its state immediately and becomes independent. The other server
|
||
# reads the INACTIVE state on its next --check and finalises automatically.
|
||
# No message passing, no coordination required.
|
||
#
|
||
# Appdata cleanup on offboard
|
||
# Partner containers are stopped and removed. Their appdata bind-mount paths
|
||
# (collected via docker inspect before removal) are also deleted. Safety gate:
|
||
# only paths matching /mnt/*/appdata* are deleted — media and config shares
|
||
# outside the appdata tree are never touched.
|
||
#
|
||
# Blocklist is application-layer; Tailscale removal is network-layer
|
||
# Both happen on offboard. The blocklist prevents re-onboard until explicitly
|
||
# cleared with --unblock. Tailscale removal ends encrypted access at the network
|
||
# level. Grace period controls both simultaneously — one var, consistent behaviour.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root check
|
||
# All operations require root.
|
||
#
|
||
# Role enforcement
|
||
# Mirror cannot run --onboard or --transfer. Blocked with a clear error message.
|
||
#
|
||
# Version parity check
|
||
# --onboard verifies both servers are on compatible unRAID versions.
|
||
#
|
||
# SSH_TIMEOUT on all remote calls
|
||
# Every ssh/scp call is timeout-protected. No operation hangs on an unreachable peer.
|
||
#
|
||
# flock on state writes
|
||
# Prevents concurrent state file corruption from overlapping --check cycles.
|
||
#
|
||
# SIGTERM trap on grace period sleep
|
||
# Offboard grace period is interruptible — Ctrl-C aborts cleanly.
|
||
#
|
||
# Silent by default
|
||
# --check produces no output when both servers are healthy. Only state changes
|
||
# and threshold crossings produce output.
|
||
#
|
||
# ==============================================================================================
|
||
# STATE FILES
|
||
# ==============================================================================================
|
||
#
|
||
# /boot/config/partnership_HOST1.db — HOST1 writes, HOST2 reads via SSH
|
||
# /boot/config/partnership_HOST2.db — HOST2 writes, HOST1 reads via SSH
|
||
# /boot/config/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared
|
||
#
|
||
# On /boot/config — survives reboots, available before array starts, minimal flash wear.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# PARTNERSHIP_ENABLED
|
||
# Global gate — set true once both servers are configured (default: false)
|
||
#
|
||
# PARTNERSHIP_OWNER_HOST
|
||
# "HOST1" or "HOST2" — flips on --transfer (default: "HOST1")
|
||
#
|
||
# PARTNERSHIP_REMOVE_TAILSCALE
|
||
# Remove mirror from tailnet on offboard (default: true)
|
||
#
|
||
# PARTNERSHIP_GRACE_HOURS
|
||
# Hours before Tailscale removal after offboard — backup access also expires then (default: 6)
|
||
#
|
||
# PARTNERSHIP_OFFLINE_THRESHOLD
|
||
# Days of missed sync cycles before auto-offboard triggers (default: 30)
|
||
#
|
||
# PARTNERSHIP_TRANSFER_CONFIRM
|
||
# Exact string required for --transfer (default: "i-understand-this-transfers-ownership")
|
||
#
|
||
# PARTNERSHIP_TRANSFER_STRIKES
|
||
# Consecutive health checks required before transfer proceeds (default: 3)
|
||
#
|
||
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS
|
||
# Max health check attempts before giving up (default: 20)
|
||
#
|
||
# PARTNERSHIP_ONBOARD_VERIFY
|
||
# Curl-verify each WebUI after reconfigure to confirm Tailscale routing works (default: true)
|
||
#
|
||
# PARTNERSHIP_ONBOARD_NOTIFY
|
||
# Notify both servers on successful onboard (default: true)
|
||
#
|
||
# PARTNERSHIP_SYNC_INTERVAL
|
||
# Informational — actual schedule is in cron (default: 15)
|
||
#
|
||
# TAILSCALE_API_KEY / TAILSCALE_TAILNET
|
||
# Required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_PARTNERSHIP_AUTH_WEBUIS
|
||
# Containers reconfigured on onboard/offboard. Format: "ContainerName|WebUIPort"
|
||
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_WEBUIS
|
||
#
|
||
# HOST*_PARTNERSHIP_MIRROR_BACKUPS
|
||
# Paths accessible to the partner during grace window after offboard.
|
||
# Aliased by detect_hosts() → PARTNERSHIP_MIRROR_BACKUPS
|
||
#
|
||
# HOST*_PARTNERSHIP_OWN_CONTAINERS
|
||
# Containers parked here during partnership, restarted on offboard.
|
||
# Aliased by detect_hosts() → PARTNERSHIP_OWN_CONTAINERS
|
||
#
|
||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||
# Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) deployed on mirror during onboard.
|
||
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# partnership_manager.sh --onboard
|
||
# Establish mirror relationship — owner only
|
||
#
|
||
# partnership_manager.sh --offboard
|
||
# Clean separation — either server. 10-second countdown before executing.
|
||
#
|
||
# partnership_manager.sh --offboard --dry-run
|
||
# Show the complete offboard sequence without executing
|
||
#
|
||
# partnership_manager.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||
# Flip ownership — owner only. Requires exact confirmation string.
|
||
#
|
||
# partnership_manager.sh --check --remote-seen|--remote-unseen
|
||
# Called by critical_sync_maintenance.sh every 30min — do not run manually
|
||
#
|
||
# partnership_manager.sh --status
|
||
# Show state files, blocklist, SSH key status from both servers
|
||
#
|
||
# partnership_manager.sh --unblock <hostname>
|
||
# Remove hostname from blocklist to permit re-onboarding
|
||
#
|
||
# Any mode supports --dry-run and --log
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
||
|
||
SSH_TIMEOUT=15
|
||
BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}"
|
||
|
||
# ── Parse mode flags before parse_args ────────────────────────────────────────────────────────
|
||
MODE=""
|
||
TRANSFER_CONFIRM_INPUT=""
|
||
REMOTE_SEEN=false
|
||
REMOTE_UNSEEN=false
|
||
REASON="manual"
|
||
UNBLOCK_HOST=""
|
||
FILTERED_ARGS=()
|
||
|
||
LOCAL_ONLY=false # --onboard --local-only: skips remote steps, runs HOST1-local setup only
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--onboard) MODE="onboard" ;;
|
||
--offboard) MODE="offboard" ;;
|
||
--transfer) MODE="transfer" ;;
|
||
--check) MODE="check" ;;
|
||
--status) MODE="status" ;;
|
||
--unblock) MODE="unblock" ;;
|
||
--local-only) LOCAL_ONLY=true ;;
|
||
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
||
--remote-seen) REMOTE_SEEN=true ;;
|
||
--remote-unseen) REMOTE_UNSEEN=true ;;
|
||
--reason=*) REASON="${arg#--reason=}" ;;
|
||
*)
|
||
if [[ "$MODE" == "unblock" ]] && [[ -z "$UNBLOCK_HOST" ]] && [[ "$arg" != --* ]]; then
|
||
UNBLOCK_HOST="$arg"
|
||
else
|
||
FILTERED_ARGS+=("$arg")
|
||
fi
|
||
;;
|
||
esac
|
||
done
|
||
|
||
parse_args "${FILTERED_ARGS[@]}"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
# detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS
|
||
detect_hosts
|
||
|
||
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
|
||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1"
|
||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||
|
||
OWNER="${!OWNER_ID}" # hostname string
|
||
MIRROR="${!MIRROR_ID}"
|
||
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
||
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
||
# With sparse checkout, each server only has its own host{N}.conf — the other server's
|
||
# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode.
|
||
MIRROR_SSH_KEY="$SSH_KEY"
|
||
OWNER_SSH_KEY="$SSH_KEY"
|
||
|
||
AM_OWNER=false
|
||
AM_MIRROR=false
|
||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||
|
||
# State files
|
||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||
OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db"
|
||
MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db"
|
||
OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db"
|
||
|
||
# ── Exit Trap — restart locally stopped containers if script crashes mid-cleanup ──────────────
|
||
# Used by cleanup_partner_containers() — also shared with partnership_offboard.sh which
|
||
# sources this file and registers the same trap.
|
||
declare -a _PM_TRAP_STOPPED=()
|
||
_pm_trap_restart_stopped() {
|
||
[[ ${#_PM_TRAP_STOPPED[@]} -eq 0 ]] && return
|
||
for c in "${_PM_TRAP_STOPPED[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
if docker inspect "$c" >/dev/null 2>&1; then
|
||
warn "Exit trap: restarting $c (stopped but not removed)"
|
||
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
||
fi
|
||
done
|
||
}
|
||
|
||
if [[ "${PARTNERSHIP_LIB_MODE:-}" != "1" ]]; then
|
||
if [[ -z "$MODE" ]]; then
|
||
error "No mode specified"
|
||
echo "Usage:"
|
||
echo " partnership_manager.sh --onboard"
|
||
echo " partnership_manager.sh --offboard"
|
||
echo " partnership_manager.sh --transfer --confirm=..."
|
||
echo " partnership_manager.sh --check --remote-seen|--remote-unseen"
|
||
echo " partnership_manager.sh --status"
|
||
echo " partnership_manager.sh --unblock <hostname>"
|
||
exit 1
|
||
fi
|
||
|
||
# Role-based access control
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
case "$MODE" in
|
||
onboard|transfer)
|
||
error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE"
|
||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||
exit 1
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
# Lock for all modes except --check (frequent) and --offboard (offboard script holds its own)
|
||
[[ "$MODE" != "check" && "$MODE" != "offboard" ]] && acquire_lock "strict"
|
||
|
||
trap _pm_trap_restart_stopped EXIT
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
read_state_file() {
|
||
local file="$1" key="$2"
|
||
grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2
|
||
}
|
||
|
||
write_state_file() {
|
||
local file="$1"
|
||
shift
|
||
# flock prevents concurrent writes to the same state file
|
||
(
|
||
flock -x 200
|
||
cat > "$file" << EOF
|
||
state=${1:-UNKNOWN}
|
||
owner=${OWNER}
|
||
mirror=${MIRROR}
|
||
onboarded=${2:-}
|
||
offboarded=${3:-}
|
||
triggered_by=${4:-}
|
||
reason=${5:-}
|
||
last_seen_remote=${6:-}
|
||
updated=$(date '+%Y-%m-%d %H:%M:%S')
|
||
EOF
|
||
) 200>"${file}.lock"
|
||
}
|
||
|
||
push_state_to_remote() {
|
||
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would push state file to remote"
|
||
return 0
|
||
fi
|
||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
||
echo "State file pushed to remote ✅" || \
|
||
warn "Could not push state file to remote — will propagate on next sync"
|
||
}
|
||
|
||
read_remote_state() {
|
||
local remote_ip="$1" ssh_key="$2" remote_file="$3"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
||
}
|
||
|
||
|
||
get_tailscale_device_id() {
|
||
local hostname="$1"
|
||
curl -sf --max-time 10 \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \
|
||
2>/dev/null | \
|
||
grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \
|
||
grep -o '"id":"[^"]*"' | \
|
||
grep -o '[^"]*"$' | tr -d '"'
|
||
}
|
||
|
||
remove_tailscale_device() {
|
||
local hostname="$1"
|
||
|
||
if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||
warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal"
|
||
return 1
|
||
fi
|
||
|
||
log "Looking up Tailscale device ID for $hostname..."
|
||
local device_id
|
||
device_id=$(get_tailscale_device_id "$hostname")
|
||
|
||
if [[ -z "$device_id" ]]; then
|
||
warn "$hostname not found in Tailscale — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
log "Removing $hostname (device $device_id) from Tailscale..."
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $hostname from Tailscale tailnet"
|
||
return 0
|
||
fi
|
||
|
||
curl -sf --max-time 10 -X DELETE \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \
|
||
warn "$hostname removed from Tailscale ✅" || \
|
||
error "Failed to remove $hostname from Tailscale — remove manually"
|
||
}
|
||
|
||
# ── Blocklist helpers ─────────────────────────────────────────────────────────────────────────
|
||
|
||
is_blocklisted() {
|
||
local hostname="$1"
|
||
[[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${hostname}|" "$BLOCKLIST_FILE" 2>/dev/null
|
||
}
|
||
|
||
add_to_blocklist() {
|
||
local hostname="$1" reason="${2:-offboard}"
|
||
local now
|
||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||
if ! is_blocklisted "$hostname"; then
|
||
echo "${hostname}|${now}|${reason}" >> "$BLOCKLIST_FILE"
|
||
log "Blocklisted: $hostname (reason: $reason)"
|
||
else
|
||
log "$hostname already on blocklist"
|
||
fi
|
||
}
|
||
|
||
remove_from_blocklist() {
|
||
local hostname="$1"
|
||
if [[ ! -f "$BLOCKLIST_FILE" ]]; then
|
||
log "Blocklist empty — nothing to remove"
|
||
return 0
|
||
fi
|
||
sed -i "/^${hostname}|/d" "$BLOCKLIST_FILE" 2>/dev/null
|
||
log "Removed from blocklist: $hostname"
|
||
}
|
||
|
||
# Revoke SSH access on both sides — call after all other SSH operations complete.
|
||
# Matches by key comment (format: keyname@hostname — set by ssh_setup.sh at keygen time).
|
||
# SSH_REVOKE_REMOTE_OK / SSH_REVOKE_LOCAL_OK set in caller scope for summary display.
|
||
do_ssh_key_revocation() {
|
||
local remote_ip="$1"
|
||
local pub_key_file="${SSH_KEY}.pub"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD SSH Key Revocation ━━━"
|
||
|
||
# Step 1: remove our pubkey from remote's authorized_keys while SSH still works
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
if [[ -f "$pub_key_file" ]]; then
|
||
local our_comment
|
||
our_comment=$(awk '{print $3}' "$pub_key_file" 2>/dev/null)
|
||
if [[ -n "$our_comment" ]]; then
|
||
log "Revoking our pubkey ($our_comment) from $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove our pubkey from $REMOTE_SERVER_NAME authorized_keys"
|
||
SSH_REVOKE_REMOTE_OK=true
|
||
elif [[ -n "$remote_ip" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||
"grep -v '${our_comment}' /root/.ssh/authorized_keys \
|
||
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
|
||
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
|
||
&& echo removed" 2>/dev/null | grep -q removed; then
|
||
echo "Our pubkey revoked from $REMOTE_SERVER_NAME ✅"
|
||
SSH_REVOKE_REMOTE_OK=true
|
||
else
|
||
warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:"
|
||
warn " grep -v '@${LOCAL_SERVER_NAME}' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys"
|
||
fi
|
||
else
|
||
warn "Could not read pubkey comment from $pub_key_file — skipping remote revocation"
|
||
fi
|
||
else
|
||
warn "Pubkey not found at $pub_key_file — skipping remote revocation"
|
||
fi
|
||
|
||
# Step 2: remove remote's pubkey from our local authorized_keys
|
||
# Remote key comment ends with @REMOTE_SERVER_NAME — unique, no special chars
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
log "Revoking $REMOTE_SERVER_NAME pubkey from local authorized_keys..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $REMOTE_SERVER_NAME pubkey from local authorized_keys"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
elif [[ -f /root/.ssh/authorized_keys ]]; then
|
||
if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then
|
||
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
|
||
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
|
||
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then
|
||
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
else
|
||
warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually"
|
||
fi
|
||
else
|
||
log "$REMOTE_SERVER_NAME pubkey not found in local authorized_keys — already removed"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
fi
|
||
else
|
||
log "/root/.ssh/authorized_keys not found — nothing to remove locally"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
fi
|
||
}
|
||
|
||
# Gather all partner fallback containers for this server (all tiers)
|
||
gather_partner_fallback_containers() {
|
||
local out_var="$1"
|
||
eval "${out_var}=()"
|
||
local tier var
|
||
for tier in TIER1 TIER2 TIER3 TIER4; do
|
||
var="FALLBACK_${REMOTE_ID}_${tier}"
|
||
# Check if var is set and is an array
|
||
if declare -p "$var" >/dev/null 2>&1; then
|
||
local -a _tmp_arr
|
||
eval "_tmp_arr=(\"\${${var}[@]}\")"
|
||
for c in "${_tmp_arr[@]}"; do
|
||
[[ -n "$c" ]] && eval "${out_var}+=(\"\$c\")"
|
||
done
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Read a scalar var from the mirror's own config via SSH.
|
||
# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works.
|
||
read_remote_conf_var() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
|
||
}
|
||
|
||
# Read an array var from the mirror's own config via SSH — one element per line.
|
||
read_remote_conf_array() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
|
||
}
|
||
|
||
# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe"
|
||
derive_short_name() {
|
||
local hostname="$1"
|
||
local short="${hostname,,}"
|
||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||
echo "${short^}"
|
||
}
|
||
|
||
# Start this server's own parked containers after partnership ends.
|
||
start_own_stack() {
|
||
echo ""
|
||
echo "━━━ $ICON_START Restart Own Stack ━━━"
|
||
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No PARTNERSHIP_OWN_CONTAINERS configured — skipping"
|
||
return 0
|
||
fi
|
||
for container in "${PARTNERSHIP_OWN_CONTAINERS[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start: $container"
|
||
continue
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then
|
||
echo "$container started ✅"
|
||
else
|
||
warn "$container failed to start — check manually"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Remove partnership containers on this server + their appdata bind-mount paths.
|
||
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
|
||
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
|
||
cleanup_partner_containers() {
|
||
declare -a containers=()
|
||
gather_partner_fallback_containers containers
|
||
|
||
if [[ ${#containers[@]} -eq 0 ]]; then
|
||
log "No partner containers found to remove"
|
||
return 0
|
||
fi
|
||
|
||
# Collect appdata paths BEFORE any removal — inspect fails once container is gone
|
||
local all_appdata_paths=""
|
||
for container in "${containers[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
local cpaths
|
||
cpaths=$(docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
|
||
"$container" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
||
[[ -n "$cpaths" ]] && all_appdata_paths+=$'\n'"$cpaths"
|
||
fi
|
||
done
|
||
|
||
# Remove containers
|
||
for container in "${containers[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop + rm: $container"
|
||
continue
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
|
||
_PM_TRAP_STOPPED+=("$container")
|
||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
|
||
echo "$container removed ✅" || warn "$container rm failed"
|
||
else
|
||
log "$container not found — skipping"
|
||
fi
|
||
done
|
||
|
||
# Delete appdata after containers are gone
|
||
while IFS= read -r path; do
|
||
[[ -z "$path" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn " DRY RUN — would rm -rf $path"
|
||
continue
|
||
fi
|
||
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
||
done <<< "$all_appdata_paths"
|
||
}
|
||
|
||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
|
||
# and their appdata bind-mount paths.
|
||
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
|
||
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
|
||
cleanup_owner_containers_on_mirror() {
|
||
local mirror_ip="$1"
|
||
local owner_short
|
||
owner_short=$(derive_short_name "$OWNER")
|
||
|
||
log "Removing owner-deployed containers from $MIRROR..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove *-${owner_short} containers + appdata from $MIRROR"
|
||
return 0
|
||
fi
|
||
|
||
local container_list
|
||
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
|
||
|
||
if [[ -z "$container_list" ]]; then
|
||
log "No *-${owner_short} containers found on $MIRROR"
|
||
return 0
|
||
fi
|
||
|
||
while IFS= read -r container; do
|
||
[[ -z "$container" ]] && continue
|
||
|
||
# Collect appdata paths before removal
|
||
local appdata_paths
|
||
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker stop '$container' >/dev/null 2>&1
|
||
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||
grep -q removed && \
|
||
echo "$container removed from $MIRROR ✅" || \
|
||
warn "Failed to remove $container from $MIRROR"
|
||
|
||
# Delete appdata on remote after container removal
|
||
while IFS= read -r path; do
|
||
[[ -z "$path" ]] && continue
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
||
warn " Failed to remove appdata on $MIRROR: $path"
|
||
done <<< "$appdata_paths"
|
||
done <<< "$container_list"
|
||
}
|
||
|
||
# SSH to mirror — start mirror's own parked containers.
|
||
# Reads PARTNERSHIP_OWN_CONTAINERS from the mirror's own conf via SSH.
|
||
start_mirror_own_stack() {
|
||
local mirror_ip="$1"
|
||
|
||
log "Reading own stack list from $MIRROR conf..."
|
||
local -a mirror_own=()
|
||
mapfile -t mirror_own < <(read_remote_conf_array "$mirror_ip" "PARTNERSHIP_OWN_CONTAINERS")
|
||
# Remove empty entries
|
||
local -a filtered=()
|
||
for c in "${mirror_own[@]}"; do [[ -n "$c" ]] && filtered+=("$c"); done
|
||
mirror_own=("${filtered[@]}")
|
||
|
||
if [[ ${#mirror_own[@]} -eq 0 ]]; then
|
||
log "No PARTNERSHIP_OWN_CONTAINERS configured on $MIRROR — skipping remote stack restart"
|
||
return 0
|
||
fi
|
||
|
||
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
|
||
for container in "${mirror_own[@]}"; do
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container on $MIRROR"
|
||
continue
|
||
fi
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
||
grep -q started && \
|
||
echo "$container started on $MIRROR ✅" || \
|
||
warn "$container failed to start on $MIRROR — check manually"
|
||
done
|
||
}
|
||
|
||
# Create the mirror's Emby admin account on the owner's deployed Emby.
|
||
# Credentials come from the MIRROR's conf (HOST*_PARTNERSHIP_EMBY_ADMIN_USER/PASS).
|
||
# Checks for username collision before creating — exits with guidance if taken.
|
||
provision_emby_admin() {
|
||
local mirror_ip="$1"
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://${mirror_ip}:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Provisioning ━━━"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "$EMBY_API_KEY" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping Emby admin provisioning"
|
||
return 1
|
||
fi
|
||
|
||
# Read mirror's desired credentials from their own conf via SSH
|
||
log "Reading Emby credentials from $MIRROR conf..."
|
||
local username password
|
||
username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER")
|
||
password=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_PASS")
|
||
[[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")"
|
||
|
||
if [[ -z "$password" ]]; then
|
||
warn "PARTNERSHIP_EMBY_ADMIN_PASS is empty in $MIRROR conf"
|
||
warn "$MIRROR must set HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_PASS before onboard"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would create Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
# Collision check — username already exists?
|
||
local existing_users
|
||
existing_users=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
if echo "$existing_users" | grep -q "\"Name\":\"${username}\""; then
|
||
error "Emby username '${username}' is already taken on the shared instance"
|
||
error "Options:"
|
||
error " 1. Sign in with that account — it may already be yours"
|
||
error " 2. Set a different name in ${user_var} and re-run --onboard"
|
||
return 1
|
||
fi
|
||
|
||
log "Creating Emby admin '$username' at $emby_url..."
|
||
|
||
local create_response http_code body
|
||
create_response=$(curl -sf --max-time 15 -w "\n%{http_code}" \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"Name\": \"$username\"}" \
|
||
"${emby_url}/Users/New" 2>/dev/null)
|
||
http_code=$(echo "$create_response" | tail -1)
|
||
body=$(echo "$create_response" | head -n -1)
|
||
|
||
if [[ "$http_code" != "200" ]] && [[ "$http_code" != "204" ]]; then
|
||
warn "Failed to create Emby user '$username' (HTTP $http_code)"
|
||
return 1
|
||
fi
|
||
|
||
local user_id
|
||
user_id=$(echo "$body" | grep -o '"Id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user created but could not parse user ID — set password manually"
|
||
return 1
|
||
fi
|
||
|
||
local pw_code
|
||
pw_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"NewPw\": \"$password\"}" \
|
||
"${emby_url}/Users/${user_id}/Password" 2>/dev/null)
|
||
|
||
if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then
|
||
echo "Emby admin '$username' created (id: $user_id) ✅"
|
||
else
|
||
warn "User created but password set failed (HTTP $pw_code) — set password manually"
|
||
fi
|
||
|
||
curl -sf --max-time 15 -o /dev/null \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"IsAdministrator": true, "IsDisabled": false}' \
|
||
"${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \
|
||
echo "$username granted admin policy ✅" || \
|
||
warn "Could not set admin policy — grant manually in Emby dashboard"
|
||
}
|
||
|
||
# Delete the mirror's Emby admin account from the owner's deployed Emby.
|
||
revoke_emby_admin() {
|
||
local mirror_ip="$1"
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://${mirror_ip}:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||
|
||
local username
|
||
username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER")
|
||
[[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "$EMBY_API_KEY" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping Emby admin revocation"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
log "Looking up Emby user '$username' at $emby_url..."
|
||
|
||
local users_json user_id
|
||
users_json=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
user_id=$(echo "$users_json" | \
|
||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
local del_code
|
||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X DELETE \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||
|
||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||
echo "Emby admin '$username' removed ✅"
|
||
else
|
||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||
fi
|
||
}
|
||
|
||
check_both_healthy() {
|
||
platform_storage_healthy || { error "Local array not healthy"; return 1; }
|
||
|
||
local mirror_ip
|
||
mirror_ip=$(resolve_tailscale_ip "$MIRROR")
|
||
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || {
|
||
error "Mirror $MIRROR not healthy"
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
do_final_sync() {
|
||
log "Running final critical sync..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||
[[ -z "$_share" ]] && continue
|
||
local _path="${_share%%|*}"
|
||
local _profile="${_share##*|}"
|
||
if [[ "$_path" == "$_profile" ]]; then
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
|
||
else
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
||
--profile="$_profile" --log
|
||
fi
|
||
done
|
||
else
|
||
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
||
fi
|
||
else
|
||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"
|
||
fi
|
||
warn "Final sync complete — mirror has current state ✅"
|
||
}
|
||
|
||
# Safe master.conf modification with error handling
|
||
update_master_conf() {
|
||
local key="$1" value="$2"
|
||
local conf="$SCRIPT_DIR/../master.conf"
|
||
if [[ ! -f "$conf" ]]; then
|
||
error "master.conf not found at $conf"
|
||
return 1
|
||
fi
|
||
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
|
||
echo "master.conf updated: ${key}=${value}"
|
||
return 0
|
||
else
|
||
error "Failed to update master.conf: ${key}=${value}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ── Library-mode guard — source only, skip all mode dispatch ─────────────────────────────────
|
||
# partnership_offboard.sh sources this file with PARTNERSHIP_LIB_MODE=1 to get helper
|
||
# functions without triggering any mode execution.
|
||
[[ "${PARTNERSHIP_LIB_MODE:-}" == "1" ]] && { return 0 2>/dev/null || exit 0; }
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Unblock ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "unblock" ]]; then
|
||
if [[ -z "$UNBLOCK_HOST" ]]; then
|
||
error "Usage: partnership_manager.sh --unblock <hostname>"
|
||
error "Example: partnership_manager.sh --unblock unRAID-Jayred365"
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo ""
|
||
echo "Currently blocklisted:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo "Blocklist is empty."
|
||
fi
|
||
exit 1
|
||
fi
|
||
|
||
if ! is_blocklisted "$UNBLOCK_HOST"; then
|
||
warn "$UNBLOCK_HOST is not on the blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $UNBLOCK_HOST from blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
remove_from_blocklist "$UNBLOCK_HOST"
|
||
warn "$UNBLOCK_HOST unblocked — re-onboarding is now permitted ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "status" ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━"
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER" || echo "unreachable")
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR" || echo "unreachable")
|
||
echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)"
|
||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||
echo " Enabled: ${PARTNERSHIP_ENABLED:-false}"
|
||
echo ""
|
||
|
||
# Local state
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded")
|
||
LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded")
|
||
LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote")
|
||
echo " Local state: $LOCAL_STATE"
|
||
[[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED"
|
||
[[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED"
|
||
[[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN"
|
||
else
|
||
echo " Local state: no state file found"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# Remote state
|
||
REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||
if [[ -n "$REMOTE_IP" ]]; then
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -n "$REMOTE_CONTENT" ]]; then
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2)
|
||
echo " Remote state: $REMOTE_STATE"
|
||
[[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED"
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then
|
||
echo ""
|
||
echo " ✅ Both servers agree: $LOCAL_STATE"
|
||
else
|
||
echo ""
|
||
echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
fi
|
||
else
|
||
echo " Remote state: not found on $REMOTE_SERVER_NAME"
|
||
fi
|
||
else
|
||
echo " Remote state: $REMOTE_SERVER_NAME unreachable"
|
||
fi
|
||
|
||
echo ""
|
||
echo " Auth WebUIs:"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
echo " ${entry%%|*} → port ${entry##*|}"
|
||
done
|
||
|
||
if [[ -f "$OFFLINE_COUNTER" ]]; then
|
||
OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
[[ "$OFFLINE_DAYS" -gt 0 ]] && \
|
||
echo "" && \
|
||
echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}"
|
||
fi
|
||
|
||
echo ""
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo " Blocklist:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " ⛔ $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo " Blocklist: empty"
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Check ━━━
|
||
# Called every 30min by critical_sync_maintenance.sh — must be silent when healthy
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "check" ]]; then
|
||
|
||
# Update last_seen_remote and offline counter based on rsync outcome
|
||
if [[ "$REMOTE_SEEN" == true ]]; then
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \
|
||
"$LOCAL_STATE_FILE" 2>/dev/null
|
||
fi
|
||
elif [[ "$REMOTE_UNSEEN" == true ]]; then
|
||
OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 ))
|
||
echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER"
|
||
|
||
# Auto-offboard threshold: threshold_days × 48 intervals/day (every 30min)
|
||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 48 ))
|
||
if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then
|
||
warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard"
|
||
notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \
|
||
"Partnership" "warning"
|
||
bash "$0" --offboard --reason=auto-offboard-timeout
|
||
exit 0
|
||
fi
|
||
log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)"
|
||
fi
|
||
|
||
# Skip blocklisted partners — they can't auto-reconnect; only --onboard re-establishes
|
||
if is_blocklisted "$REMOTE_SERVER_NAME"; then
|
||
log "Partnership check — $REMOTE_SERVER_NAME is blocklisted, skipping"
|
||
exit 0
|
||
fi
|
||
|
||
# Read remote state file
|
||
REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||
if [[ -z "$REMOTE_IP" ]]; then
|
||
log "Partnership check — remote unreachable, skipping state check"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -z "$REMOTE_CONTENT" ]]; then
|
||
# IP resolved but SSH returned nothing — could be auth failure, not just missing file
|
||
if [[ -f "$SCRIPT_DIR/ssh_setup.sh" ]]; then
|
||
bash "$SCRIPT_DIR/ssh_setup.sh" --validate 2>/dev/null || true
|
||
fi
|
||
log "Partnership check — remote state file not found"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN")
|
||
|
||
# Both agree and active — healthy, silent
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
echo "Partnership check — ACTIVE, both servers agree ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# Remote requested offboard — run full offboard in background so cron is not blocked
|
||
if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership check — $REMOTE_SERVER_NAME requested offboard"
|
||
if [[ "$AM_OWNER" == true ]]; then
|
||
warn "Owner finalising offboard — spawning partnership_offboard.sh in background"
|
||
nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \
|
||
--reason=mirror-requested \
|
||
>> /var/log/partnership_offboard.log 2>&1 &
|
||
warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow"
|
||
else
|
||
warn "Owner offboarded — spawning mirror-side cleanup in background"
|
||
nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \
|
||
--reason=owner-offboarded \
|
||
>> /var/log/partnership_offboard.log 2>&1 &
|
||
warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# Both inactive — nothing to do
|
||
if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then
|
||
echo "Partnership check — INACTIVE on both servers"
|
||
exit 0
|
||
fi
|
||
|
||
log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Onboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "onboard" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
# ── LOCAL-ONLY PATH ──────────────────────────────────────────────────────────
|
||
# HOST1-local setup steps that don't need HOST2 present. Called from
|
||
# partnership_onboard.sh --phase1-only so HOST1 can complete its own side
|
||
# (PARTNERSHIP_ENABLED flag, setup.db) while waiting for HOST2 to install and onboard.
|
||
if [[ "$LOCAL_ONLY" == true ]]; then
|
||
log "Mode: local-only — skipping remote pre-flight and WebUI steps"
|
||
log "Owner: $OWNER_ID ($OWNER) · Mirror: $MIRROR_ID ($MIRROR)"
|
||
echo ""
|
||
|
||
# Enable partnership in master.conf + push to all hosts
|
||
echo ""
|
||
echo "Enabling partnership in master.conf..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
local _master_conf="${SCRIPTS_ROOT:-$(dirname "$SCRIPT_DIR")}/master.conf"
|
||
if grep -q "^[[:space:]]*PARTNERSHIP_ENABLED=" "$_master_conf" 2>/dev/null; then
|
||
sed -i "s|^[[:space:]]*PARTNERSHIP_ENABLED=.*|PARTNERSHIP_ENABLED=true|" "$_master_conf"
|
||
else
|
||
echo "PARTNERSHIP_ENABLED=true" >> "$_master_conf"
|
||
fi
|
||
echo "PARTNERSHIP_ENABLED=true in master.conf ✅"
|
||
platform_push_conf | while IFS= read -r line; do log "$line"; done
|
||
else
|
||
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
|
||
fi
|
||
|
||
# Write HOST1_LOCAL_DONE flag to setup.db
|
||
local_state_file="$(platform_setup_db_path)"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
flag_key="${MY_ID}_LOCAL_DONE"
|
||
if grep -q "^${flag_key}=" "$local_state_file" 2>/dev/null; then
|
||
sed -i "s|^${flag_key}=.*|${flag_key}=true|" "$local_state_file"
|
||
else
|
||
echo "${flag_key}=true" >> "$local_state_file"
|
||
fi
|
||
platform_push_setup_state
|
||
echo "${MY_ID}_LOCAL_DONE=true written to setup.db ✅"
|
||
else
|
||
warn "DRY RUN — would write ${MY_ID}_LOCAL_DONE=true"
|
||
fi
|
||
|
||
echo ""
|
||
echo "HOST1 local setup complete. Waiting for $MIRROR_ID ($MIRROR) to onboard."
|
||
exit 0
|
||
fi
|
||
|
||
# ── FULL ONBOARD PATH (requires HOST2) ────────────────────────────────────
|
||
|
||
# Check already onboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership already ACTIVE — use --status for detail or --offboard to separate"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
# Warn if re-onboarding a previously blocked partner — onboard is deliberate so it proceeds
|
||
if is_blocklisted "$MIRROR"; then
|
||
warn "$MIRROR is on the blocklist from a previous offboard"
|
||
warn "Proceeding — blocklist will be cleared on successful onboard"
|
||
fi
|
||
|
||
log "Owner: $OWNER_ID ($OWNER)"
|
||
log "Mirror: $MIRROR_ID ($MIRROR)"
|
||
|
||
# Pre-flight
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||
|
||
resolve_remote_ip
|
||
check_connectivity
|
||
|
||
# Version parity — both servers must agree on unRAID version
|
||
check_os_version_parity || exit 1
|
||
|
||
# Remote array and Docker daemon
|
||
check_remote_array || exit 1
|
||
check_remote_docker_daemon || exit 1
|
||
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||
|
||
[[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; }
|
||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; }
|
||
|
||
log "Owner IP: $OWNER_IP"
|
||
log "Mirror IP: $MIRROR_IP"
|
||
|
||
# Reconfigure mirror WebUIs → owner IP
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$OWNER_IP" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# Verify WebUI connectivity
|
||
if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
|
||
echo "$container reachable at http://${OWNER_IP}:${port}/ ✅"
|
||
else
|
||
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# Write state files
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard"
|
||
echo "Local state: ACTIVE ✅"
|
||
remove_from_blocklist "$MIRROR"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
else
|
||
warn "DRY RUN — would write ACTIVE state and push to remote"
|
||
fi
|
||
|
||
# Emby admin provisioning — runs after container deployment (deploy step not yet built)
|
||
provision_emby_admin "$MIRROR_IP"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||
echo " Owner: $OWNER_ID ($OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then
|
||
warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅"
|
||
[[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \
|
||
notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually"
|
||
notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \
|
||
"Partnership" "warning"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Offboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "offboard" ]]; then
|
||
exec bash "$SCRIPT_DIR/partnership_offboard.sh" "${FILTERED_ARGS[@]}" --reason="$REASON"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Transfer ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "transfer" ]]; then
|
||
exec bash "$SCRIPT_DIR/partnership_transfer.sh" \
|
||
${TRANSFER_CONFIRM_INPUT:+--confirm="$TRANSFER_CONFIRM_INPUT"} \
|
||
"${FILTERED_ARGS[@]}"
|
||
fi
|
||
|
||
# (legacy inline transfer block — kept below for reference, unreachable)
|
||
if false; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Current owner: $OWNER_ID ($OWNER)"
|
||
echo " Current mirror: $MIRROR_ID ($MIRROR)"
|
||
echo ""
|
||
echo " After transfer:"
|
||
echo " New owner: $MIRROR_ID ($MIRROR)"
|
||
echo " New mirror: $OWNER_ID ($OWNER)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
# Confirmation check
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
||
echo ""
|
||
echo "To proceed pass exactly:"
|
||
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
||
echo ""
|
||
error "Transfer cancelled — confirmation required"
|
||
exit 1
|
||
fi
|
||
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
||
error "Confirmation string does not match — transfer cancelled"
|
||
exit 1
|
||
fi
|
||
log "Confirmation accepted"
|
||
else
|
||
warn "DRY RUN — confirmation check skipped"
|
||
fi
|
||
|
||
# Health strike system
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Health Verification ━━━"
|
||
log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
||
|
||
STRIKES=0
|
||
ATTEMPTS=0
|
||
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
||
|
||
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
||
(( ATTEMPTS++ ))
|
||
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
||
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
||
error "Transfer cancelled — try again when both servers are healthy"
|
||
exit 1
|
||
fi
|
||
|
||
if check_both_healthy; then
|
||
(( STRIKES++ ))
|
||
log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
||
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
||
else
|
||
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
||
STRIKES=0
|
||
sleep 30
|
||
fi
|
||
done
|
||
warn "Both servers healthy — proceeding with transfer"
|
||
|
||
NEW_OWNER_ID="$MIRROR_ID"
|
||
NEW_MIRROR_ID="$OWNER_ID"
|
||
NEW_OWNER="$MIRROR"
|
||
NEW_MIRROR="$OWNER"
|
||
NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY"
|
||
NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY"
|
||
NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}"
|
||
NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}"
|
||
|
||
NEW_OWNER_IP=$(resolve_tailscale_ip "$NEW_OWNER")
|
||
NEW_MIRROR_IP=$(resolve_tailscale_ip "$NEW_MIRROR")
|
||
|
||
[[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; }
|
||
[[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; }
|
||
|
||
# Final sync in current direction
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━"
|
||
do_final_sync
|
||
|
||
# Reconfigure new mirror WebUIs → new owner
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━"
|
||
log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \
|
||
"$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# New owner WebUIs → localhost (now manages directly)
|
||
log "New owner ($NEW_OWNER) WebUIs → localhost"
|
||
reconfigure_local_webuis "localhost"
|
||
|
||
# Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Update Ownership ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
|
||
|
||
# Update remote master.conf
|
||
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
|
||
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
|
||
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
|
||
echo "Remote master.conf updated ✅" || \
|
||
error "Failed to update remote master.conf — update manually"
|
||
else
|
||
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
|
||
fi
|
||
|
||
# Write state files
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
||
echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)"
|
||
echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR"
|
||
echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
else
|
||
warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅"
|
||
notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \
|
||
"Partnership" "normal"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi |