Gmer4Lfe
587440b6bc
The scp reused the local absolute path on the remote, so an appdata-mode mirror never received it, and nothing wrote the mirror's own state file at all — a fully onboarded mirror rendered as having no partnership.
1572 lines
65 KiB
Bash
Executable File
1572 lines
65 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= Partnership Manager ============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Manages the full lifecycle of a two-server partnership — onboard, daily health
|
||
# monitoring, offboard, and ownership transfer. Called by partnership_onboard.sh
|
||
# during initial setup, and by critical_sync_maintenance.sh every 30 minutes for
|
||
# the --check mode. All other modes are run manually.
|
||
#
|
||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer.
|
||
# AM_OWNER / AM_MIRROR flags (set by detect_hosts) control all routing —
|
||
# no hostname string comparisons anywhere in this script.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# --onboard (owner only)
|
||
# Reconfigures mirror auth WebUIs → owner's Tailscale IP
|
||
# Mirror operator clicks NPM → gets owner's NPM via Tailscale automatically
|
||
# Writes ACTIVE state on both servers
|
||
#
|
||
# --offboard (either server)
|
||
# Mirror-initiated: reconfigures own WebUIs → localhost, writes INACTIVE state
|
||
# Owner finalises on next --check: final sync, Tailscale removal
|
||
# Owner-initiated: final sync, reconfigures mirror WebUIs → localhost, removes
|
||
# mirror containers + appdata, SSH key revocation, Tailscale removal
|
||
# Both leave with clean state ✅
|
||
#
|
||
# --transfer (owner only)
|
||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf
|
||
# Requires confirmation string + consecutive health check passes
|
||
#
|
||
# --check (called every 30min by critical_sync_maintenance.sh)
|
||
# --remote-seen: rsync succeeded → reset offline counter, read remote state
|
||
# --remote-unseen: rsync failed → increment offline counter → auto-offboard at threshold
|
||
# Silent when healthy — never noisy on clean runs
|
||
#
|
||
# --status (either server)
|
||
# Show state files from both servers, blocklist, SSH key status
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Deferred offboard
|
||
# Either server can offboard without the other being reachable. The initiating
|
||
# server writes its state immediately and becomes independent. The other server
|
||
# reads the INACTIVE state on its next --check and finalises automatically.
|
||
# No message passing, no coordination required.
|
||
#
|
||
# Appdata cleanup on offboard
|
||
# Partner containers are stopped and removed. Their appdata bind-mount paths
|
||
# (collected via docker inspect before removal) are also deleted. Safety gate:
|
||
# only paths matching /mnt/*/appdata* are deleted — media and config shares
|
||
# outside the appdata tree are never touched.
|
||
#
|
||
# Blocklist is application-layer; Tailscale removal is network-layer
|
||
# Both happen on offboard. The blocklist prevents re-onboard until explicitly
|
||
# cleared with --unblock. Tailscale removal ends encrypted access at the network
|
||
# level. Grace period controls both simultaneously — one var, consistent behaviour.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root check
|
||
# All operations require root.
|
||
#
|
||
# Role enforcement
|
||
# Mirror cannot run --onboard or --transfer. Blocked with a clear error message.
|
||
#
|
||
# Version parity check
|
||
# --onboard verifies both servers are on compatible unRAID versions.
|
||
#
|
||
# SSH_TIMEOUT on all remote calls
|
||
# Every ssh/scp call is timeout-protected. No operation hangs on an unreachable peer.
|
||
#
|
||
# No acquire_lock — Deliberate
|
||
# This file is dual-role: an executable dispatcher AND a library that
|
||
# partnership_offboard.sh and partnership_transfer.sh source with
|
||
# PARTNERSHIP_LIB_MODE=1. A top-level acquire_lock would fire for every sourcing
|
||
# script, and --check can re-invoke this file as itself (bash "$0" --offboard),
|
||
# which a strict lock would deadlock. Concurrency is handled per-write with flock
|
||
# instead. Do not "fix" this to match the single-role scripts.
|
||
#
|
||
# flock on state writes
|
||
# write_state_file() serialises full state-file rewrites, and the --check counter
|
||
# updates are flocked separately on their own lock file. The offline counter is a
|
||
# read-modify-write: without the lock two overlapping --check cycles both read N and
|
||
# both write N+1, silently losing an increment and pushing the auto-offboard threshold
|
||
# past its configured window. The last_seen_remote sed is inside the same lock because
|
||
# it edits a file write_state_file() rewrites wholesale from other paths.
|
||
#
|
||
# Note the redirection must sit INSIDE a command substitution — "$( ... ) 201>file"
|
||
# attaches the descriptor to the assignment rather than the subshell, and flock then
|
||
# fails with "Bad file descriptor" while the unlocked write proceeds anyway.
|
||
#
|
||
# SIGTERM trap on grace period sleep
|
||
# Offboard grace period is interruptible — Ctrl-C aborts cleanly.
|
||
#
|
||
# Silent by default
|
||
# --check produces no output when both servers are healthy. Only state changes
|
||
# and threshold crossings produce output.
|
||
#
|
||
# ==============================================================================================
|
||
# STATE FILES
|
||
# ==============================================================================================
|
||
#
|
||
# $STATE_DIR/partnership_<hostname>.db — each host writes its own, partner reads via SSH
|
||
# $STATE_DIR/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared
|
||
# $STATE_DIR/partnership_offline_days.db — cumulative offline day counter
|
||
#
|
||
# All in STATE_DIR — survives reboots (on /boot in internal mode, appdata in flash mode).
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# PARTNERSHIP_ENABLED
|
||
# Global gate — set true once both servers are configured (default: false)
|
||
#
|
||
# PARTNERSHIP_OWNER_HOST
|
||
# "HOST1" or "HOST2" — flips on --transfer (default: "HOST1")
|
||
#
|
||
# PARTNERSHIP_REMOVE_TAILSCALE
|
||
# Remove mirror from tailnet on offboard (default: true)
|
||
#
|
||
# PARTNERSHIP_GRACE_HOURS
|
||
# Hours before Tailscale removal after offboard — backup access also expires then (default: 6)
|
||
#
|
||
# PARTNERSHIP_OFFLINE_THRESHOLD
|
||
# Days of missed sync cycles before auto-offboard triggers (default: 30)
|
||
#
|
||
# PARTNERSHIP_TRANSFER_CONFIRM
|
||
# Exact string required for --transfer (default: "i-understand-this-transfers-ownership")
|
||
#
|
||
# PARTNERSHIP_TRANSFER_STRIKES
|
||
# Consecutive health checks required before transfer proceeds (default: 3)
|
||
#
|
||
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS
|
||
# Max health check attempts before giving up (default: 20)
|
||
#
|
||
# PARTNERSHIP_ONBOARD_VERIFY
|
||
# Curl-verify each WebUI after reconfigure to confirm Tailscale routing works (default: true)
|
||
#
|
||
# PARTNERSHIP_ONBOARD_NOTIFY
|
||
# Notify both servers on successful onboard (default: true)
|
||
#
|
||
# PARTNERSHIP_SYNC_INTERVAL
|
||
# Informational — actual schedule is in cron (default: 15)
|
||
#
|
||
# TAILSCALE_API_KEY / TAILSCALE_TAILNET
|
||
# Required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_PARTNERSHIP_AUTH_WEBUIS
|
||
# Containers reconfigured on onboard/offboard. Format: "ContainerName|WebUIPort"
|
||
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_WEBUIS
|
||
#
|
||
# HOST*_PARTNERSHIP_MIRROR_BACKUPS
|
||
# Paths accessible to the partner during grace window after offboard.
|
||
# Aliased by detect_hosts() → PARTNERSHIP_MIRROR_BACKUPS
|
||
#
|
||
# HOST*_PARTNERSHIP_OWN_CONTAINERS
|
||
# Containers parked here during partnership, restarted on offboard.
|
||
# Aliased by detect_hosts() → PARTNERSHIP_OWN_CONTAINERS
|
||
#
|
||
# HOST*_PARTNERSHIP_SERVICES_STACK
|
||
# Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) deployed on mirror during onboard.
|
||
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# partnership_manager.sh --onboard
|
||
# Establish mirror relationship — owner only
|
||
#
|
||
# partnership_manager.sh --offboard
|
||
# Clean separation — either server. 10-second countdown before executing.
|
||
#
|
||
# partnership_manager.sh --offboard --dry-run
|
||
# Show the complete offboard sequence without executing
|
||
#
|
||
# partnership_manager.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||
# Flip ownership — owner only. Requires exact confirmation string.
|
||
#
|
||
# partnership_manager.sh --check --remote-seen|--remote-unseen
|
||
# Called by critical_sync_maintenance.sh every 30min — do not run manually
|
||
#
|
||
# partnership_manager.sh --status
|
||
# Show state files, blocklist, SSH key status from both servers
|
||
#
|
||
# partnership_manager.sh --unblock <hostname>
|
||
# Remove hostname from blocklist to permit re-onboarding
|
||
#
|
||
# Any mode supports --dry-run and --log
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
||
|
||
SSH_TIMEOUT=15
|
||
BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}"
|
||
|
||
# ── Parse mode flags before parse_args ────────────────────────────────────────────────────────
|
||
MODE=""
|
||
TRANSFER_CONFIRM_INPUT=""
|
||
REMOTE_SEEN=false
|
||
REMOTE_UNSEEN=false
|
||
REASON="manual"
|
||
UNBLOCK_HOST=""
|
||
FILTERED_ARGS=()
|
||
|
||
LOCAL_ONLY=false # --onboard --local-only: skips remote steps, runs HOST1-local setup only
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--onboard) MODE="onboard" ;;
|
||
--offboard) MODE="offboard" ;;
|
||
--transfer) MODE="transfer" ;;
|
||
--check) MODE="check" ;;
|
||
--status) MODE="status" ;;
|
||
--unblock) MODE="unblock" ;;
|
||
--local-only) LOCAL_ONLY=true ;;
|
||
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
||
--remote-seen) REMOTE_SEEN=true ;;
|
||
--remote-unseen) REMOTE_UNSEEN=true ;;
|
||
--reason=*) REASON="${arg#--reason=}" ;;
|
||
*)
|
||
if [[ "$MODE" == "unblock" ]] && [[ -z "$UNBLOCK_HOST" ]] && [[ "$arg" != --* ]]; then
|
||
UNBLOCK_HOST="$arg"
|
||
else
|
||
FILTERED_ARGS+=("$arg")
|
||
fi
|
||
;;
|
||
esac
|
||
done
|
||
|
||
parse_args "${FILTERED_ARGS[@]}"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
# detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS
|
||
detect_hosts
|
||
|
||
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
|
||
partnership_resolve_roles
|
||
|
||
# State files
|
||
LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
|
||
REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db"
|
||
OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db"
|
||
MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db"
|
||
OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db"
|
||
|
||
# ── Exit Trap — restart locally stopped containers if script crashes mid-cleanup ──────────────
|
||
# Used by cleanup_partner_containers() — also shared with partnership_offboard.sh which
|
||
# sources this file and registers the same trap.
|
||
declare -a _PM_TRAP_STOPPED=()
|
||
_pm_trap_restart_stopped() {
|
||
[[ ${#_PM_TRAP_STOPPED[@]} -eq 0 ]] && return
|
||
for c in "${_PM_TRAP_STOPPED[@]}"; do
|
||
[[ -z "$c" ]] && continue
|
||
if docker inspect "$c" >/dev/null 2>&1; then
|
||
warn "Exit trap: restarting $c (stopped but not removed)"
|
||
docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c"
|
||
fi
|
||
done
|
||
}
|
||
|
||
if [[ "${PARTNERSHIP_LIB_MODE:-}" != "1" ]]; then
|
||
if [[ -z "$MODE" ]]; then
|
||
error "No mode specified"
|
||
echo "Usage:"
|
||
echo " partnership_manager.sh --onboard"
|
||
echo " partnership_manager.sh --offboard"
|
||
echo " partnership_manager.sh --transfer --confirm=..."
|
||
echo " partnership_manager.sh --check --remote-seen|--remote-unseen"
|
||
echo " partnership_manager.sh --status"
|
||
echo " partnership_manager.sh --unblock <hostname>"
|
||
exit 1
|
||
fi
|
||
|
||
# Role-based access control
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
case "$MODE" in
|
||
onboard|transfer)
|
||
error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE"
|
||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||
exit 1
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
# Lock for all modes except --check (frequent) and --offboard (offboard script holds its own)
|
||
[[ "$MODE" != "check" && "$MODE" != "offboard" ]] && acquire_lock "strict"
|
||
|
||
trap _pm_trap_restart_stopped EXIT
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
read_state_file() {
|
||
local file="$1" key="$2"
|
||
grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2
|
||
}
|
||
|
||
write_state_file() {
|
||
local file="$1"
|
||
shift
|
||
# flock prevents concurrent writes to the same state file
|
||
(
|
||
flock -x 200
|
||
cat > "$file" << EOF
|
||
state=${1:-UNKNOWN}
|
||
owner=${OWNER}
|
||
mirror=${MIRROR}
|
||
onboarded=${2:-}
|
||
offboarded=${3:-}
|
||
triggered_by=${4:-}
|
||
reason=${5:-}
|
||
last_seen_remote=${6:-}
|
||
updated=$(date '+%Y-%m-%d %H:%M:%S')
|
||
EOF
|
||
) 200>"${file}.lock"
|
||
}
|
||
|
||
# Deliver the partnership state to the partner, at the path the partner actually reads, under
|
||
# both names its UI looks for.
|
||
#
|
||
# This used to be one scp to "root@ip:$local_file" — the LOCAL absolute path reused verbatim on
|
||
# the remote. That only works while both hosts install to the same place. HOST1 is on flash at
|
||
# /boot/config/plugins/varaverk and HOST2 is in appdata mode at /mnt/user/appdata/Varaverk, so
|
||
# the copy went to a directory HOST2 does not read and, more often, does not have — and the
|
||
# failure surfaced as "will propagate on next sync", which nothing does.
|
||
#
|
||
# Both names, because the page resolves one file per node: partnership_<that node's hostname>.db.
|
||
# The mirror needs partnership_<mirror>.db for its own card and partnership_<owner>.db for the
|
||
# owner's. Nothing on the mirror writes the first one during an owner-driven onboard, which is
|
||
# why a mirror that was fully onboarded still rendered as having no partnership at all.
|
||
# The content is symmetric — state/owner/mirror/onboarded — so one file serves as both.
|
||
push_state_to_remote() {
|
||
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would push state file to remote"
|
||
return 0
|
||
fi
|
||
|
||
# Resolve where the partner keeps its state, from the partner. varaverk.cfg names its
|
||
# SCRIPTS_DIR; absent it, the flash default is the right guess for a stock install.
|
||
local remote_sd remote_state_dir
|
||
remote_sd=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \
|
||
'grep -oP "(?<=SCRIPTS_DIR=\")[^\"]+" /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' \
|
||
2>/dev/null | tr -d '\r')
|
||
[[ -z "$remote_sd" ]] && remote_sd="/boot/config/plugins/varaverk"
|
||
remote_state_dir="${remote_sd}/data/state"
|
||
|
||
local rc=0
|
||
local name
|
||
for name in "partnership_${REMOTE_SERVER_NAME}.db" "partnership_${LOCAL_SERVER_NAME}.db"; do
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \
|
||
"mkdir -p '$remote_state_dir' && cat > '${remote_state_dir}/${name}'" \
|
||
< "$local_file" 2>/dev/null || rc=1
|
||
done
|
||
|
||
if [[ $rc -eq 0 ]]; then
|
||
echo "State file pushed to $REMOTE_SERVER_NAME:${remote_state_dir} ✅"
|
||
return 0
|
||
fi
|
||
warn "Could not push state to $REMOTE_SERVER_NAME:${remote_state_dir} — it will keep showing"
|
||
warn " no partnership until this succeeds. Nothing retries this on a schedule."
|
||
return 1
|
||
}
|
||
|
||
read_remote_state() {
|
||
local remote_ip="$1" ssh_key="$2" remote_file="$3"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
||
}
|
||
|
||
|
||
get_tailscale_device_id() {
|
||
local hostname="$1"
|
||
curl -sf --max-time 10 \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \
|
||
2>/dev/null | \
|
||
grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \
|
||
grep -o '"id":"[^"]*"' | \
|
||
grep -o '[^"]*"$' | tr -d '"'
|
||
}
|
||
|
||
remove_tailscale_device() {
|
||
local hostname="$1"
|
||
|
||
if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||
warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal"
|
||
return 1
|
||
fi
|
||
|
||
log "Looking up Tailscale device ID for $hostname..."
|
||
local device_id
|
||
device_id=$(get_tailscale_device_id "$hostname")
|
||
|
||
if [[ -z "$device_id" ]]; then
|
||
warn "$hostname not found in Tailscale — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
log "Removing $hostname (device $device_id) from Tailscale..."
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $hostname from Tailscale tailnet"
|
||
return 0
|
||
fi
|
||
|
||
curl -sf --max-time 10 -X DELETE \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \
|
||
warn "$hostname removed from Tailscale ✅" || \
|
||
error "Failed to remove $hostname from Tailscale — remove manually"
|
||
}
|
||
|
||
# ── Blocklist helpers ─────────────────────────────────────────────────────────────────────────
|
||
|
||
is_blocklisted() {
|
||
local hostname="$1"
|
||
[[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${hostname}|" "$BLOCKLIST_FILE" 2>/dev/null
|
||
}
|
||
|
||
add_to_blocklist() {
|
||
local hostname="$1" reason="${2:-offboard}"
|
||
local now
|
||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||
if ! is_blocklisted "$hostname"; then
|
||
echo "${hostname}|${now}|${reason}" >> "$BLOCKLIST_FILE"
|
||
log "Blocklisted: $hostname (reason: $reason)"
|
||
else
|
||
log "$hostname already on blocklist"
|
||
fi
|
||
}
|
||
|
||
remove_from_blocklist() {
|
||
local hostname="$1"
|
||
if [[ ! -f "$BLOCKLIST_FILE" ]]; then
|
||
log "Blocklist empty — nothing to remove"
|
||
return 0
|
||
fi
|
||
sed -i "/^${hostname}|/d" "$BLOCKLIST_FILE" 2>/dev/null
|
||
log "Removed from blocklist: $hostname"
|
||
}
|
||
|
||
# Revoke SSH access on both sides — call after all other SSH operations complete.
|
||
# Matches by key comment (format: keyname@hostname — set by ssh_setup.sh at keygen time).
|
||
# SSH_REVOKE_REMOTE_OK / SSH_REVOKE_LOCAL_OK set in caller scope for summary display.
|
||
do_ssh_key_revocation() {
|
||
local remote_ip="$1"
|
||
local pub_key_file="${SSH_KEY}.pub"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD SSH Key Revocation ━━━"
|
||
|
||
# Step 1: remove our pubkey from remote's authorized_keys while SSH still works
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
if [[ -f "$pub_key_file" ]]; then
|
||
local our_comment
|
||
our_comment=$(awk '{print $3}' "$pub_key_file" 2>/dev/null)
|
||
if [[ -n "$our_comment" ]]; then
|
||
log "Revoking our pubkey ($our_comment) from $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove our pubkey from $REMOTE_SERVER_NAME authorized_keys"
|
||
SSH_REVOKE_REMOTE_OK=true
|
||
elif [[ -n "$remote_ip" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||
"grep -v '${our_comment}' /root/.ssh/authorized_keys \
|
||
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
|
||
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
|
||
&& chmod 600 /root/.ssh/authorized_keys \
|
||
&& echo removed" 2>/dev/null | grep -q removed; then
|
||
echo "Our pubkey revoked from $REMOTE_SERVER_NAME ✅"
|
||
SSH_REVOKE_REMOTE_OK=true
|
||
else
|
||
warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:"
|
||
warn " grep -v '@${LOCAL_SERVER_NAME}' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys"
|
||
fi
|
||
else
|
||
warn "Could not read pubkey comment from $pub_key_file — skipping remote revocation"
|
||
fi
|
||
else
|
||
warn "Pubkey not found at $pub_key_file — skipping remote revocation"
|
||
fi
|
||
|
||
# Step 2: remove remote's pubkey from our local authorized_keys
|
||
# Remote key comment ends with @REMOTE_SERVER_NAME — unique, no special chars
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
log "Revoking $REMOTE_SERVER_NAME pubkey from local authorized_keys..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $REMOTE_SERVER_NAME pubkey from local authorized_keys"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
elif [[ -f /root/.ssh/authorized_keys ]]; then
|
||
if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then
|
||
# chmod after the mv, every time. `>` creates the temp file under the shell's umask
|
||
# and `mv` keeps the NEW file's mode, so this rewrite left authorized_keys 0666 on a
|
||
# filesystem that permits it. sshd's StrictModes then refuses every key in it without
|
||
# saying so to the client — the key is present, byte-correct, and inert, and the next
|
||
# onboard's SSH step fails with nothing in any Varaverk log to explain it. Only
|
||
# /var/log/syslog knows: "Authentication refused: bad ownership or modes".
|
||
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
|
||
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
|
||
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys && \
|
||
chmod 600 /root/.ssh/authorized_keys; then
|
||
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
else
|
||
warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually"
|
||
fi
|
||
else
|
||
log "$REMOTE_SERVER_NAME pubkey not found in local authorized_keys — already removed"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
fi
|
||
else
|
||
log "/root/.ssh/authorized_keys not found — nothing to remove locally"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
fi
|
||
}
|
||
|
||
# Gather all partner fallback containers for this server (all tiers)
|
||
gather_partner_fallback_containers() {
|
||
local out_var="$1"
|
||
eval "${out_var}=()"
|
||
local tier var
|
||
for tier in TIER1 TIER2 TIER3 TIER4; do
|
||
var="FALLBACK_${REMOTE_ID}_${tier}"
|
||
# Check if var is set and is an array
|
||
if declare -p "$var" >/dev/null 2>&1; then
|
||
local -a _tmp_arr
|
||
eval "_tmp_arr=(\"\${${var}[@]}\")"
|
||
for c in "${_tmp_arr[@]}"; do
|
||
[[ -n "$c" ]] && eval "${out_var}+=(\"\$c\")"
|
||
done
|
||
fi
|
||
done
|
||
}
|
||
|
||
# read_remote_conf_var() / read_remote_conf_array() — provided by common.sh
|
||
|
||
# derive_short_name() — provided by common.sh
|
||
|
||
# Start this server's own parked containers after partnership ends.
|
||
start_own_stack() {
|
||
# Returns non-zero if any container failed. It used to return whatever the loop's last
|
||
# docker start happened to produce, so a caller checking it learned nothing — and the
|
||
# offboard summary just printed "Step 6 — Own stack: started" either way.
|
||
local _rc=0
|
||
echo ""
|
||
echo "━━━ $ICON_START Restart Own Stack ━━━"
|
||
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No PARTNERSHIP_OWN_CONTAINERS configured — skipping"
|
||
return 0
|
||
fi
|
||
for container in "${PARTNERSHIP_OWN_CONTAINERS[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start: $container"
|
||
continue
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then
|
||
echo "$container started ✅"
|
||
else
|
||
warn "$container failed to start — check manually"
|
||
_rc=1
|
||
fi
|
||
done
|
||
return "$_rc"
|
||
}
|
||
|
||
# Remove partnership containers on this server + their appdata bind-mount paths.
|
||
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
|
||
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
|
||
cleanup_partner_containers() {
|
||
# Returns non-zero if any container or appdata path could not be removed. Previously the
|
||
# exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅"
|
||
# was printed over a container that failed to remove.
|
||
local _rc=0
|
||
declare -a containers=()
|
||
gather_partner_fallback_containers containers
|
||
|
||
if [[ ${#containers[@]} -eq 0 ]]; then
|
||
log "No partner containers found to remove"
|
||
return 0
|
||
fi
|
||
|
||
# Collect appdata paths BEFORE any removal — inspect fails once container is gone
|
||
local all_appdata_paths=""
|
||
for container in "${containers[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
local cpaths
|
||
cpaths=$(docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
|
||
"$container" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
||
[[ -n "$cpaths" ]] && all_appdata_paths+=$'\n'"$cpaths"
|
||
fi
|
||
done
|
||
|
||
# Remove containers
|
||
for container in "${containers[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop + rm: $container"
|
||
continue
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
|
||
_PM_TRAP_STOPPED+=("$container")
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
|
||
echo "$container removed ✅"
|
||
else
|
||
warn "$container rm failed"
|
||
_rc=1
|
||
fi
|
||
else
|
||
log "$container not found — skipping"
|
||
fi
|
||
done
|
||
|
||
# Delete appdata after containers are gone
|
||
while IFS= read -r path; do
|
||
[[ -z "$path" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn " DRY RUN — would rm -rf $path"
|
||
continue
|
||
fi
|
||
if rm -rf "$path"; then
|
||
echo " Appdata removed: $path ✅"
|
||
else
|
||
warn " Failed to remove: $path"
|
||
_rc=1
|
||
fi
|
||
done <<< "$all_appdata_paths"
|
||
return "$_rc"
|
||
}
|
||
|
||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
|
||
# and their appdata bind-mount paths.
|
||
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
|
||
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
|
||
cleanup_owner_containers_on_mirror() {
|
||
# Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly
|
||
# rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the
|
||
# containers on it are gone.
|
||
local _rc=0
|
||
local mirror_ip="$1"
|
||
local owner_short
|
||
owner_short=$(derive_short_name "$OWNER")
|
||
|
||
log "Removing owner-deployed containers from $MIRROR..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove *-${owner_short} containers + appdata from $MIRROR"
|
||
return 0
|
||
fi
|
||
|
||
local container_list
|
||
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
|
||
|
||
if [[ -z "$container_list" ]]; then
|
||
log "No *-${owner_short} containers found on $MIRROR"
|
||
return 0
|
||
fi
|
||
|
||
while IFS= read -r container; do
|
||
[[ -z "$container" ]] && continue
|
||
|
||
# Collect appdata paths before removal
|
||
local appdata_paths
|
||
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||
|
||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"docker stop '$container' >/dev/null 2>&1
|
||
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||
grep -q removed; then
|
||
echo "$container removed from $MIRROR ✅"
|
||
else
|
||
warn "Failed to remove $container from $MIRROR"
|
||
_rc=1
|
||
fi
|
||
|
||
# Delete appdata on remote after container removal
|
||
while IFS= read -r path; do
|
||
[[ -z "$path" ]] && continue
|
||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
|
||
echo " Appdata removed on $MIRROR: $path ✅"
|
||
else
|
||
warn " Failed to remove appdata on $MIRROR: $path"
|
||
_rc=1
|
||
fi
|
||
done <<< "$appdata_paths"
|
||
done <<< "$container_list"
|
||
return "$_rc"
|
||
}
|
||
|
||
# SSH to mirror — start mirror's own parked containers.
|
||
# Reads PARTNERSHIP_OWN_CONTAINERS from the mirror's own conf via SSH.
|
||
start_mirror_own_stack() {
|
||
local mirror_ip="$1"
|
||
|
||
log "Reading own stack list from $MIRROR conf..."
|
||
local -a mirror_own=()
|
||
mapfile -t mirror_own < <(read_remote_conf_array "$mirror_ip" "PARTNERSHIP_OWN_CONTAINERS")
|
||
# Remove empty entries
|
||
local -a filtered=()
|
||
for c in "${mirror_own[@]}"; do [[ -n "$c" ]] && filtered+=("$c"); done
|
||
mirror_own=("${filtered[@]}")
|
||
|
||
if [[ ${#mirror_own[@]} -eq 0 ]]; then
|
||
log "No PARTNERSHIP_OWN_CONTAINERS configured on $MIRROR — skipping remote stack restart"
|
||
return 0
|
||
fi
|
||
|
||
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
|
||
local _rc=0
|
||
for container in "${mirror_own[@]}"; do
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container on $MIRROR"
|
||
continue
|
||
fi
|
||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
||
grep -q started; then
|
||
echo "$container started on $MIRROR ✅"
|
||
else
|
||
warn "$container failed to start on $MIRROR — check manually"
|
||
_rc=1
|
||
fi
|
||
done
|
||
return "$_rc"
|
||
}
|
||
|
||
# Create the mirror's Emby admin account on the owner's deployed Emby.
|
||
# Credentials come from the MIRROR's conf (HOST*_PARTNERSHIP_EMBY_ADMIN_USER/PASS).
|
||
# Checks for username collision before creating — exits with guidance if taken.
|
||
provision_emby_admin() {
|
||
local mirror_ip="$1"
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://${mirror_ip}:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Provisioning ━━━"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "$EMBY_API_KEY" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping Emby admin provisioning"
|
||
return 1
|
||
fi
|
||
|
||
# Read mirror's desired credentials from their own conf via SSH
|
||
log "Reading Emby credentials from $MIRROR conf..."
|
||
local username password
|
||
username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER")
|
||
password=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_PASS")
|
||
[[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")"
|
||
|
||
if [[ -z "$password" ]]; then
|
||
warn "PARTNERSHIP_EMBY_ADMIN_PASS is empty in $MIRROR conf"
|
||
warn "$MIRROR must set HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_PASS before onboard"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would create Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
# Collision check — username already exists?
|
||
local existing_users
|
||
existing_users=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
if echo "$existing_users" | grep -q "\"Name\":\"${username}\""; then
|
||
error "Emby username '${username}' is already taken on the shared instance"
|
||
error "Options:"
|
||
error " 1. Sign in with that account — it may already be yours"
|
||
error " 2. Set a different name in HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_USER and re-run --onboard"
|
||
return 1
|
||
fi
|
||
|
||
log "Creating Emby admin '$username' at $emby_url..."
|
||
|
||
local create_response http_code body
|
||
create_response=$(curl -sf --max-time 15 -w "\n%{http_code}" \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"Name\": \"$username\"}" \
|
||
"${emby_url}/Users/New" 2>/dev/null)
|
||
http_code=$(echo "$create_response" | tail -1)
|
||
body=$(echo "$create_response" | head -n -1)
|
||
|
||
if [[ "$http_code" != "200" ]] && [[ "$http_code" != "204" ]]; then
|
||
warn "Failed to create Emby user '$username' (HTTP $http_code)"
|
||
return 1
|
||
fi
|
||
|
||
local user_id
|
||
user_id=$(echo "$body" | grep -o '"Id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user created but could not parse user ID — set password manually"
|
||
return 1
|
||
fi
|
||
|
||
local pw_code
|
||
pw_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"NewPw\": \"$password\"}" \
|
||
"${emby_url}/Users/${user_id}/Password" 2>/dev/null)
|
||
|
||
if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then
|
||
echo "Emby admin '$username' created (id: $user_id) ✅"
|
||
else
|
||
warn "User created but password set failed (HTTP $pw_code) — set password manually"
|
||
fi
|
||
|
||
curl -sf --max-time 15 -o /dev/null \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"IsAdministrator": true, "IsDisabled": false}' \
|
||
"${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \
|
||
echo "$username granted admin policy ✅" || \
|
||
warn "Could not set admin policy — grant manually in Emby dashboard"
|
||
}
|
||
|
||
# Delete the mirror's Emby admin account from the owner's deployed Emby.
|
||
revoke_emby_admin() {
|
||
local mirror_ip="$1"
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://${mirror_ip}:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||
|
||
local username
|
||
username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER")
|
||
[[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "$EMBY_API_KEY" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping Emby admin revocation"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
log "Looking up Emby user '$username' at $emby_url..."
|
||
|
||
local users_json user_id
|
||
users_json=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
user_id=$(echo "$users_json" | \
|
||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
local del_code
|
||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X DELETE \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||
|
||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||
echo "Emby admin '$username' removed ✅"
|
||
else
|
||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||
fi
|
||
}
|
||
|
||
check_both_healthy() {
|
||
platform_storage_healthy || { error "Local array not healthy"; return 1; }
|
||
|
||
local mirror_ip
|
||
mirror_ip=$(resolve_tailscale_ip "$MIRROR")
|
||
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || {
|
||
error "Mirror $MIRROR not healthy"
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
# Returns non-zero when the mirror did NOT leave with current state, so the caller can say so.
|
||
# The whole point of this step is the guarantee in the summary — "mirror leaves with current
|
||
# Critical-Data" — and it used to print that unconditionally at the end of the function. An
|
||
# offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs"
|
||
# immediately followed by "Final sync complete — mirror has current state ✅", and the summary
|
||
# scored Step 2 as a pass. The mirror left with whatever it happened to have.
|
||
do_final_sync() {
|
||
log "Running final critical sync..."
|
||
local _synced=0 _failed=0
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||
[[ -z "$_share" ]] && continue
|
||
local _path="${_share%%|*}"
|
||
local _profile="${_share##*|}"
|
||
if [[ "$_path" == "$_profile" ]]; then
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
|
||
else
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
||
--profile="$_profile" --log
|
||
fi
|
||
# rsync.sh exits 0 both on a real sync and on a clean Tier-1 gate exit, so the
|
||
# gate is checked here rather than inferred from its status.
|
||
if [[ "${RSYNC_ENABLED:-true}" == true ]]; then (( _synced++ )); else (( _failed++ )); fi
|
||
done
|
||
else
|
||
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
||
_failed=1
|
||
fi
|
||
else
|
||
# ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution",
|
||
# so this line aborted Step 2 of every offboard --dry-run with a shell error instead of
|
||
# printing. A count of an unset array is already 0, which is the only default needed.
|
||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)"
|
||
return 0
|
||
fi
|
||
|
||
if [[ "$_failed" -gt 0 ]]; then
|
||
warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had"
|
||
[[ "${RSYNC_ENABLED:-true}" != true ]] && \
|
||
warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent"
|
||
return 1
|
||
fi
|
||
warn "Final sync complete — mirror has current state ✅"
|
||
return 0
|
||
}
|
||
|
||
# Safe master.conf modification with error handling — appends the key if not already present,
|
||
# since sed -i returns 0 whether or not it matched anything.
|
||
update_master_conf() {
|
||
local key="$1" value="$2"
|
||
local conf="$CONF_DIR/master.conf"
|
||
if [[ ! -f "$conf" ]]; then
|
||
error "master.conf not found at $conf"
|
||
return 1
|
||
fi
|
||
if grep -q "^[[:space:]]*${key}=" "$conf" 2>/dev/null; then
|
||
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
|
||
echo "master.conf updated: ${key}=${value}"
|
||
return 0
|
||
else
|
||
error "Failed to update master.conf: ${key}=${value}"
|
||
return 1
|
||
fi
|
||
else
|
||
if echo " ${key}=${value}" >> "$conf" 2>/dev/null; then
|
||
echo "master.conf updated: ${key}=${value} (appended)"
|
||
return 0
|
||
else
|
||
error "Failed to append to master.conf: ${key}=${value}"
|
||
return 1
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# ── Library-mode guard — source only, skip all mode dispatch ─────────────────────────────────
|
||
# partnership_offboard.sh sources this file with PARTNERSHIP_LIB_MODE=1 to get helper
|
||
# functions without triggering any mode execution.
|
||
[[ "${PARTNERSHIP_LIB_MODE:-}" == "1" ]] && { return 0 2>/dev/null || exit 0; }
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Unblock ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "unblock" ]]; then
|
||
if [[ -z "$UNBLOCK_HOST" ]]; then
|
||
error "Usage: partnership_manager.sh --unblock <hostname>"
|
||
error "Example: partnership_manager.sh --unblock unRAID-Jayred365"
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo ""
|
||
echo "Currently blocklisted:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo "Blocklist is empty."
|
||
fi
|
||
exit 1
|
||
fi
|
||
|
||
if ! is_blocklisted "$UNBLOCK_HOST"; then
|
||
warn "$UNBLOCK_HOST is not on the blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $UNBLOCK_HOST from blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
remove_from_blocklist "$UNBLOCK_HOST"
|
||
warn "$UNBLOCK_HOST unblocked — re-onboarding is now permitted ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "status" ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━"
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER" || echo "unreachable")
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR" || echo "unreachable")
|
||
echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)"
|
||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||
echo " Enabled: ${PARTNERSHIP_ENABLED:-false}"
|
||
echo ""
|
||
|
||
# Local state
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded")
|
||
LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded")
|
||
LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote")
|
||
echo " Local state: $LOCAL_STATE"
|
||
[[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED"
|
||
[[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED"
|
||
[[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN"
|
||
else
|
||
echo " Local state: no state file found"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# Remote state
|
||
REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||
if [[ -n "$REMOTE_IP" ]]; then
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -n "$REMOTE_CONTENT" ]]; then
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2)
|
||
echo " Remote state: $REMOTE_STATE"
|
||
[[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED"
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then
|
||
echo ""
|
||
echo " ✅ Both servers agree: $LOCAL_STATE"
|
||
else
|
||
echo ""
|
||
echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
fi
|
||
else
|
||
echo " Remote state: not found on $REMOTE_SERVER_NAME"
|
||
fi
|
||
else
|
||
echo " Remote state: $REMOTE_SERVER_NAME unreachable"
|
||
fi
|
||
|
||
echo ""
|
||
echo " Auth WebUIs:"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
echo " ${entry%%|*} → port ${entry##*|}"
|
||
done
|
||
|
||
if [[ -f "$OFFLINE_COUNTER" ]]; then
|
||
OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
[[ "$OFFLINE_DAYS" -gt 0 ]] && \
|
||
echo "" && \
|
||
echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}"
|
||
fi
|
||
|
||
echo ""
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo " Blocklist:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " ⛔ $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo " Blocklist: empty"
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Check ━━━
|
||
# Called every 30min by critical_sync_maintenance.sh — must be silent when healthy
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "check" ]]; then
|
||
|
||
# Update last_seen_remote and offline counter based on rsync outcome.
|
||
#
|
||
# Both branches are flocked. The counter is a read-modify-write, so two overlapping
|
||
# --check cycles would otherwise both read N and both write N+1 — silently losing an
|
||
# increment and pushing the auto-offboard threshold further out than configured. The
|
||
# sed on the state file is included because write_state_file() flocks the same file
|
||
# from other code paths, and an unsynchronised sed -i can land mid-rewrite.
|
||
if [[ "$REMOTE_SEEN" == true ]]; then
|
||
(
|
||
flock -x 201
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \
|
||
"$LOCAL_STATE_FILE" 2>/dev/null
|
||
fi
|
||
) 201>"${OFFLINE_COUNTER}.lock"
|
||
elif [[ "$REMOTE_UNSEEN" == true ]]; then
|
||
# Redirection must live INSIDE the substitution — "$( ... ) 201>file" attaches the fd
|
||
# to the assignment, not to the subshell doing the work, and flock then fails with
|
||
# "Bad file descriptor" while the increment silently proceeds unlocked.
|
||
OFFLINE_COUNT=$( {
|
||
flock -x 201
|
||
_c=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
_c=$(( _c + 1 ))
|
||
echo "$_c" > "$OFFLINE_COUNTER"
|
||
echo "$_c"
|
||
} 201>"${OFFLINE_COUNTER}.lock" )
|
||
|
||
# Auto-offboard threshold: threshold_days × 48 intervals/day (every 30min)
|
||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 48 ))
|
||
if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then
|
||
warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard"
|
||
notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \
|
||
"Partnership" "warning"
|
||
bash "$0" --offboard --reason=auto-offboard-timeout
|
||
exit 0
|
||
fi
|
||
log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)"
|
||
fi
|
||
|
||
# Skip blocklisted partners — they can't auto-reconnect; only --onboard re-establishes
|
||
if is_blocklisted "$REMOTE_SERVER_NAME"; then
|
||
log "Partnership check — $REMOTE_SERVER_NAME is blocklisted, skipping"
|
||
exit 0
|
||
fi
|
||
|
||
# Read remote state file
|
||
REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||
if [[ -z "$REMOTE_IP" ]]; then
|
||
log "Partnership check — remote unreachable, skipping state check"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -z "$REMOTE_CONTENT" ]]; then
|
||
# IP resolved but SSH returned nothing — could be auth failure, not just missing file
|
||
if [[ -f "$SCRIPT_DIR/ssh_setup.sh" ]]; then
|
||
bash "$SCRIPT_DIR/ssh_setup.sh" --validate 2>/dev/null || true
|
||
fi
|
||
log "Partnership check — remote state file not found"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN")
|
||
|
||
# Both agree and active — healthy, silent
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
echo "Partnership check — ACTIVE, both servers agree ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# Remote requested offboard — run full offboard in background so cron is not blocked
|
||
if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership check — $REMOTE_SERVER_NAME requested offboard"
|
||
if [[ "$AM_OWNER" == true ]]; then
|
||
warn "Owner finalising offboard — spawning partnership_offboard.sh in background"
|
||
nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \
|
||
--reason=mirror-requested \
|
||
>> /var/log/partnership_offboard.log 2>&1 &
|
||
warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow"
|
||
else
|
||
warn "Owner offboarded — spawning mirror-side cleanup in background"
|
||
nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \
|
||
--reason=owner-offboarded \
|
||
>> /var/log/partnership_offboard.log 2>&1 &
|
||
warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# Both inactive — nothing to do
|
||
if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then
|
||
echo "Partnership check — INACTIVE on both servers"
|
||
exit 0
|
||
fi
|
||
|
||
log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Onboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "onboard" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
# ── LOCAL-ONLY PATH ──────────────────────────────────────────────────────────
|
||
# HOST1-local setup steps that don't need HOST2 present. Called from
|
||
# partnership_onboard.sh --phase1-only so HOST1 can complete its own side
|
||
# (PARTNERSHIP_ENABLED flag, setup.db) while waiting for HOST2 to install and onboard.
|
||
if [[ "$LOCAL_ONLY" == true ]]; then
|
||
log "Mode: local-only — skipping remote pre-flight and WebUI steps"
|
||
log "Owner: $OWNER_ID ($OWNER) · Mirror: $MIRROR_ID ($MIRROR)"
|
||
echo ""
|
||
|
||
# Enable partnership in master.conf + push to all hosts
|
||
echo ""
|
||
echo "Enabling partnership in master.conf..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "PARTNERSHIP_ENABLED" "true"
|
||
platform_push_conf | while IFS= read -r line; do log "$line"; done
|
||
else
|
||
warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push"
|
||
fi
|
||
|
||
# Write HOST1_LOCAL_DONE flag to setup.db
|
||
local_state_file="$(platform_setup_db_path)"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
flag_key="${MY_ID}_LOCAL_DONE"
|
||
if grep -q "^${flag_key}=" "$local_state_file" 2>/dev/null; then
|
||
sed -i "s|^${flag_key}=.*|${flag_key}=true|" "$local_state_file"
|
||
else
|
||
echo "${flag_key}=true" >> "$local_state_file"
|
||
fi
|
||
platform_push_setup_state
|
||
echo "${MY_ID}_LOCAL_DONE=true written to setup.db ✅"
|
||
else
|
||
warn "DRY RUN — would write ${MY_ID}_LOCAL_DONE=true"
|
||
fi
|
||
|
||
echo ""
|
||
echo "HOST1 local setup complete. Waiting for $MIRROR_ID ($MIRROR) to onboard."
|
||
exit 0
|
||
fi
|
||
|
||
# ── FULL ONBOARD PATH (requires HOST2) ────────────────────────────────────
|
||
|
||
# Check already onboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership already ACTIVE — use --status for detail or --offboard to separate"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
# Warn if re-onboarding a previously blocked partner — onboard is deliberate so it proceeds
|
||
if is_blocklisted "$MIRROR"; then
|
||
warn "$MIRROR is on the blocklist from a previous offboard"
|
||
warn "Proceeding — blocklist will be cleared on successful onboard"
|
||
fi
|
||
|
||
log "Owner: $OWNER_ID ($OWNER)"
|
||
log "Mirror: $MIRROR_ID ($MIRROR)"
|
||
|
||
# Pre-flight
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||
|
||
resolve_remote_ip
|
||
check_connectivity
|
||
|
||
# Version parity — both servers must agree on unRAID version
|
||
check_os_version_parity || exit 1
|
||
|
||
# Remote array and Docker daemon
|
||
check_remote_array || exit 1
|
||
check_remote_docker_daemon || exit 1
|
||
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||
|
||
[[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; }
|
||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; }
|
||
|
||
log "Owner IP: $OWNER_IP"
|
||
log "Mirror IP: $MIRROR_IP"
|
||
|
||
# Reconfigure mirror WebUIs → owner IP
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$OWNER_IP" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# Verify WebUI connectivity
|
||
if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
|
||
echo "$container reachable at http://${OWNER_IP}:${port}/ ✅"
|
||
else
|
||
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# Write state files
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard"
|
||
echo "Local state: ACTIVE ✅"
|
||
remove_from_blocklist "$MIRROR"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
else
|
||
warn "DRY RUN — would write ACTIVE state and push to remote"
|
||
fi
|
||
|
||
# Emby admin provisioning — runs after container deployment
|
||
provision_emby_admin "$MIRROR_IP"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||
echo " Owner: $OWNER_ID ($OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then
|
||
warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅"
|
||
[[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \
|
||
notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually"
|
||
notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \
|
||
"Partnership" "warning"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Offboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "offboard" ]]; then
|
||
exec bash "$SCRIPT_DIR/partnership_offboard.sh" "${FILTERED_ARGS[@]}" --reason="$REASON"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Transfer ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "transfer" ]]; then
|
||
exec bash "$SCRIPT_DIR/partnership_transfer.sh" \
|
||
${TRANSFER_CONFIRM_INPUT:+--confirm="$TRANSFER_CONFIRM_INPUT"} \
|
||
"${FILTERED_ARGS[@]}"
|
||
fi
|
||
|
||
# (legacy inline transfer block — kept below for reference, unreachable)
|
||
if false; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Current owner: $OWNER_ID ($OWNER)"
|
||
echo " Current mirror: $MIRROR_ID ($MIRROR)"
|
||
echo ""
|
||
echo " After transfer:"
|
||
echo " New owner: $MIRROR_ID ($MIRROR)"
|
||
echo " New mirror: $OWNER_ID ($OWNER)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
# Confirmation check
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
||
echo ""
|
||
echo "To proceed pass exactly:"
|
||
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
||
echo ""
|
||
error "Transfer cancelled — confirmation required"
|
||
exit 1
|
||
fi
|
||
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
||
error "Confirmation string does not match — transfer cancelled"
|
||
exit 1
|
||
fi
|
||
log "Confirmation accepted"
|
||
else
|
||
warn "DRY RUN — confirmation check skipped"
|
||
fi
|
||
|
||
# Health strike system
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Health Verification ━━━"
|
||
log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
||
|
||
STRIKES=0
|
||
ATTEMPTS=0
|
||
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
||
|
||
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
||
(( ATTEMPTS++ ))
|
||
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
||
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
||
error "Transfer cancelled — try again when both servers are healthy"
|
||
exit 1
|
||
fi
|
||
|
||
if check_both_healthy; then
|
||
(( STRIKES++ ))
|
||
log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
||
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
||
else
|
||
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
||
STRIKES=0
|
||
sleep 30
|
||
fi
|
||
done
|
||
warn "Both servers healthy — proceeding with transfer"
|
||
|
||
NEW_OWNER_ID="$MIRROR_ID"
|
||
NEW_MIRROR_ID="$OWNER_ID"
|
||
NEW_OWNER="$MIRROR"
|
||
NEW_MIRROR="$OWNER"
|
||
NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY"
|
||
NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY"
|
||
NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}"
|
||
NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}"
|
||
|
||
NEW_OWNER_IP=$(resolve_tailscale_ip "$NEW_OWNER")
|
||
NEW_MIRROR_IP=$(resolve_tailscale_ip "$NEW_MIRROR")
|
||
|
||
[[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; }
|
||
[[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; }
|
||
|
||
# Final sync in current direction
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━"
|
||
do_final_sync
|
||
|
||
# Reconfigure new mirror WebUIs → new owner
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━"
|
||
log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \
|
||
"$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# New owner WebUIs → localhost (now manages directly)
|
||
log "New owner ($NEW_OWNER) WebUIs → localhost"
|
||
reconfigure_local_webuis "localhost"
|
||
|
||
# Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Update Ownership ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
|
||
|
||
# Update remote master.conf
|
||
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$NEW_MIRROR_IP" \
|
||
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
|
||
'$SCRIPT_DIR/../Configurations/master.conf'" 2>/dev/null && \
|
||
echo "Remote master.conf updated ✅" || \
|
||
error "Failed to update remote master.conf — update manually"
|
||
else
|
||
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
|
||
fi
|
||
|
||
# Write state files
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
||
echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)"
|
||
echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR"
|
||
echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
else
|
||
warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅"
|
||
notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \
|
||
"Partnership" "normal"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi |