#!/bin/bash # ============================================================================================== # ============================= Partnership Manager ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Manages the full lifecycle of a two-server partnership — onboard, daily health # monitoring, offboard, and ownership transfer. Called by partnership_onboard.sh # during initial setup, and by critical_sync_maintenance.sh every 30 minutes for # the --check mode. All other modes are run manually. # # PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer. # AM_OWNER / AM_MIRROR flags (set by detect_hosts) control all routing — # no hostname string comparisons anywhere in this script. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # --onboard (owner only) # Reconfigures mirror auth WebUIs → owner's Tailscale IP # Mirror operator clicks NPM → gets owner's NPM via Tailscale automatically # Writes ACTIVE state on both servers # # --offboard (either server) # Mirror-initiated: reconfigures own WebUIs → localhost, writes INACTIVE state # Owner finalises on next --check: final sync, Tailscale removal # Owner-initiated: final sync, reconfigures mirror WebUIs → localhost, removes # mirror containers + appdata, SSH key revocation, Tailscale removal # Both leave with clean state ✅ # # --transfer (owner only) # Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf # Requires confirmation string + consecutive health check passes # # --check (called every 30min by critical_sync_maintenance.sh) # --remote-seen: rsync succeeded → reset offline counter, read remote state # --remote-unseen: rsync failed → increment offline counter → auto-offboard at threshold # Silent when healthy — never noisy on clean runs # # --status (either server) # Show state files from both servers, blocklist, SSH key status # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Deferred offboard # Either server can offboard without the other being reachable. The initiating # server writes its state immediately and becomes independent. The other server # reads the INACTIVE state on its next --check and finalises automatically. # No message passing, no coordination required. # # Appdata cleanup on offboard # Partner containers are stopped and removed. Their appdata bind-mount paths # (collected via docker inspect before removal) are also deleted. Safety gate: # only paths matching /mnt/*/appdata* are deleted — media and config shares # outside the appdata tree are never touched. # # Blocklist is application-layer; Tailscale removal is network-layer # Both happen on offboard. The blocklist prevents re-onboard until explicitly # cleared with --unblock. Tailscale removal ends encrypted access at the network # level. Grace period controls both simultaneously — one var, consistent behaviour. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Root check # All operations require root. # # Role enforcement # Mirror cannot run --onboard or --transfer. Blocked with a clear error message. # # Version parity check # --onboard verifies both servers are on compatible unRAID versions. # # SSH_TIMEOUT on all remote calls # Every ssh/scp call is timeout-protected. No operation hangs on an unreachable peer. # # No acquire_lock — Deliberate # This file is dual-role: an executable dispatcher AND a library that # partnership_offboard.sh and partnership_transfer.sh source with # PARTNERSHIP_LIB_MODE=1. A top-level acquire_lock would fire for every sourcing # script, and --check can re-invoke this file as itself (bash "$0" --offboard), # which a strict lock would deadlock. Concurrency is handled per-write with flock # instead. Do not "fix" this to match the single-role scripts. # # flock on state writes # write_state_file() serialises full state-file rewrites, and the --check counter # updates are flocked separately on their own lock file. The offline counter is a # read-modify-write: without the lock two overlapping --check cycles both read N and # both write N+1, silently losing an increment and pushing the auto-offboard threshold # past its configured window. The last_seen_remote sed is inside the same lock because # it edits a file write_state_file() rewrites wholesale from other paths. # # Note the redirection must sit INSIDE a command substitution — "$( ... ) 201>file" # attaches the descriptor to the assignment rather than the subshell, and flock then # fails with "Bad file descriptor" while the unlocked write proceeds anyway. # # SIGTERM trap on grace period sleep # Offboard grace period is interruptible — Ctrl-C aborts cleanly. # # Silent by default # --check produces no output when both servers are healthy. Only state changes # and threshold crossings produce output. # # ============================================================================================== # STATE FILES # ============================================================================================== # # $STATE_DIR/partnership_.db — each host writes its own, partner reads via SSH # $STATE_DIR/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared # $STATE_DIR/partnership_offline_days.db — cumulative offline day counter # # All in STATE_DIR — survives reboots (on /boot in internal mode, appdata in flash mode). # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # PARTNERSHIP_ENABLED # Global gate — set true once both servers are configured (default: false) # # PARTNERSHIP_OWNER_HOST # "HOST1" or "HOST2" — flips on --transfer (default: "HOST1") # # PARTNERSHIP_REMOVE_TAILSCALE # Remove mirror from tailnet on offboard (default: true) # # PARTNERSHIP_GRACE_HOURS # Hours before Tailscale removal after offboard — backup access also expires then (default: 6) # # PARTNERSHIP_OFFLINE_THRESHOLD # Days of missed sync cycles before auto-offboard triggers (default: 30) # # PARTNERSHIP_TRANSFER_CONFIRM # Exact string required for --transfer (default: "i-understand-this-transfers-ownership") # # PARTNERSHIP_TRANSFER_STRIKES # Consecutive health checks required before transfer proceeds (default: 3) # # PARTNERSHIP_TRANSFER_MAX_ATTEMPTS # Max health check attempts before giving up (default: 20) # # PARTNERSHIP_ONBOARD_VERIFY # Curl-verify each WebUI after reconfigure to confirm Tailscale routing works (default: true) # # PARTNERSHIP_ONBOARD_NOTIFY # Notify both servers on successful onboard (default: true) # # PARTNERSHIP_SYNC_INTERVAL # Informational — actual schedule is in cron (default: 15) # # TAILSCALE_API_KEY / TAILSCALE_TAILNET # Required when PARTNERSHIP_REMOVE_TAILSCALE=true # # host*.conf # # HOST*_PARTNERSHIP_AUTH_WEBUIS # Containers reconfigured on onboard/offboard. Format: "ContainerName|WebUIPort" # Aliased by detect_hosts() → PARTNERSHIP_AUTH_WEBUIS # # HOST*_PARTNERSHIP_MIRROR_BACKUPS # Paths accessible to the partner during grace window after offboard. # Aliased by detect_hosts() → PARTNERSHIP_MIRROR_BACKUPS # # HOST*_PARTNERSHIP_OWN_CONTAINERS # Containers parked here during partnership, restarted on offboard. # Aliased by detect_hosts() → PARTNERSHIP_OWN_CONTAINERS # # HOST*_PARTNERSHIP_SERVICES_STACK # Shared services XMLs (Emby, Jellyfin, Seerr, SeerrFin) deployed on mirror during onboard. # Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # partnership_manager.sh --onboard # Establish mirror relationship — owner only # # partnership_manager.sh --offboard # Clean separation — either server. 10-second countdown before executing. # # partnership_manager.sh --offboard --dry-run # Show the complete offboard sequence without executing # # partnership_manager.sh --transfer --confirm=i-understand-this-transfers-ownership # Flip ownership — owner only. Requires exact confirmation string. # # partnership_manager.sh --check --remote-seen|--remote-unseen # Called by critical_sync_maintenance.sh every 30min — do not run manually # # partnership_manager.sh --status # Show state files, blocklist, SSH key status from both servers # # partnership_manager.sh --unblock # Remove hostname from blocklist to permit re-onboarding # # Any mode supports --dry-run and --log # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh" SSH_TIMEOUT=15 BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}" # ── Parse mode flags before parse_args ──────────────────────────────────────────────────────── MODE="" TRANSFER_CONFIRM_INPUT="" REMOTE_SEEN=false REMOTE_UNSEEN=false REASON="manual" UNBLOCK_HOST="" FILTERED_ARGS=() LOCAL_ONLY=false # --onboard --local-only: skips remote steps, runs HOST1-local setup only for arg in "$@"; do case "$arg" in --onboard) MODE="onboard" ;; --offboard) MODE="offboard" ;; --transfer) MODE="transfer" ;; --check) MODE="check" ;; --status) MODE="status" ;; --unblock) MODE="unblock" ;; --local-only) LOCAL_ONLY=true ;; --confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;; --remote-seen) REMOTE_SEEN=true ;; --remote-unseen) REMOTE_UNSEEN=true ;; --reason=*) REASON="${arg#--reason=}" ;; *) if [[ "$MODE" == "unblock" ]] && [[ -z "$UNBLOCK_HOST" ]] && [[ "$arg" != --* ]]; then UNBLOCK_HOST="$arg" else FILTERED_ARGS+=("$arg") fi ;; esac done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi # detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS detect_hosts # ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ─────────────────────────────────────── partnership_resolve_roles # State files LOCAL_STATE_FILE="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db" REMOTE_STATE_FILE="${STATE_DIR}/partnership_${REMOTE_SERVER_NAME}.db" OWNER_STATE_FILE="${STATE_DIR}/partnership_${OWNER}.db" MIRROR_STATE_FILE="${STATE_DIR}/partnership_${MIRROR}.db" OFFLINE_COUNTER="${STATE_DIR}/partnership_offline_days.db" # ── Exit Trap — restart locally stopped containers if script crashes mid-cleanup ────────────── # Used by cleanup_partner_containers() — also shared with partnership_offboard.sh which # sources this file and registers the same trap. declare -a _PM_TRAP_STOPPED=() _pm_trap_restart_stopped() { [[ ${#_PM_TRAP_STOPPED[@]} -eq 0 ]] && return for c in "${_PM_TRAP_STOPPED[@]}"; do [[ -z "$c" ]] && continue if docker inspect "$c" >/dev/null 2>&1; then warn "Exit trap: restarting $c (stopped but not removed)" docker start "$c" >/dev/null 2>&1 || warn " Failed to restart $c" fi done } if [[ "${PARTNERSHIP_LIB_MODE:-}" != "1" ]]; then if [[ -z "$MODE" ]]; then error "No mode specified" echo "Usage:" echo " partnership_manager.sh --onboard" echo " partnership_manager.sh --offboard" echo " partnership_manager.sh --transfer --confirm=..." echo " partnership_manager.sh --check --remote-seen|--remote-unseen" echo " partnership_manager.sh --status" echo " partnership_manager.sh --unblock " exit 1 fi # Role-based access control if [[ "$AM_MIRROR" == true ]]; then case "$MODE" in onboard|transfer) error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE" error "Run from $OWNER or use --offboard to separate cleanly" exit 1 ;; esac fi # Lock for all modes except --check (frequent) and --offboard (offboard script holds its own) [[ "$MODE" != "check" && "$MODE" != "offboard" ]] && acquire_lock "strict" trap _pm_trap_restart_stopped EXIT [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" fi # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== read_state_file() { local file="$1" key="$2" grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2 } write_state_file() { local file="$1" shift # flock prevents concurrent writes to the same state file ( flock -x 200 cat > "$file" << EOF state=${1:-UNKNOWN} owner=${OWNER} mirror=${MIRROR} onboarded=${2:-} offboarded=${3:-} triggered_by=${4:-} reason=${5:-} last_seen_remote=${6:-} updated=$(date '+%Y-%m-%d %H:%M:%S') EOF ) 200>"${file}.lock" } # Deliver the partnership state to the partner, at the path the partner actually reads, under # both names its UI looks for. # # This used to be one scp to "root@ip:$local_file" — the LOCAL absolute path reused verbatim on # the remote. That only works while both hosts install to the same place. HOST1 is on flash at # /boot/config/plugins/varaverk and HOST2 is in appdata mode at /mnt/user/appdata/Varaverk, so # the copy went to a directory HOST2 does not read and, more often, does not have — and the # failure surfaced as "will propagate on next sync", which nothing does. # # Both names, because the page resolves one file per node: partnership_.db. # The mirror needs partnership_.db for its own card and partnership_.db for the # owner's. Nothing on the mirror writes the first one during an owner-driven onboard, which is # why a mirror that was fully onboarded still rendered as having no partnership at all. # The content is symmetric — state/owner/mirror/onboarded — so one file serves as both. push_state_to_remote() { local local_file="$1" remote_ip="$2" ssh_key="$3" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push state file to remote" return 0 fi # Resolve where the partner keeps its state, from the partner. varaverk.cfg names its # SCRIPTS_DIR; absent it, the flash default is the right guess for a stock install. local remote_sd remote_state_dir remote_sd=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ -o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \ 'grep -oP "(?<=SCRIPTS_DIR=\")[^\"]+" /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' \ 2>/dev/null | tr -d '\r') [[ -z "$remote_sd" ]] && remote_sd="/boot/config/plugins/varaverk" remote_state_dir="${remote_sd}/data/state" local rc=0 local name for name in "partnership_${REMOTE_SERVER_NAME}.db" "partnership_${LOCAL_SERVER_NAME}.db"; do timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \ -o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \ "mkdir -p '$remote_state_dir' && cat > '${remote_state_dir}/${name}'" \ < "$local_file" 2>/dev/null || rc=1 done if [[ $rc -eq 0 ]]; then echo "State file pushed to $REMOTE_SERVER_NAME:${remote_state_dir} ✅" return 0 fi warn "Could not push state to $REMOTE_SERVER_NAME:${remote_state_dir} — it will keep showing" warn " no partnership until this succeeds. Nothing retries this on a schedule." return 1 } read_remote_state() { local remote_ip="$1" ssh_key="$2" remote_file="$3" timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "cat '$remote_file' 2>/dev/null" 2>/dev/null } get_tailscale_device_id() { local hostname="$1" curl -sf --max-time 10 \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \ 2>/dev/null | \ grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \ grep -o '"id":"[^"]*"' | \ grep -o '[^"]*"$' | tr -d '"' } remove_tailscale_device() { local hostname="$1" if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal" return 1 fi log "Looking up Tailscale device ID for $hostname..." local device_id device_id=$(get_tailscale_device_id "$hostname") if [[ -z "$device_id" ]]; then warn "$hostname not found in Tailscale — may already be removed" return 0 fi log "Removing $hostname (device $device_id) from Tailscale..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $hostname from Tailscale tailnet" return 0 fi curl -sf --max-time 10 -X DELETE \ -H "Authorization: Bearer $TAILSCALE_API_KEY" \ "https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \ warn "$hostname removed from Tailscale ✅" || \ error "Failed to remove $hostname from Tailscale — remove manually" } # ── Blocklist helpers ───────────────────────────────────────────────────────────────────────── is_blocklisted() { local hostname="$1" [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${hostname}|" "$BLOCKLIST_FILE" 2>/dev/null } add_to_blocklist() { local hostname="$1" reason="${2:-offboard}" local now now=$(date '+%Y-%m-%d %H:%M:%S') if ! is_blocklisted "$hostname"; then echo "${hostname}|${now}|${reason}" >> "$BLOCKLIST_FILE" log "Blocklisted: $hostname (reason: $reason)" else log "$hostname already on blocklist" fi } remove_from_blocklist() { local hostname="$1" if [[ ! -f "$BLOCKLIST_FILE" ]]; then log "Blocklist empty — nothing to remove" return 0 fi sed -i "/^${hostname}|/d" "$BLOCKLIST_FILE" 2>/dev/null log "Removed from blocklist: $hostname" } # Revoke SSH access on both sides — call after all other SSH operations complete. # Matches by key comment (format: keyname@hostname — set by ssh_setup.sh at keygen time). # SSH_REVOKE_REMOTE_OK / SSH_REVOKE_LOCAL_OK set in caller scope for summary display. do_ssh_key_revocation() { local remote_ip="$1" local pub_key_file="${SSH_KEY}.pub" echo "" echo "━━━ $ICON_SHIELD SSH Key Revocation ━━━" # Step 1: remove our pubkey from remote's authorized_keys while SSH still works SSH_REVOKE_REMOTE_OK=false if [[ -f "$pub_key_file" ]]; then local our_comment our_comment=$(awk '{print $3}' "$pub_key_file" 2>/dev/null) if [[ -n "$our_comment" ]]; then log "Revoking our pubkey ($our_comment) from $REMOTE_SERVER_NAME..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove our pubkey from $REMOTE_SERVER_NAME authorized_keys" SSH_REVOKE_REMOTE_OK=true elif [[ -n "$remote_ip" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ "grep -v '${our_comment}' /root/.ssh/authorized_keys \ > /root/.ssh/authorized_keys.tmp 2>/dev/null \ && mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \ && chmod 600 /root/.ssh/authorized_keys \ && echo removed" 2>/dev/null | grep -q removed; then echo "Our pubkey revoked from $REMOTE_SERVER_NAME ✅" SSH_REVOKE_REMOTE_OK=true else warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:" warn " grep -v '@${LOCAL_SERVER_NAME}' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys" fi else warn "Could not read pubkey comment from $pub_key_file — skipping remote revocation" fi else warn "Pubkey not found at $pub_key_file — skipping remote revocation" fi # Step 2: remove remote's pubkey from our local authorized_keys # Remote key comment ends with @REMOTE_SERVER_NAME — unique, no special chars SSH_REVOKE_LOCAL_OK=false log "Revoking $REMOTE_SERVER_NAME pubkey from local authorized_keys..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $REMOTE_SERVER_NAME pubkey from local authorized_keys" SSH_REVOKE_LOCAL_OK=true elif [[ -f /root/.ssh/authorized_keys ]]; then if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then # chmod after the mv, every time. `>` creates the temp file under the shell's umask # and `mv` keeps the NEW file's mode, so this rewrite left authorized_keys 0666 on a # filesystem that permits it. sshd's StrictModes then refuses every key in it without # saying so to the client — the key is present, byte-correct, and inert, and the next # onboard's SSH step fails with nothing in any Varaverk log to explain it. Only # /var/log/syslog knows: "Authentication refused: bad ownership or modes". if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \ > /root/.ssh/authorized_keys.tmp 2>/dev/null && \ mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys && \ chmod 600 /root/.ssh/authorized_keys; then echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅" SSH_REVOKE_LOCAL_OK=true else warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually" fi else log "$REMOTE_SERVER_NAME pubkey not found in local authorized_keys — already removed" SSH_REVOKE_LOCAL_OK=true fi else log "/root/.ssh/authorized_keys not found — nothing to remove locally" SSH_REVOKE_LOCAL_OK=true fi } # Gather all partner fallback containers for this server (all tiers) gather_partner_fallback_containers() { local out_var="$1" eval "${out_var}=()" local tier var for tier in TIER1 TIER2 TIER3 TIER4; do var="FALLBACK_${REMOTE_ID}_${tier}" # Check if var is set and is an array if declare -p "$var" >/dev/null 2>&1; then local -a _tmp_arr eval "_tmp_arr=(\"\${${var}[@]}\")" for c in "${_tmp_arr[@]}"; do [[ -n "$c" ]] && eval "${out_var}+=(\"\$c\")" done fi done } # read_remote_conf_var() / read_remote_conf_array() — provided by common.sh # derive_short_name() — provided by common.sh # Start this server's own parked containers after partnership ends. start_own_stack() { # Returns non-zero if any container failed. It used to return whatever the loop's last # docker start happened to produce, so a caller checking it learned nothing — and the # offboard summary just printed "Step 6 — Own stack: started" either way. local _rc=0 echo "" echo "━━━ $ICON_START Restart Own Stack ━━━" if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then log "No PARTNERSHIP_OWN_CONTAINERS configured — skipping" return 0 fi for container in "${PARTNERSHIP_OWN_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start: $container" continue fi if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then echo "$container started ✅" else warn "$container failed to start — check manually" _rc=1 fi done return "$_rc" } # Remove partnership containers on this server + their appdata bind-mount paths. # Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers. # Safety gate: only paths matching /mnt/*/appdata* are deleted. cleanup_partner_containers() { # Returns non-zero if any container or appdata path could not be removed. Previously the # exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅" # was printed over a container that failed to remove. local _rc=0 declare -a containers=() gather_partner_fallback_containers containers if [[ ${#containers[@]} -eq 0 ]]; then log "No partner containers found to remove" return 0 fi # Collect appdata paths BEFORE any removal — inspect fails once container is gone local all_appdata_paths="" for container in "${containers[@]}"; do [[ -z "$container" ]] && continue if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then local cpaths cpaths=$(docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \ "$container" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata') [[ -n "$cpaths" ]] && all_appdata_paths+=$'\n'"$cpaths" fi done # Remove containers for container in "${containers[@]}"; do [[ -z "$container" ]] && continue if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would stop + rm: $container" continue fi if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true _PM_TRAP_STOPPED+=("$container") if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then echo "$container removed ✅" else warn "$container rm failed" _rc=1 fi else log "$container not found — skipping" fi done # Delete appdata after containers are gone while IFS= read -r path; do [[ -z "$path" ]] && continue if [[ "$DRY_RUN" == true ]]; then warn " DRY RUN — would rm -rf $path" continue fi if rm -rf "$path"; then echo " Appdata removed: $path ✅" else warn " Failed to remove: $path" _rc=1 fi done <<< "$all_appdata_paths" return "$_rc" } # SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers) # and their appdata bind-mount paths. # Appdata paths collected via SSH docker inspect before removal, then deleted via SSH. # Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote. cleanup_owner_containers_on_mirror() { # Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly # rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the # containers on it are gone. local _rc=0 local mirror_ip="$1" local owner_short owner_short=$(derive_short_name "$OWNER") log "Removing owner-deployed containers from $MIRROR..." if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove *-${owner_short} containers + appdata from $MIRROR" return 0 fi local container_list container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null) if [[ -z "$container_list" ]]; then log "No *-${owner_short} containers found on $MIRROR" return 0 fi while IFS= read -r container; do [[ -z "$container" ]] && continue # Collect appdata paths before removal local appdata_paths appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \ | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null) if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker stop '$container' >/dev/null 2>&1 docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \ grep -q removed; then echo "$container removed from $MIRROR ✅" else warn "Failed to remove $container from $MIRROR" _rc=1 fi # Delete appdata on remote after container removal while IFS= read -r path; do [[ -z "$path" ]] && continue if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then echo " Appdata removed on $MIRROR: $path ✅" else warn " Failed to remove appdata on $MIRROR: $path" _rc=1 fi done <<< "$appdata_paths" done <<< "$container_list" return "$_rc" } # SSH to mirror — start mirror's own parked containers. # Reads PARTNERSHIP_OWN_CONTAINERS from the mirror's own conf via SSH. start_mirror_own_stack() { local mirror_ip="$1" log "Reading own stack list from $MIRROR conf..." local -a mirror_own=() mapfile -t mirror_own < <(read_remote_conf_array "$mirror_ip" "PARTNERSHIP_OWN_CONTAINERS") # Remove empty entries local -a filtered=() for c in "${mirror_own[@]}"; do [[ -n "$c" ]] && filtered+=("$c"); done mirror_own=("${filtered[@]}") if [[ ${#mirror_own[@]} -eq 0 ]]; then log "No PARTNERSHIP_OWN_CONTAINERS configured on $MIRROR — skipping remote stack restart" return 0 fi log "Restarting own stack on $MIRROR: ${mirror_own[*]}" local _rc=0 for container in "${mirror_own[@]}"; do if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start $container on $MIRROR" continue fi if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \ grep -q started; then echo "$container started on $MIRROR ✅" else warn "$container failed to start on $MIRROR — check manually" _rc=1 fi done return "$_rc" } # Create the mirror's Emby admin account on the owner's deployed Emby. # Credentials come from the MIRROR's conf (HOST*_PARTNERSHIP_EMBY_ADMIN_USER/PASS). # Checks for username collision before creating — exits with guidance if taken. provision_emby_admin() { local mirror_ip="$1" local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" local emby_url="http://${mirror_ip}:${emby_port}" echo "" echo "━━━ $ICON_EMBY Emby Admin Provisioning ━━━" if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" return 0 fi if [[ -z "$EMBY_API_KEY" ]]; then warn "EMBY_API_KEY not set — skipping Emby admin provisioning" return 1 fi # Read mirror's desired credentials from their own conf via SSH log "Reading Emby credentials from $MIRROR conf..." local username password username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER") password=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_PASS") [[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")" if [[ -z "$password" ]]; then warn "PARTNERSHIP_EMBY_ADMIN_PASS is empty in $MIRROR conf" warn "$MIRROR must set HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_PASS before onboard" return 1 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would create Emby admin '$username' at $emby_url" return 0 fi # Collision check — username already exists? local existing_users existing_users=$(curl -sf --max-time 15 \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users" 2>/dev/null) if echo "$existing_users" | grep -q "\"Name\":\"${username}\""; then error "Emby username '${username}' is already taken on the shared instance" error "Options:" error " 1. Sign in with that account — it may already be yours" error " 2. Set a different name in HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_USER and re-run --onboard" return 1 fi log "Creating Emby admin '$username' at $emby_url..." local create_response http_code body create_response=$(curl -sf --max-time 15 -w "\n%{http_code}" \ -X POST \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ -H "Content-Type: application/json" \ -d "{\"Name\": \"$username\"}" \ "${emby_url}/Users/New" 2>/dev/null) http_code=$(echo "$create_response" | tail -1) body=$(echo "$create_response" | head -n -1) if [[ "$http_code" != "200" ]] && [[ "$http_code" != "204" ]]; then warn "Failed to create Emby user '$username' (HTTP $http_code)" return 1 fi local user_id user_id=$(echo "$body" | grep -o '"Id":"[^"]*"' | head -1 | cut -d'"' -f4) if [[ -z "$user_id" ]]; then warn "Emby user created but could not parse user ID — set password manually" return 1 fi local pw_code pw_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ -X POST \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ -H "Content-Type: application/json" \ -d "{\"NewPw\": \"$password\"}" \ "${emby_url}/Users/${user_id}/Password" 2>/dev/null) if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then echo "Emby admin '$username' created (id: $user_id) ✅" else warn "User created but password set failed (HTTP $pw_code) — set password manually" fi curl -sf --max-time 15 -o /dev/null \ -X POST \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ -H "Content-Type: application/json" \ -d '{"IsAdministrator": true, "IsDisabled": false}' \ "${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \ echo "$username granted admin policy ✅" || \ warn "Could not set admin policy — grant manually in Emby dashboard" } # Delete the mirror's Emby admin account from the owner's deployed Emby. revoke_emby_admin() { local mirror_ip="$1" local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}" local emby_url="http://${mirror_ip}:${emby_port}" echo "" echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━" local username username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER") [[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")" if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping" return 0 fi if [[ -z "$EMBY_API_KEY" ]]; then warn "EMBY_API_KEY not set — skipping Emby admin revocation" return 1 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would delete Emby admin '$username' at $emby_url" return 0 fi log "Looking up Emby user '$username' at $emby_url..." local users_json user_id users_json=$(curl -sf --max-time 15 \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users" 2>/dev/null) user_id=$(echo "$users_json" | \ grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \ grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1) if [[ -z "$user_id" ]]; then warn "Emby user '$username' not found at $emby_url — may already be removed" return 0 fi local del_code del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \ -X DELETE \ -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \ "${emby_url}/Users/${user_id}" 2>/dev/null) if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then echo "Emby admin '$username' removed ✅" else warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually" fi } check_both_healthy() { platform_storage_healthy || { error "Local array not healthy"; return 1; } local mirror_ip mirror_ip=$(resolve_tailscale_ip "$MIRROR") [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "mountpoint -q '$REMOTE_STORAGE_PATH' && timeout 10 docker ps" >/dev/null 2>&1 || { error "Mirror $MIRROR not healthy" return 1 } return 0 } # Returns non-zero when the mirror did NOT leave with current state, so the caller can say so. # The whole point of this step is the guarantee in the summary — "mirror leaves with current # Critical-Data" — and it used to print that unconditionally at the end of the function. An # offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs" # immediately followed by "Final sync complete — mirror has current state ✅", and the summary # scored Step 2 as a pass. The mirror left with whatever it happened to have. do_final_sync() { log "Running final critical sync..." local _synced=0 _failed=0 if [[ "$DRY_RUN" == false ]]; then if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then for _share in "${CRITICAL_SYNC_SHARES[@]}"; do [[ -z "$_share" ]] && continue local _path="${_share%%|*}" local _profile="${_share##*|}" if [[ "$_path" == "$_profile" ]]; then bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log else bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \ --profile="$_profile" --log fi # rsync.sh exits 0 both on a real sync and on a clean Tier-1 gate exit, so the # gate is checked here rather than inferred from its status. if [[ "${RSYNC_ENABLED:-true}" == true ]]; then (( _synced++ )); else (( _failed++ )); fi done else warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)" _failed=1 fi else # ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution", # so this line aborted Step 2 of every offboard --dry-run with a shell error instead of # printing. A count of an unset array is already 0, which is the only default needed. warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)" return 0 fi if [[ "$_failed" -gt 0 ]]; then warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had" [[ "${RSYNC_ENABLED:-true}" != true ]] && \ warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent" return 1 fi warn "Final sync complete — mirror has current state ✅" return 0 } # Safe master.conf modification with error handling — appends the key if not already present, # since sed -i returns 0 whether or not it matched anything. update_master_conf() { local key="$1" value="$2" local conf="$CONF_DIR/master.conf" if [[ ! -f "$conf" ]]; then error "master.conf not found at $conf" return 1 fi if grep -q "^[[:space:]]*${key}=" "$conf" 2>/dev/null; then if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then echo "master.conf updated: ${key}=${value}" return 0 else error "Failed to update master.conf: ${key}=${value}" return 1 fi else if echo " ${key}=${value}" >> "$conf" 2>/dev/null; then echo "master.conf updated: ${key}=${value} (appended)" return 0 else error "Failed to append to master.conf: ${key}=${value}" return 1 fi fi } # ── Library-mode guard — source only, skip all mode dispatch ───────────────────────────────── # partnership_offboard.sh sources this file with PARTNERSHIP_LIB_MODE=1 to get helper # functions without triggering any mode execution. [[ "${PARTNERSHIP_LIB_MODE:-}" == "1" ]] && { return 0 2>/dev/null || exit 0; } # ============================================================================================== # ━━━ Unblock ━━━ # ============================================================================================== if [[ "$MODE" == "unblock" ]]; then if [[ -z "$UNBLOCK_HOST" ]]; then error "Usage: partnership_manager.sh --unblock " error "Example: partnership_manager.sh --unblock unRAID-Jayred365" if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then echo "" echo "Currently blocklisted:" while IFS='|' read -r host ts reason; do echo " $host (blocked $ts — $reason)" done < "$BLOCKLIST_FILE" else echo "Blocklist is empty." fi exit 1 fi if ! is_blocklisted "$UNBLOCK_HOST"; then warn "$UNBLOCK_HOST is not on the blocklist" exit 0 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove $UNBLOCK_HOST from blocklist" exit 0 fi remove_from_blocklist "$UNBLOCK_HOST" warn "$UNBLOCK_HOST unblocked — re-onboarding is now permitted ✅" exit 0 fi # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$MODE" == "status" ]]; then echo "" echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━" OWNER_IP=$(resolve_tailscale_ip "$OWNER" || echo "unreachable") MIRROR_IP=$(resolve_tailscale_ip "$MIRROR" || echo "unreachable") echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)" echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)" echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)" echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " Enabled: ${PARTNERSHIP_ENABLED:-false}" echo "" # Local state if [[ -f "$LOCAL_STATE_FILE" ]]; then LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded") LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded") LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote") echo " Local state: $LOCAL_STATE" [[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED" [[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED" [[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN" else echo " Local state: no state file found" fi echo "" # Remote state REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -n "$REMOTE_IP" ]]; then REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -n "$REMOTE_CONTENT" ]]; then REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2) echo " Remote state: $REMOTE_STATE" [[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED" if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then echo "" echo " ✅ Both servers agree: $LOCAL_STATE" else echo "" echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE" fi else echo " Remote state: not found on $REMOTE_SERVER_NAME" fi else echo " Remote state: $REMOTE_SERVER_NAME unreachable" fi echo "" echo " Auth WebUIs:" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue echo " ${entry%%|*} → port ${entry##*|}" done if [[ -f "$OFFLINE_COUNTER" ]]; then OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) [[ "$OFFLINE_DAYS" -gt 0 ]] && \ echo "" && \ echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}" fi echo "" if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then echo " Blocklist:" while IFS='|' read -r host ts reason; do echo " ⛔ $host (blocked $ts — $reason)" done < "$BLOCKLIST_FILE" else echo " Blocklist: empty" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Check ━━━ # Called every 30min by critical_sync_maintenance.sh — must be silent when healthy # ============================================================================================== if [[ "$MODE" == "check" ]]; then # Update last_seen_remote and offline counter based on rsync outcome. # # Both branches are flocked. The counter is a read-modify-write, so two overlapping # --check cycles would otherwise both read N and both write N+1 — silently losing an # increment and pushing the auto-offboard threshold further out than configured. The # sed on the state file is included because write_state_file() flocks the same file # from other code paths, and an unsynchronised sed -i can land mid-rewrite. if [[ "$REMOTE_SEEN" == true ]]; then ( flock -x 201 echo "0" > "$OFFLINE_COUNTER" if [[ -f "$LOCAL_STATE_FILE" ]]; then sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \ "$LOCAL_STATE_FILE" 2>/dev/null fi ) 201>"${OFFLINE_COUNTER}.lock" elif [[ "$REMOTE_UNSEEN" == true ]]; then # Redirection must live INSIDE the substitution — "$( ... ) 201>file" attaches the fd # to the assignment, not to the subshell doing the work, and flock then fails with # "Bad file descriptor" while the increment silently proceeds unlocked. OFFLINE_COUNT=$( { flock -x 201 _c=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) _c=$(( _c + 1 )) echo "$_c" > "$OFFLINE_COUNTER" echo "$_c" } 201>"${OFFLINE_COUNTER}.lock" ) # Auto-offboard threshold: threshold_days × 48 intervals/day (every 30min) THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 48 )) if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard" notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \ "Partnership" "warning" bash "$0" --offboard --reason=auto-offboard-timeout exit 0 fi log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)" fi # Skip blocklisted partners — they can't auto-reconnect; only --onboard re-establishes if is_blocklisted "$REMOTE_SERVER_NAME"; then log "Partnership check — $REMOTE_SERVER_NAME is blocklisted, skipping" exit 0 fi # Read remote state file REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME") if [[ -z "$REMOTE_IP" ]]; then log "Partnership check — remote unreachable, skipping state check" exit 0 fi REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") if [[ -z "$REMOTE_CONTENT" ]]; then # IP resolved but SSH returned nothing — could be auth failure, not just missing file if [[ -f "$SCRIPT_DIR/ssh_setup.sh" ]]; then bash "$SCRIPT_DIR/ssh_setup.sh" --validate 2>/dev/null || true fi log "Partnership check — remote state file not found" exit 0 fi REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN") # Both agree and active — healthy, silent if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then echo "Partnership check — ACTIVE, both servers agree ✅" exit 0 fi # Remote requested offboard — run full offboard in background so cron is not blocked if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then warn "Partnership check — $REMOTE_SERVER_NAME requested offboard" if [[ "$AM_OWNER" == true ]]; then warn "Owner finalising offboard — spawning partnership_offboard.sh in background" nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \ --reason=mirror-requested \ >> /var/log/partnership_offboard.log 2>&1 & warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow" else warn "Owner offboarded — spawning mirror-side cleanup in background" nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \ --reason=owner-offboarded \ >> /var/log/partnership_offboard.log 2>&1 & warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow" fi exit 0 fi # Both inactive — nothing to do if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then echo "Partnership check — INACTIVE on both servers" exit 0 fi log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE" exit 0 fi # ============================================================================================== # ━━━ Onboard ━━━ # ============================================================================================== if [[ "$MODE" == "onboard" ]]; then echo "" echo "━━━ $ICON_FALLBACK Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" # ── LOCAL-ONLY PATH ────────────────────────────────────────────────────────── # HOST1-local setup steps that don't need HOST2 present. Called from # partnership_onboard.sh --phase1-only so HOST1 can complete its own side # (PARTNERSHIP_ENABLED flag, setup.db) while waiting for HOST2 to install and onboard. if [[ "$LOCAL_ONLY" == true ]]; then log "Mode: local-only — skipping remote pre-flight and WebUI steps" log "Owner: $OWNER_ID ($OWNER) · Mirror: $MIRROR_ID ($MIRROR)" echo "" # Enable partnership in master.conf + push to all hosts echo "" echo "Enabling partnership in master.conf..." if [[ "$DRY_RUN" == false ]]; then update_master_conf "PARTNERSHIP_ENABLED" "true" platform_push_conf | while IFS= read -r line; do log "$line"; done else warn "DRY RUN — would set PARTNERSHIP_ENABLED=true in master.conf and push" fi # Write HOST1_LOCAL_DONE flag to setup.db local_state_file="$(platform_setup_db_path)" if [[ "$DRY_RUN" == false ]]; then flag_key="${MY_ID}_LOCAL_DONE" if grep -q "^${flag_key}=" "$local_state_file" 2>/dev/null; then sed -i "s|^${flag_key}=.*|${flag_key}=true|" "$local_state_file" else echo "${flag_key}=true" >> "$local_state_file" fi platform_push_setup_state echo "${MY_ID}_LOCAL_DONE=true written to setup.db ✅" else warn "DRY RUN — would write ${MY_ID}_LOCAL_DONE=true" fi echo "" echo "HOST1 local setup complete. Waiting for $MIRROR_ID ($MIRROR) to onboard." exit 0 fi # ── FULL ONBOARD PATH (requires HOST2) ──────────────────────────────────── # Check already onboarded if [[ -f "$LOCAL_STATE_FILE" ]]; then CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then warn "Partnership already ACTIVE — use --status for detail or --offboard to separate" exit 0 fi fi # Warn if re-onboarding a previously blocked partner — onboard is deliberate so it proceeds if is_blocklisted "$MIRROR"; then warn "$MIRROR is on the blocklist from a previous offboard" warn "Proceeding — blocklist will be cleared on successful onboard" fi log "Owner: $OWNER_ID ($OWNER)" log "Mirror: $MIRROR_ID ($MIRROR)" # Pre-flight echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" resolve_remote_ip check_connectivity # Version parity — both servers must agree on unRAID version check_os_version_parity || exit 1 # Remote array and Docker daemon check_remote_array || exit 1 check_remote_docker_daemon || exit 1 OWNER_IP=$(resolve_tailscale_ip "$OWNER") MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") [[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; } [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; } log "Owner IP: $OWNER_IP" log "Mirror IP: $MIRROR_IP" # Reconfigure mirror WebUIs → owner IP echo "" echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$OWNER_IP" \ "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ )) done # Verify WebUI connectivity if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then echo "" echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━" for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then echo "$container reachable at http://${OWNER_IP}:${port}/ ✅" else warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running" fi done fi # Write state files echo "" echo "━━━ $ICON_GEAR Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') if [[ "$DRY_RUN" == false ]]; then write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" echo "Local state: ACTIVE ✅" remove_from_blocklist "$MIRROR" push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" echo "0" > "$OFFLINE_COUNTER" else warn "DRY RUN — would write ACTIVE state and push to remote" fi # Emby admin provisioning — runs after container deployment provision_emby_admin "$MIRROR_IP" # Summary echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" echo " Owner: $OWNER_ID ($OWNER_IP)" echo " Mirror: $MIRROR_ID ($MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅" [[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \ notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \ "Partnership" "normal" else warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually" notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \ "Partnership" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Offboard ━━━ # ============================================================================================== if [[ "$MODE" == "offboard" ]]; then exec bash "$SCRIPT_DIR/partnership_offboard.sh" "${FILTERED_ARGS[@]}" --reason="$REASON" fi # ============================================================================================== # ━━━ Transfer ━━━ # ============================================================================================== if [[ "$MODE" == "transfer" ]]; then exec bash "$SCRIPT_DIR/partnership_transfer.sh" \ ${TRANSFER_CONFIRM_INPUT:+--confirm="$TRANSFER_CONFIRM_INPUT"} \ "${FILTERED_ARGS[@]}" fi # (legacy inline transfer block — kept below for reference, unreachable) if false; then echo "" echo "━━━ $ICON_FALLBACK Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "⚠️ WARNING — OWNERSHIP TRANSFER" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " Current owner: $OWNER_ID ($OWNER)" echo " Current mirror: $MIRROR_ID ($MIRROR)" echo "" echo " After transfer:" echo " New owner: $MIRROR_ID ($MIRROR)" echo " New mirror: $OWNER_ID ($OWNER)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Confirmation check if [[ "$DRY_RUN" == false ]]; then if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then echo "" echo "To proceed pass exactly:" echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}" echo "" error "Transfer cancelled — confirmation required" exit 1 fi if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then error "Confirmation string does not match — transfer cancelled" exit 1 fi log "Confirmation accepted" else warn "DRY RUN — confirmation check skipped" fi # Health strike system echo "" echo "━━━ $ICON_SHIELD Health Verification ━━━" log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" STRIKES=0 ATTEMPTS=0 MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}" while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do (( ATTEMPTS++ )) if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable" error "Transfer cancelled — try again when both servers are healthy" exit 1 fi if check_both_healthy; then (( STRIKES++ )) log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10 else warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)" STRIKES=0 sleep 30 fi done warn "Both servers healthy — proceeding with transfer" NEW_OWNER_ID="$MIRROR_ID" NEW_MIRROR_ID="$OWNER_ID" NEW_OWNER="$MIRROR" NEW_MIRROR="$OWNER" NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY" NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY" NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}" NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}" NEW_OWNER_IP=$(resolve_tailscale_ip "$NEW_OWNER") NEW_MIRROR_IP=$(resolve_tailscale_ip "$NEW_MIRROR") [[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; } [[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; } # Final sync in current direction echo "" echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━" do_final_sync # Reconfigure new mirror WebUIs → new owner echo "" echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━" log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)" WEBUI_FAILURES=0 for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do [[ -z "$entry" ]] && continue container="${entry%%|*}" port="${entry##*|}" reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \ "$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ )) done # New owner WebUIs → localhost (now manages directly) log "New owner ($NEW_OWNER) WebUIs → localhost" reconfigure_local_webuis "localhost" # Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers echo "" echo "━━━ $ICON_GEAR Update Ownership ━━━" if [[ "$DRY_RUN" == false ]]; then update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\"" # Update remote master.conf timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$NEW_MIRROR_IP" \ "sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \ '$SCRIPT_DIR/../Configurations/master.conf'" 2>/dev/null && \ echo "Remote master.conf updated ✅" || \ error "Failed to update remote master.conf — update manually" else warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers" fi # Write state files NOW=$(date '+%Y-%m-%d %H:%M:%S') write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer" push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY" # Summary echo "" echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━" echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)" echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)" echo " WebUI failures: $WEBUI_FAILURES" echo " Sync direction: $NEW_OWNER → $NEW_MIRROR" echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" else warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅" notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \ "Partnership" "normal" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi