Files
Varaverk/Partnership/partnership_onboard.sh
T
Gmer4Lfe 4f95cc6d13 Deliver master.conf to a node that has no Varaverk on it yet, which is the node that needs it
Phase 1 exists to hand the partner its identity before the install, but the push refused any
host without varaverk.cfg — so the conf could only ever reach a node that no longer needed it
to be told who it was. It now resolves the remote's conf directory across both install layouts
and creates the internal one when neither exists.
2026-08-16 16:14:23 -04:00

897 lines
44 KiB
Bash
Executable File

#!/bin/bash
# ==============================================================================================
# ============================= Partnership Onboard ============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Runs once on both servers to establish a new partnership. Role is detected
# automatically via detect_hosts() — no flags needed to declare which side you are.
# Run on the mirror first (generates its SSH key), then on the owner to complete
# setup remotely.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# MIRROR PATH (1 step)
# Step 1: SSH key setup — generate keypair, copy to owner, update conf
# Owner completes the rest remotely. Mirror is done.
#
# OWNER PATH (14 steps)
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
# Step 1b: Docker network — ensure varaverk docker network exists on mirror
# Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent)
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
# Mariadb/Redis health-checked before Authelia deploys
# Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing
# Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror
# Step 6: Stop mirror services — stop mirror's existing services containers before replacing
# Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
# Step 9b: Webhook setup — register download webhook in arrs on both servers
# Step 9c: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed)
# prevents arrs treating every file as missing after bootstrap
# Step 9d: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
# Step 9e: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the
# template ships closed so a fresh node cannot sync early
# Step 10: Conf push — push master.conf + setup state to all listed hosts
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Credentials never in SSH command strings
# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
# locally, SCPed to the remote, and executed there. Command-line args are never used
# to pass credentials — they'd appear in `ps` output and shell history on both servers.
#
# XML templates are the single source of truth for deployed containers
# The owner's templates-user/ XMLs define every container deployed on the mirror.
# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
# Docker Manager can manage the containers after onboard without additional config.
#
# Dependency ordering in the auth stack is owner-enforced
# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
# the script waits for the container to be healthy before continuing. This is a remote
# health check — the container must be running (or report healthy) before the next
# dependent is deployed.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root check
# All operations run as root — SSH key management, docker operations, conf updates.
#
# SSH timeout on all remote calls
# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
# slow or unreachable mirror.
#
# --dry-run shows exact actions without executing
# Every step prints what it would do. SCP, deploy, plugin install, arr sync —
# all dry-run safe.
#
# Step skip flags for partial re-runs
# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
# resuming after a partial failure without re-running completed steps.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
# HOST*_PARTNERSHIP_AUTH_STACK
# XML filenames (from this server's templates-user/) to push and deploy on the
# mirror as its auth stack. Order matters: database deps before Authelia.
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
#
# HOST*_PARTNERSHIP_REPLACE_CONTAINERS
# Containers to stop on the mirror before deploying the auth stack.
# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
#
# HOST*_PARTNERSHIP_ARR_STACK
# XML filenames to push and deploy on the mirror as its arr stack.
# Leave empty to skip arr stack deploy.
# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
#
# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
# Arr containers to stop on the mirror before deploying the arr stack.
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
#
# HOST*_PARTNERSHIP_SERVICES_STACK
# XML filenames to push and deploy on the mirror as its shared services stack.
# Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy.
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
#
# HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS
# Services containers to stop on the mirror before deploying the services stack.
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# Partnership/partnership_onboard.sh
# Full onboard — role detected automatically
#
# Partnership/partnership_onboard.sh --dry-run
# Preview all steps without making changes
#
# Partnership/partnership_onboard.sh --log
# Verbose per-step output
#
# Partnership/partnership_onboard.sh --skip-ssh
# Skip SSH key setup (key already in place)
#
# Partnership/partnership_onboard.sh --skip-share-setup
# Skip share creation on mirror (shares already exist)
#
# Partnership/partnership_onboard.sh --skip-auth-stack
# Skip auth stack stop + deploy (Steps 3-4)
#
# Partnership/partnership_onboard.sh --skip-arr-stack
# Skip arr stack stop + deploy (Steps 4-5)
#
# Partnership/partnership_onboard.sh --skip-services-stack
# Skip services stack stop + deploy (Steps 6-7)
#
# Partnership/partnership_onboard.sh --skip-arr-sync
# Skip arr library bootstrap (Step 9)
#
# Partnership/partnership_onboard.sh --skip-webhook-setup
# Skip webhook registration in arrs (Step 9b)
#
# Partnership/partnership_onboard.sh --skip-media-seed
# Skip initial media share rsync to mirror (Step 9c)
# Use when mirror already has files or you want to seed manually
#
# Partnership/partnership_onboard.sh --skip-webhook-listener
# Skip starting webhook listener on mirror (Step 9d)
# Listener will start automatically on next array restart
#
# Partnership/partnership_onboard.sh --no-arm
# Leave RSYNC_ENABLED / CONF_SYNC_ENABLED / ARR_SYNC_ENABLED as they are (Step 9e).
# Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose
# shares are not populated yet.
#
# Partnership/partnership_onboard.sh --phase1-only
# OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
#
# Partnership/partnership_onboard.sh --phase2-only
# OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
# by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
# Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
SSH_TIMEOUT=15
source "$SCRIPTS_ROOT/load_config.sh"
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
SKIP_SSH=false
SKIP_SHARE_SETUP=false
SKIP_AUTH_STACK=false
SKIP_ARR_STACK=false
SKIP_SERVICES_STACK=false
SKIP_ARR_SYNC=false
SKIP_WEBHOOK_SETUP=false
SKIP_MEDIA_SEED=false
SKIP_WEBHOOK_LISTENER=false
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
SKIP_ARM=false # leave the sync gates as they are — see Step 9e
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--skip-ssh) SKIP_SSH=true ;;
--skip-share-setup) SKIP_SHARE_SETUP=true ;;
--skip-auth-stack) SKIP_AUTH_STACK=true ;;
--skip-arr-stack) SKIP_ARR_STACK=true ;;
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
--skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;;
--skip-media-seed) SKIP_MEDIA_SEED=true ;;
--skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;;
--phase1-only) PHASE1_ONLY=true ;;
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
--no-arm) SKIP_ARM=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_args "${FILTERED_ARGS[@]}"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
acquire_lock
if ! command -v docker &>/dev/null; then
error "Docker command not found"
exit 1
fi
detect_hosts
partnership_resolve_roles
EXTRA_FLAGS=()
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
[[ "$ENABLE_LOGGING" == true ]] && EXTRA_FLAGS+=("--log")
START=$(date +%s)
# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
write_onboard_phase() {
local target_id="$1" phase="$2"
local key="${target_id}_PHASE${phase}_DONE"
local state_file="$(platform_setup_db_path)"
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
set_state_var "$state_file" "$key" "true"
platform_push_setup_state
}
# ── Helper: set a boolean toggle in master.conf, in place ─────────────────────────────────────
# Booleans only. The value pattern stops at whitespace, so a quoted value containing spaces would
# be truncated — every gate this arms is true/false and nothing else should use it.
#
# Preserves what the line already looks like: leading indent, the existing quoting style, and the
# column the trailing comment sits in. master.conf is hand-aligned and read by a person; an edit
# that reflows a line makes a diff that looks bigger than the change actually is.
_arm_conf_var() {
local key="$1" val="$2" file="$3" cur pad spaces quoted
if ! grep -qE "^[[:space:]]*${key}=" "$file"; then
warn " ${key} is not in master.conf — skipped (add it to the template first)"
return 1
fi
cur=$(grep -E "^[[:space:]]*${key}=" "$file" | head -1 | sed -E "s|^[[:space:]]*${key}=||; s|[[:space:]]*#.*$||")
quoted=""; [[ "$cur" == \"*\" ]] && quoted='"'
cur="${cur//\"/}"
[[ "$cur" == "$val" ]] && { echo " ${key} already ${val}"; return 0; }
# Pad only when a comment follows — otherwise a shorter value leaves trailing whitespace.
pad=0
if grep -qE "^[[:space:]]*${key}=[^#]*#" "$file"; then
pad=$(( ${#cur} - ${#val} )); (( pad < 0 )) && pad=0
fi
spaces=$(printf '%*s' "$pad" '')
sed -i -E "s|^([[:space:]]*)${key}=[^#[:space:]]*([[:space:]]*)(#.*)?$|\1${key}=${quoted}${val}${quoted}${spaces}\2\3|" "$file"
echo " ${key}: ${cur}${val}"
}
echo ""
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
# ==============================================================================================
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
#
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
# use the same convention), and reads the named config array from the mirror's own conf.
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
#
# deploy_container_from_xml() already stops/removes containers with the same name as what's
# being deployed. This step handles containers with DIFFERENT names that conflict.
# ==============================================================================================
stop_mirror_stack() {
local config_var="$1" label="$2"
local -a to_stop=()
mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$')
if [[ ${#to_stop[@]} -eq 0 ]]; then
log "No $label containers to stop on $MIRROR — skipping"
return 0
fi
log "Stopping $label on $MIRROR: ${to_stop[*]}"
for container in "${to_stop[@]}"; do
if [[ "$DRY_RUN" == true ]]; then
warn " DRY RUN — would stop + rm $container on $MIRROR"
continue
fi
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"docker stop '$container' 2>/dev/null
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
grep -q removed && \
echo " $container removed ✅" || \
log " $container not found on $MIRROR — skipping"
done
}
# ==============================================================================================
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
if [[ "$AM_MIRROR" == true ]]; then
echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
echo ""
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
echo ""
if [[ "$SKIP_SSH" == true ]]; then
warn "Skipping SSH setup (--skip-ssh)"
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
echo "SSH key ready ✅"
else
error "SSH key setup failed"
exit 1
fi
echo ""
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
echo ""
OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
PHASE2_TRIGGERED=false
if [[ -n "$OWNER_IP" ]]; then
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
PHASE2_TRIGGERED=true
elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
2>/dev/null | grep -q triggered; then
echo "Phase 2 triggered on $OWNER ✅"
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
PHASE2_TRIGGERED=true
else
warn "Could not auto-trigger Phase 2 on $OWNER"
fi
else
warn "Cannot resolve $OWNER Tailscale IP"
fi
echo ""
echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
echo " SSH key: ready"
echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
if [[ "$PHASE2_TRIGGERED" == false ]]; then
echo ""
echo " Run manually on $OWNER:"
echo " bash Partnership/partnership_onboard.sh --phase2-only"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
log "Mirror: $MIRROR ($MIRROR_IP)"
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
echo ""
STEP_SSH_OK=false
STEP_NETWORK_OK=false
STEP_STOP_AUTH_OK=true
STEP_AUTH_OK=true
AUTH_DEPLOYED=0
AUTH_FAILED=0
STEP_STOP_ARR_OK=true
STEP_ARR_OK=true
ARR_DEPLOYED=0
ARR_FAILED=0
STEP_STOP_SERVICES_OK=true
STEP_SERVICES_OK=true
SERVICES_DEPLOYED=0
SERVICES_FAILED=0
ONBOARD_OK=false
ARR_SYNC_OK=false
WEBHOOK_SETUP_OK=false
MEDIA_SEED_OK=false
MEDIA_SEED_COUNT=0
WEBHOOK_LISTENER_OK=false
MASTER_PUSH_OK=false
# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
# Skipped when --phase2-only (SSH was already done in Phase 1).
echo "━━━ Step 1 — SSH Key Setup ━━━"
if [[ "$SKIP_SSH" == true ]]; then
warn "Skipping (--skip-ssh)"
STEP_SSH_OK=true
elif [[ "$PHASE1_ONLY" == true ]]; then
# Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
# hanging for a password prompt with no TTY.
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
echo "SSH to $MIRROR already works ✅ — skipping key install"
STEP_SSH_OK=true
else
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
echo "SSH keys ready ✅"
STEP_SSH_OK=true
else
# Soft-fail: generate key locally if not present, then tell user to install manually
warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
if [[ -f "$SSH_KEY" ]]; then
log "Local key exists at: $SSH_KEY"
else
bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
fi
if [[ -f "${SSH_KEY}.pub" ]]; then
echo ""
echo " Install this key on $MIRROR to complete SSH setup:"
echo " ┌─────────────────────────────────────────────────────"
cat "${SSH_KEY}.pub" | sed 's/^/ │ /'
echo " └─────────────────────────────────────────────────────"
echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
echo " Then click 'Push Conf' in the Partnership tab."
# Write key-ready flag so UI can show the manual-install state
[[ "$DRY_RUN" == false ]] && {
kflag="${MIRROR_ID}_KEY_READY"
_setup_f="$(platform_setup_db_path)"
set_state_var "$_setup_f" "$kflag" "true"
}
fi
STEP_SSH_OK=false
fi
fi
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
echo "SSH keys ready ✅"
STEP_SSH_OK=true
else
error "SSH key setup failed — aborting"
error "Re-run or use --skip-ssh if key is already set up"
exit 1
fi
# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
if [[ "$PHASE1_ONLY" == true ]]; then
if [[ "$STEP_SSH_OK" == false ]]; then
# SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
echo ""
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
warn "Local setup had issues — check partnership_manager.sh output above"
END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
echo " Conf push: skipped (needs SSH access to $MIRROR)"
echo " HOST1 setup: done ✅"
echo " Duration: $(format_duration $(( END - START )))"
echo ""
echo " ACTION NEEDED: install the key on $MIRROR:"
echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
echo " Then click 'Push Conf' in Partnership tab, or run:"
echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
echo ""
echo "━━━ Phase 1 — Conf Push ━━━"
CONF_PUSH_OK=false
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would push master.conf + state file to $MIRROR"
CONF_PUSH_OK=true
else
push_output=$(platform_push_conf)
push_rc=$?
[[ -n "$push_output" ]] && echo "$push_output"
platform_push_setup_state
if [[ $push_rc -eq 0 ]]; then
echo "Conf push complete ✅"
CONF_PUSH_OK=true
else
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
fi
fi
# HOST1 local setup — runs immediately without needing HOST2
echo ""
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
warn "Local setup had issues — check partnership_manager.sh output above"
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
echo " HOST1 setup: done ✅"
echo " Duration: $(format_duration $(( END - START )))"
echo ""
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
# The push no longer waits for a Varaverk install on the far side. It resolves the partner's
# conf directory across both layouts and creates the internal one if neither exists, so the
# conf arrives BEFORE the plugin — which is the order that makes it useful. The .plg only
# seeds master.conf from the template when none is present, so what lands here survives the
# install and the wizard reads its identity straight out of it.
if [[ "$CONF_PUSH_OK" == true ]]; then
echo " master.conf is on $MIRROR — the wizard will find it and take the partner path,"
echo " already knowing $MY_ID and $MIRROR_ID. If the operator picks flash storage there,"
echo " storage_migrate.sh moves the conf to appdata with the rest of the install."
else
echo " master.conf was NOT delivered. Phase 1 seeds it into a bare host, so this is a"
echo " real failure, not the pre-install state — check SSH and that /boot is writable"
echo " on $MIRROR. Retry with:"
echo " • bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
echo " • or push from Scheduler → master.conf → Save Conf"
echo " • or, once the plugin is installed, 'Pull from HOST1' on HOST2's Setup tab"
fi
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ── Step 1b: Ensure custom Docker network exists on mirror ────────────────────────────────────
# Must run before any container deploy — docker create fails if the network is missing.
echo ""
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
_net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
STEP_NETWORK_OK=true
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"bash '$_net_script'" 2>/dev/null; then
echo "Docker network ready on $MIRROR ✅"
STEP_NETWORK_OK=true
else
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
fi
# ── Step 1c: Share setup ─────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 1c — Share Setup (Mirror) ━━━"
if [[ "$SKIP_SHARE_SETUP" == true ]]; then
warn "Skipping (--skip-share-setup)"
elif [[ "$DRY_RUN" == true ]]; then
bash "$SCRIPT_DIR/share_setup.sh" --dry-run
else
bash "$SCRIPT_DIR/share_setup.sh"
fi
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
echo ""
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
if [[ "$SKIP_AUTH_STACK" == true ]]; then
warn "Skipping (--skip-auth-stack)"
else
stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
fi
# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
echo ""
echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━"
if [[ "$SKIP_AUTH_STACK" == true ]]; then
warn "Skipping (--skip-auth-stack)"
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
STEP_AUTH_OK=false
else
deploy_xml_stack PARTNERSHIP_AUTH_STACK
AUTH_DEPLOYED=$_STACK_DEPLOYED
AUTH_FAILED=$_STACK_FAILED
echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
[[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
fi
# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
echo ""
echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━"
if [[ "$SKIP_ARR_STACK" == true ]]; then
warn "Skipping (--skip-arr-stack)"
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
SKIP_ARR_STACK=true
else
stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
fi
# ── Step 5: Deploy arr stack on mirror ───────────────────────────────────────────────────────
echo ""
echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━"
if [[ "$SKIP_ARR_STACK" == true ]]; then
warn "Skipping (--skip-arr-stack)"
else
deploy_xml_stack PARTNERSHIP_ARR_STACK
ARR_DEPLOYED=$_STACK_DEPLOYED
ARR_FAILED=$_STACK_FAILED
echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
[[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
fi
# ── Step 6: Stop mirror's existing services stack ─────────────────────────────────────────────
echo ""
echo "━━━ Step 6 — Stop Mirror Services Stack ━━━"
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
warn "Skipping (--skip-services-stack)"
elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then
log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy"
SKIP_SERVICES_STACK=true
else
stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack"
fi
# ── Step 7: Deploy services stack on mirror ───────────────────────────────────────────────────
echo ""
echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━"
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
warn "Skipping (--skip-services-stack)"
else
deploy_xml_stack PARTNERSHIP_SERVICES_STACK
SERVICES_DEPLOYED=$_STACK_DEPLOYED
SERVICES_FAILED=$_STACK_FAILED
echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed"
[[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false
fi
# ── Step 8: Partnership onboard ───────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 8 — Partnership Onboard ━━━"
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
echo "Partnership onboard complete ✅"
ONBOARD_OK=true
else
error "Partnership onboard failed"
ONBOARD_OK=false
fi
# ── Step 9: Arr library bootstrap ─────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 9 — Arr Library Bootstrap ━━━"
if [[ "$ONBOARD_OK" == false ]]; then
warn "Skipping — onboard did not complete"
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
warn "Skipping (--skip-arr-sync)"
elif [[ ! -f "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ]]; then
warn "arr_sync.sh not found — run Arrs_Stack/arr_sync.sh manually once arrs are live"
elif bash "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
echo "Arr bootstrap complete ✅"
ARR_SYNC_OK=true
else
warn "Arr sync had errors — partnership still valid"
warn "Re-run Arrs_Stack/arr_sync.sh once all arr containers are live"
fi
# ── Step 9b: Webhook setup ────────────────────────────────────────────────────────────────────
# Register the download webhook in each arr on both servers. Arrs must be running.
# webhook_setup.sh handles local + SSH to remote in one call.
echo ""
echo "━━━ Step 9b — Webhook Setup ━━━"
_webhook_script="$SCRIPTS_ROOT/Tools/webhook_setup.sh"
if [[ "$SKIP_WEBHOOK_SETUP" == true ]]; then
warn "Skipping (--skip-webhook-setup)"
elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then
warn "WEBHOOK_PORT=0 — webhook disabled, skipping"
elif [[ ! -f "$_webhook_script" ]]; then
warn "Tools/webhook_setup.sh not found — run manually after onboard"
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would run webhook_setup.sh (local + remote)"
WEBHOOK_SETUP_OK=true
elif bash "$_webhook_script" "${EXTRA_FLAGS[@]}"; then
echo "Webhook setup complete ✅"
WEBHOOK_SETUP_OK=true
else
warn "Webhook setup had errors — run Tools/webhook_setup.sh manually once arrs are settled"
fi
unset _webhook_script
# ── Step 9c: Seed media shares on mirror ─────────────────────────────────────────────────────
# arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but
# have no files on disk. Without this rsync, every imported item looks missing and arrs
# will immediately queue searches. --seed skips the empty-remote guard and does a clean push.
echo ""
echo "━━━ Step 9c — Media Share Seed ━━━"
if [[ "$SKIP_MEDIA_SEED" == true ]]; then
warn "Skipping (--skip-media-seed)"
elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed"
warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually"
else
echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while"
_rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh"
_seed_flags=(--seed)
[[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run)
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
for _share in "${DAILY_SYNC_SHARES[@]}"; do
echo " Seeding: $_share"
if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then
(( MEDIA_SEED_COUNT++ )) || true
else
warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
fi
done
if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then
echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅"
MEDIA_SEED_OK=true
else
warn "Media seed: no shares completed — check errors above"
fi
unset _rsync_script _seed_flags _share
fi
# ── Step 9d: Start webhook listener on mirror ─────────────────────────────────────────────────
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
# already running — kick it now so events are captured immediately after onboard.
echo ""
echo "━━━ Step 9d — Webhook Listener (Mirror) ━━━"
_listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh"
if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then
warn "Skipping (--skip-webhook-listener)"
elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then
warn "WEBHOOK_PORT=0 — webhook disabled, skipping"
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would start webhook listener on $MIRROR"
WEBHOOK_LISTENER_OK=true
elif timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
"nohup bash '$_listener_script' > /var/log/varaverk/upgrade_webhook.log 2>&1 & echo started" \
2>/dev/null | grep -q started; then
echo "Webhook listener started on $MIRROR ✅"
WEBHOOK_LISTENER_OK=true
else
warn "Could not start listener on $MIRROR — it will start automatically on next array restart"
fi
unset _listener_script
# ── Step 9e: Arm the sync gates ───────────────────────────────────────────────────────────────
# master.conf.template ships a fresh node inert — RSYNC_ENABLED, CONF_SYNC_ENABLED and
# ARR_SYNC_ENABLED all false — because a node that has just been seeded has empty shares and no
# verified partner, and those two facts are what make unattended syncing safe to do.
#
# A successful onboard is the event that makes them true. Without this step the defaults were a
# one-way door: the node stayed inert for ever and somebody had to remember to hand-edit three
# toggles, on the machine where forgetting looks exactly like everything working.
#
# Runs before Step 10 on purpose. The push carries the owner's master.conf to every listed host,
# so arming here means both sides come up agreeing about whether sync is on; arming after the
# push would leave the mirror a version behind until the next conf save.
#
# Owner only — the mirror receives these values in the push rather than deciding for itself.
ARM_OK=true
echo ""
echo "━━━ $ICON_GEAR Step 9e — Arm Sync Gates ━━━"
if [[ "$ONBOARD_OK" == false ]]; then
warn "Skipping — onboard did not complete, leaving the gates closed"
ARM_OK=false
elif [[ "$SKIP_ARM" == true ]]; then
echo " --no-arm — leaving the sync gates as they are"
elif [[ "$AM_OWNER" != true ]]; then
echo " mirror — the owner's push decides these"
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would set RSYNC_ENABLED, CONF_SYNC_ENABLED, ARR_SYNC_ENABLED to true"
else
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
if [[ ! -f "$_master_conf" ]]; then
warn "master.conf not found at $_master_conf — gates left closed"
ARM_OK=false
else
cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)"
for _gate in RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED; do
_arm_conf_var "$_gate" "true" "$_master_conf" || ARM_OK=false
done
# Tier 1 is now open, so say what that actually turned on rather than leaving it implied.
echo " rsync tier 2 gates were left as configured — check them before the first run"
unset _gate
fi
unset _master_conf
fi
# ── Step 10: Push master.conf to all listed hosts ─────────────────────────────────────────────
# SSH is now established and all partners have the plugin installed.
# Push the authoritative master.conf so every listed host is in sync immediately.
echo ""
echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━"
if [[ "$ONBOARD_OK" == false ]]; then
warn "Skipping — onboard did not complete"
elif [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would push master.conf to all listed hosts"
MASTER_PUSH_OK=true
else
push_output=$(platform_push_conf)
push_rc=$?
[[ -n "$push_output" ]] && echo "$push_output"
platform_push_setup_state
if [[ $push_rc -eq 0 ]]; then
echo "master.conf sync complete ✅"
MASTER_PUSH_OK=true
else
warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
fi
fi
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
echo " Mirror: $MIRROR ($MIRROR_IP)"
[[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)"
echo " Duration: $(format_duration $(( END - START )))"
echo ""
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")"
echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )"
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
echo " Step 9c — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" || echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")" )"
echo " Step 9d — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
echo " Step 9e — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )"
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
echo ""
if [[ "$ONBOARD_OK" == true ]]; then
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
echo "$ICON_DONE DONE — partnership established ✅"
echo "Verify with: Partnership/partnership_manager.sh --status"
else
error "Setup incomplete — resolve errors above and re-run"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$ONBOARD_OK" == false ]] && exit 1
exit 0