#!/bin/bash # ============================================================================================== # ============================= Partnership Onboard ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Runs once on both servers to establish a new partnership. Role is detected # automatically via detect_hosts() — no flags needed to declare which side you are. # Run on the mirror first (generates its SSH key), then on the owner to complete # setup remotely. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # MIRROR PATH (1 step) # Step 1: SSH key setup — generate keypair, copy to owner, update conf # Owner completes the rest remotely. Mirror is done. # # OWNER PATH (14 steps) # Step 1: SSH key setup — generate keypair, install on mirror, update conf # Step 1b: Docker network — ensure varaverk docker network exists on mirror # Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent) # Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing # Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror # Mariadb/Redis health-checked before Authelia deploys # Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing # Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror # Step 6: Stop mirror services — stop mirror's existing services containers before replacing # Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start # Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby # Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh) # Step 9b: Webhook setup — register download webhook in arrs on both servers # Step 9c: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the # template ships closed so a fresh node cannot sync early. # Ahead of the seed because the seed is itself an rsync and # Tier 1 stops every rsync, including that one # Step 9d: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed) # prevents arrs treating every file as missing after bootstrap # Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed) # Step 10: Conf push — push master.conf + setup state to all listed hosts # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Credentials never in SSH command strings # Auth stack containers hold API keys, DB passwords, etc. The deploy script is written # locally, SCPed to the remote, and executed there. Command-line args are never used # to pass credentials — they'd appear in `ps` output and shell history on both servers. # # XML templates are the single source of truth for deployed containers # The owner's templates-user/ XMLs define every container deployed on the mirror. # The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's # Docker Manager can manage the containers after onboard without additional config. # # Dependency ordering in the auth stack is owner-enforced # PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia. # The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy, # the script waits for the container to be healthy before continuing. This is a remote # health check — the container must be running (or report healthy) before the next # dependent is deployed. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Root check # All operations run as root — SSH key management, docker operations, conf updates. # # SSH timeout on all remote calls # Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a # slow or unreachable mirror. # # --dry-run shows exact actions without executing # Every step prints what it would do. SCP, deploy, plugin install, arr sync — # all dry-run safe. # # Step skip flags for partial re-runs # --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow # resuming after a partial failure without re-running completed steps. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_PARTNERSHIP_AUTH_STACK # XML filenames (from this server's templates-user/) to push and deploy on the # mirror as its auth stack. Order matters: database deps before Authelia. # Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK # # HOST*_PARTNERSHIP_REPLACE_CONTAINERS # Containers to stop on the mirror before deploying the auth stack. # Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf. # Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at # the same $SCRIPTS_ROOT path — convention: both servers use the same repo location). # Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop). # Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror) # # HOST*_PARTNERSHIP_ARR_STACK # XML filenames to push and deploy on the mirror as its arr stack. # Leave empty to skip arr stack deploy. # Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK # # HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS # Arr containers to stop on the mirror before deploying the arr stack. # Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's. # Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror) # # HOST*_PARTNERSHIP_SERVICES_STACK # XML filenames to push and deploy on the mirror as its shared services stack. # Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy. # Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK # # HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS # Services containers to stop on the mirror before deploying the services stack. # Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's. # Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # Partnership/partnership_onboard.sh # Full onboard — role detected automatically # # Partnership/partnership_onboard.sh --dry-run # Preview all steps without making changes # # Partnership/partnership_onboard.sh --log # Verbose per-step output # # Partnership/partnership_onboard.sh --skip-ssh # Skip SSH key setup (key already in place) # # Partnership/partnership_onboard.sh --skip-share-setup # Skip share creation on mirror (shares already exist) # # Partnership/partnership_onboard.sh --skip-auth-stack # Skip auth stack stop + deploy (Steps 3-4) # # Partnership/partnership_onboard.sh --skip-arr-stack # Skip arr stack stop + deploy (Steps 4-5) # # Partnership/partnership_onboard.sh --skip-services-stack # Skip services stack stop + deploy (Steps 6-7) # # Partnership/partnership_onboard.sh --skip-arr-sync # Skip arr library bootstrap (Step 9) # # Partnership/partnership_onboard.sh --skip-webhook-setup # Skip webhook registration in arrs (Step 9b) # # Partnership/partnership_onboard.sh --skip-media-seed # Skip initial media share rsync to mirror (Step 9d) # Use when mirror already has files or you want to seed manually # # Partnership/partnership_onboard.sh --skip-webhook-listener # Skip starting webhook listener on mirror (Step 9e) # Listener will start automatically on next array restart # # Partnership/partnership_onboard.sh --no-arm # Leave RSYNC_ENABLED / CONF_SYNC_ENABLED / ARR_SYNC_ENABLED as they are (Step 9c). # Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose # shares are not populated yet. # # Partnership/partnership_onboard.sh --phase1-only # OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk. # Writes HOST2_PHASE1_DONE=true to varaverk_setup.db. # # Partnership/partnership_onboard.sh --phase2-only # OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically # by HOST2 after it completes its Mirror-path onboard. Can also be run manually. # Writes HOST2_PHASE2_DONE=true to varaverk_setup.db. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPTS_ROOT="$SCRIPT_DIR/.." SSH_TIMEOUT=15 source "$SCRIPTS_ROOT/load_config.sh" source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" # ── Parse flags ─────────────────────────────────────────────────────────────────────────────── SKIP_SSH=false SKIP_SHARE_SETUP=false SKIP_AUTH_STACK=false SKIP_ARR_STACK=false SKIP_SERVICES_STACK=false SKIP_ARR_SYNC=false SKIP_WEBHOOK_SETUP=false SKIP_MEDIA_SEED=false SKIP_WEBHOOK_LISTENER=false PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed) # MIRROR: SSH key install only, no owner notification PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards) SKIP_ARM=false # leave the sync gates as they are — see Step 9c FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --skip-ssh) SKIP_SSH=true ;; --skip-share-setup) SKIP_SHARE_SETUP=true ;; --skip-auth-stack) SKIP_AUTH_STACK=true ;; --skip-arr-stack) SKIP_ARR_STACK=true ;; --skip-services-stack) SKIP_SERVICES_STACK=true ;; --skip-arr-sync) SKIP_ARR_SYNC=true ;; --skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;; --skip-media-seed) SKIP_MEDIA_SEED=true ;; --skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;; --phase1-only) PHASE1_ONLY=true ;; --phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;; --no-arm) SKIP_ARM=true ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== [[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } acquire_lock if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts partnership_resolve_roles EXTRA_FLAGS=() [[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run") [[ "$ENABLE_LOGGING" == true ]] && EXTRA_FLAGS+=("--log") START=$(date +%s) # ── Helper: write phase completion flag to setup.db + push to remotes ───────────────────────── write_onboard_phase() { local target_id="$1" phase="$2" local key="${target_id}_PHASE${phase}_DONE" local state_file="$(platform_setup_db_path)" [[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; } set_state_var "$state_file" "$key" "true" platform_push_setup_state } # ── Helper: set a boolean toggle in master.conf, in place ───────────────────────────────────── # Booleans only. The value pattern stops at whitespace, so a quoted value containing spaces would # be truncated — every gate this arms is true/false and nothing else should use it. # # Preserves what the line already looks like: leading indent, the existing quoting style, and the # column the trailing comment sits in. master.conf is hand-aligned and read by a person; an edit # that reflows a line makes a diff that looks bigger than the change actually is. _arm_conf_var() { local key="$1" val="$2" file="$3" cur pad spaces quoted if ! grep -qE "^[[:space:]]*${key}=" "$file"; then warn " ${key} is not in master.conf — skipped (add it to the template first)" return 1 fi cur=$(grep -E "^[[:space:]]*${key}=" "$file" | head -1 | sed -E "s|^[[:space:]]*${key}=||; s|[[:space:]]*#.*$||") quoted=""; [[ "$cur" == \"*\" ]] && quoted='"' cur="${cur//\"/}" [[ "$cur" == "$val" ]] && { echo " ${key} already ${val}"; return 0; } # Pad only when a comment follows — otherwise a shorter value leaves trailing whitespace. pad=0 if grep -qE "^[[:space:]]*${key}=[^#]*#" "$file"; then pad=$(( ${#cur} - ${#val} )); (( pad < 0 )) && pad=0 fi spaces=$(printf '%*s' "$pad" '') sed -i -E "s|^([[:space:]]*)${key}=[^#[:space:]]*([[:space:]]*)(#.*)?$|\1${key}=${quoted}${val}${quoted}${spaces}\2\3|" "$file" echo " ${key}: ${cur} → ${val}" } echo "" echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " This: $MY_ID ($LOCAL_SERVER_NAME)" echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" # ============================================================================================== # ── HELPER: stop containers on the mirror by reading its own conf via SSH ──────────────────── # # SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers # use the same convention), and reads the named config array from the mirror's own conf. # HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf. # Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop). # # deploy_container_from_xml() already stops/removes containers with the same name as what's # being deployed. This step handles containers with DIFFERENT names that conflict. # ============================================================================================== stop_mirror_stack() { local config_var="$1" label="$2" local -a to_stop=() mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$') if [[ ${#to_stop[@]} -eq 0 ]]; then log "No $label containers to stop on $MIRROR — skipping" return 0 fi log "Stopping $label on $MIRROR: ${to_stop[*]}" for container in "${to_stop[@]}"; do if [[ "$DRY_RUN" == true ]]; then warn " DRY RUN — would stop + rm $container on $MIRROR" continue fi timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "docker stop '$container' 2>/dev/null docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \ grep -q removed && \ echo " $container removed ✅" || \ log " $container not found on $MIRROR — skipping" done } # ============================================================================================== # ── MIRROR PATH ─────────────────────────────────────────────────────────────────────────────── # ============================================================================================== if [[ "$AM_MIRROR" == true ]]; then echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━" echo "" echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2." echo "" if [[ "$SKIP_SSH" == true ]]; then warn "Skipping SSH setup (--skip-ssh)" elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then echo "SSH key ready ✅" else error "SSH key setup failed" exit 1 fi # Stop after the key when asked. ssh_setup.sh runs ssh-copy-id, which prompts for the # owner's root password on a first install — answerable in a terminal, never from the # WebGUI button, which is why the mirror's panel sends the operator to a terminal for # exactly this step and nothing more. The flag was parsed but only ever honoured on the # owner path, so a mirror asked for phase 1 silently ran the whole thing. if [[ "$PHASE1_ONLY" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY MIRROR PHASE 1 COMPLETE ━━━━━" echo " SSH key: ready" echo " Next: press ▶ Onboard on the Partnership tab to notify $OWNER" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi echo "" echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━" echo "" OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true) PHASE2_TRIGGERED=false if [[ -n "$OWNER_IP" ]]; then # Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP") if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2 via run_job.sh" PHASE2_TRIGGERED=true else # Launched through run_job.sh, the same path cron and api/run.php use, so Phase 2 # gets a stat file, a job log and a Scheduler entry on the owner. It used to be a # bare `nohup … > /tmp/vv_phase2_onboard.log`, which ran fine and left the owner's # entire half of onboarding invisible to its own WebGUI — no job record, nothing # under /var/log/varaverk, nothing for the operator to look at when asking why # pressing Onboard here appeared to do nothing. # # setsid, not bare nohup: the job must lead its own process group so api/stop.php # can signal the whole tree. api/run.php carries the same note for the same reason. # # Reported triggered only after the stat file proves run_job.sh actually started. # The old `& echo triggered` printed unconditionally — it would have claimed success # for a path that does not exist on the owner, which is exactly the failure mode a # mirror in a different storage mode hits. # Its own timeout, not SSH_TIMEOUT: the remote waits for the runner to prove itself, # and 15s would cut that short and report a healthy launch as a failure. _phase2_out=$(timeout 40 ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \ "bash -s -- $(printf '%q' "$OWNER_SCRIPTS_DIR")" 2>/dev/null <<'PHASE2_TRIGGER' sd="$1" runner="$sd/Plugin/unraid/run_job.sh" script="$sd/Partnership/partnership_onboard.sh" stat_file="/var/log/varaverk/Partnership/partnership_onboard.json" [ -f "$runner" ] || { echo "missing-runner:$runner"; exit 1; } [ -f "$script" ] || { echo "missing-script:$script"; exit 1; } # Absolute, not "newer than the file we saw a moment ago". A previous run's stat file rewritten # inside the same second would compare equal and read as a failed launch. t0=$(date +%s) setsid nohup bash "$runner" "Partnership/partnership_onboard.sh" "$script" \ --manual --phase2-only >/dev/null 2>&1 /dev/null || echo 0) [ "$now" -ge "$t0" ] && { echo triggered; exit 0; } done echo start-failed exit 1 PHASE2_TRIGGER ) # Failure patterns are matched first, and they echo a path back. An owner whose # SCRIPTS_DIR happened to contain the word "triggered" would otherwise satisfy a # leading *triggered* glob and report success for a launch that never happened. case "$_phase2_out" in missing-runner:*) error "Phase 2 not started — no run_job.sh at ${_phase2_out#missing-runner:} on $OWNER" ;; missing-script:*) error "Phase 2 not started — no partnership_onboard.sh at ${_phase2_out#missing-script:} on $OWNER" ;; start-failed) error "Phase 2 launch on $OWNER did not produce a job record — check run_job.sh there" ;; triggered) echo "Phase 2 triggered on $OWNER ✅" log "Watch on $OWNER: Scheduler tab, or tail -f /var/log/varaverk/Partnership/partnership_onboard.log" PHASE2_TRIGGERED=true ;; *) warn "Could not auto-trigger Phase 2 on $OWNER" ;; esac fi else warn "Cannot resolve $OWNER Tailscale IP" fi echo "" echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━" echo " SSH key: ready" echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )" if [[ "$PHASE2_TRIGGERED" == false ]]; then echo "" echo " Run manually on $OWNER:" echo " bash Partnership/partnership_onboard.sh --phase2-only" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── OWNER PATH ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; } log "Mirror: $MIRROR ($MIRROR_IP)" [[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)" [[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)" echo "" STEP_SSH_OK=false STEP_NETWORK_OK=false STEP_STOP_AUTH_OK=true STEP_AUTH_OK=true AUTH_DEPLOYED=0 AUTH_FAILED=0 STEP_STOP_ARR_OK=true STEP_ARR_OK=true ARR_DEPLOYED=0 ARR_FAILED=0 STEP_STOP_SERVICES_OK=true STEP_SERVICES_OK=true SERVICES_DEPLOYED=0 SERVICES_FAILED=0 ONBOARD_OK=false ARR_SYNC_OK=false WEBHOOK_SETUP_OK=false MEDIA_SEED_OK=false MEDIA_SEED_COUNT=0 WEBHOOK_LISTENER_OK=false MASTER_PUSH_OK=false # ── Step 1: SSH ─────────────────────────────────────────────────────────────────────────────── # Skipped when --phase2-only (SSH was already done in Phase 1). echo "━━━ Step 1 — SSH Key Setup ━━━" if [[ "$SKIP_SSH" == true ]]; then warn "Skipping (--skip-ssh)" STEP_SSH_OK=true elif [[ "$PHASE1_ONLY" == true ]]; then # Phase 1 in background: test if SSH already works first — avoids ssh-copy-id # hanging for a password prompt with no TTY. if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then echo "SSH to $MIRROR already works ✅ — skipping key install" STEP_SSH_OK=true else # Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background) if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then echo "SSH keys ready ✅" STEP_SSH_OK=true else # Soft-fail: generate key locally if not present, then tell user to install manually warn "Could not install key on $MIRROR automatically (no terminal for password prompt)" if [[ -f "$SSH_KEY" ]]; then log "Local key exists at: $SSH_KEY" else bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true fi if [[ -f "${SSH_KEY}.pub" ]]; then echo "" echo " Install this key on $MIRROR to complete SSH setup:" echo " ┌─────────────────────────────────────────────────────" cat "${SSH_KEY}.pub" | sed 's/^/ │ /' echo " └─────────────────────────────────────────────────────" echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}" echo " Then click 'Push Conf' in the Partnership tab." # Write key-ready flag so UI can show the manual-install state [[ "$DRY_RUN" == false ]] && { kflag="${MIRROR_ID}_KEY_READY" _setup_f="$(platform_setup_db_path)" set_state_var "$_setup_f" "$kflag" "true" } fi STEP_SSH_OK=false fi fi elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then echo "SSH keys ready ✅" STEP_SSH_OK=true else error "SSH key setup failed — aborting" error "Re-run or use --skip-ssh if key is already set up" exit 1 fi # ── Phase 1 exit point ──────────────────────────────────────────────────────────────────────── # --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk. # HOST2's wizard will detect the pushed master.conf + state file and take the correct path. if [[ "$PHASE1_ONLY" == true ]]; then if [[ "$STEP_SSH_OK" == false ]]; then # SSH key not yet installed on HOST2 — can't push conf, but local setup still runs. # UI will show "key ready, install manually" state via HOST2_KEY_READY flag. echo "" echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━" bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \ warn "Local setup had issues — check partnership_manager.sh output above" END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━" echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠" echo " Conf push: skipped (needs SSH access to $MIRROR)" echo " HOST1 setup: done ✅" echo " Duration: $(format_duration $(( END - START )))" echo "" echo " ACTION NEEDED: install the key on $MIRROR:" echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}" echo " Then click 'Push Conf' in Partnership tab, or run:" echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi echo "" echo "━━━ Phase 1 — Conf Push ━━━" CONF_PUSH_OK=false if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push master.conf + state file to $MIRROR" CONF_PUSH_OK=true else push_output=$(platform_push_conf) push_rc=$? [[ -n "$push_output" ]] && echo "$push_output" platform_push_setup_state if [[ $push_rc -eq 0 ]]; then echo "Conf push complete ✅" CONF_PUSH_OK=true else warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf" fi fi # HOST1 local setup — runs immediately without needing HOST2 echo "" echo "━━━ Phase 1 — HOST1 Local Setup ━━━" bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \ warn "Local setup had issues — check partnership_manager.sh output above" [[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1 END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━" echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )" echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )" echo " HOST1 setup: done ✅" echo " Duration: $(format_duration $(( END - START )))" echo "" echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin." # The push no longer waits for a Varaverk install on the far side. It resolves the partner's # conf directory across both layouts and creates the internal one if neither exists, so the # conf arrives BEFORE the plugin — which is the order that makes it useful. The .plg only # seeds master.conf from the template when none is present, so what lands here survives the # install and the wizard reads its identity straight out of it. if [[ "$CONF_PUSH_OK" == true ]]; then echo " master.conf is on $MIRROR — the wizard will find it and take the partner path," echo " already knowing $MY_ID and $MIRROR_ID. If the operator picks flash storage there," echo " storage_migrate.sh moves the conf to appdata with the rest of the install." else echo " master.conf was NOT delivered. Phase 1 seeds it into a bare host, so this is a" echo " real failure, not the pre-install state — check SSH and that /boot is writable" echo " on $MIRROR. Retry with:" echo " • bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh" echo " • or push from Scheduler → master.conf → Save Conf" echo " • or, once the plugin is installed, 'Pull from HOST1' on HOST2's Setup tab" fi echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here." echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ── Step 1b: Ensure custom Docker network exists on mirror ──────────────────────────────────── # Must run before any container deploy — docker create fails if the network is missing. echo "" echo "━━━ Step 1b — Docker Network (Mirror) ━━━" _net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run docker_network_connect.sh on $MIRROR" STEP_NETWORK_OK=true elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "bash '$_net_script'" 2>/dev/null; then echo "Docker network ready on $MIRROR ✅" STEP_NETWORK_OK=true else warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing" warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed" fi # ── Step 1c: Share setup ───────────────────────────────────────────────────────────────────── echo "" echo "━━━ Step 1c — Share Setup (Mirror) ━━━" if [[ "$SKIP_SHARE_SETUP" == true ]]; then warn "Skipping (--skip-share-setup)" elif [[ "$DRY_RUN" == true ]]; then bash "$SCRIPT_DIR/share_setup.sh" --dry-run else bash "$SCRIPT_DIR/share_setup.sh" fi # ── Step 2: Stop mirror's existing auth stack ───────────────────────────────────────────────── echo "" echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━" if [[ "$SKIP_AUTH_STACK" == true ]]; then warn "Skipping (--skip-auth-stack)" else stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack" fi # ── Step 4: Deploy auth stack on mirror ─────────────────────────────────────────────────────── echo "" echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━" if [[ "$SKIP_AUTH_STACK" == true ]]; then warn "Skipping (--skip-auth-stack)" elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy" warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf" STEP_AUTH_OK=false else deploy_xml_stack PARTNERSHIP_AUTH_STACK AUTH_DEPLOYED=$_STACK_DEPLOYED AUTH_FAILED=$_STACK_FAILED echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed" [[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false fi # ── Step 5: Stop mirror's existing arr stack ────────────────────────────────────────────────── echo "" echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━" if [[ "$SKIP_ARR_STACK" == true ]]; then warn "Skipping (--skip-arr-stack)" elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy" SKIP_ARR_STACK=true else stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack" fi # ── Step 5: Deploy arr stack on mirror ─────────────────────────────────────────────────────── echo "" echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━" if [[ "$SKIP_ARR_STACK" == true ]]; then warn "Skipping (--skip-arr-stack)" else deploy_xml_stack PARTNERSHIP_ARR_STACK ARR_DEPLOYED=$_STACK_DEPLOYED ARR_FAILED=$_STACK_FAILED echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed" [[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false fi # ── Step 6: Stop mirror's existing services stack ───────────────────────────────────────────── echo "" echo "━━━ Step 6 — Stop Mirror Services Stack ━━━" if [[ "$SKIP_SERVICES_STACK" == true ]]; then warn "Skipping (--skip-services-stack)" elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy" SKIP_SERVICES_STACK=true else stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack" fi # ── Step 7: Deploy services stack on mirror ─────────────────────────────────────────────────── echo "" echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━" if [[ "$SKIP_SERVICES_STACK" == true ]]; then warn "Skipping (--skip-services-stack)" else deploy_xml_stack PARTNERSHIP_SERVICES_STACK SERVICES_DEPLOYED=$_STACK_DEPLOYED SERVICES_FAILED=$_STACK_FAILED echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed" [[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false fi # ── Step 8: Partnership onboard ─────────────────────────────────────────────────────────────── echo "" echo "━━━ Step 8 — Partnership Onboard ━━━" if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then echo "Partnership onboard complete ✅" ONBOARD_OK=true else error "Partnership onboard failed" ONBOARD_OK=false fi # ── Step 9: Arr library bootstrap ───────────────────────────────────────────────────────────── echo "" echo "━━━ Step 9 — Arr Library Bootstrap ━━━" if [[ "$ONBOARD_OK" == false ]]; then warn "Skipping — onboard did not complete" elif [[ "$SKIP_ARR_SYNC" == true ]]; then warn "Skipping (--skip-arr-sync)" elif [[ ! -f "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ]]; then warn "arr_sync.sh not found — run Arrs_Stack/arr_sync.sh manually once arrs are live" elif bash "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then echo "Arr bootstrap complete ✅" ARR_SYNC_OK=true else warn "Arr sync had errors — partnership still valid" warn "Re-run Arrs_Stack/arr_sync.sh once all arr containers are live" fi # ── Step 9b: Webhook setup ──────────────────────────────────────────────────────────────────── # Register the download webhook in each arr on both servers. Arrs must be running. # webhook_setup.sh handles local + SSH to remote in one call. echo "" echo "━━━ Step 9b — Webhook Setup ━━━" _webhook_script="$SCRIPTS_ROOT/Tools/webhook_setup.sh" if [[ "$SKIP_WEBHOOK_SETUP" == true ]]; then warn "Skipping (--skip-webhook-setup)" elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then warn "WEBHOOK_PORT=0 — webhook disabled, skipping" elif [[ ! -f "$_webhook_script" ]]; then warn "Tools/webhook_setup.sh not found — run manually after onboard" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run webhook_setup.sh (local + remote)" WEBHOOK_SETUP_OK=true elif bash "$_webhook_script" "${EXTRA_FLAGS[@]}"; then echo "Webhook setup complete ✅" WEBHOOK_SETUP_OK=true else warn "Webhook setup had errors — run Tools/webhook_setup.sh manually once arrs are settled" fi unset _webhook_script # ── Step 9c: Arm the sync gates ─────────────────────────────────────────────────────────────── # master.conf.template ships a fresh node inert — RSYNC_ENABLED, CONF_SYNC_ENABLED and # ARR_SYNC_ENABLED all false — because a node that has just been seeded has empty shares and no # verified partner, and those two facts are what make unattended syncing safe to do. # # A successful onboard is the event that makes them true. Without this step the defaults were a # one-way door: the node stayed inert for ever and somebody had to remember to hand-edit three # toggles, on the machine where forgetting looks exactly like everything working. # # Ahead of the media seed, which it used to follow. The seed calls Rsync/rsync.sh, and Tier 1 # stops *every* rsync — so with the gates still closed all of it exited cleanly having moved # nothing, and, because a clean exit is exit 0, the seed counted each share as a success and # reported "14/14 shares ✅" over an empty mirror. # # Still ahead of Step 10 on purpose. The push carries the owner's master.conf to every listed # host, so arming before it means both sides come up agreeing about whether sync is on; arming # after the push would leave the mirror a version behind until the next conf save. # # Owner only — the mirror receives these values in the push rather than deciding for itself. ARM_OK=true echo "" echo "━━━ $ICON_GEAR Step 9c — Arm Sync Gates ━━━" if [[ "$ONBOARD_OK" == false ]]; then warn "Skipping — onboard did not complete, leaving the gates closed" ARM_OK=false elif [[ "$SKIP_ARM" == true ]]; then echo " --no-arm — leaving the sync gates as they are" elif [[ "$AM_OWNER" != true ]]; then echo " mirror — the owner's push decides these" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would set RSYNC_ENABLED, CONF_SYNC_ENABLED, ARR_SYNC_ENABLED to true" else _master_conf="$SCRIPTS_ROOT/Configurations/master.conf" if [[ ! -f "$_master_conf" ]]; then warn "master.conf not found at $_master_conf — gates left closed" ARM_OK=false else cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)" for _gate in RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED; do _arm_conf_var "$_gate" "true" "$_master_conf" || ARM_OK=false done # Tier 1 is now open, so say what that actually turned on rather than leaving it implied. echo " rsync tier 2 gates were left as configured — check them before the first run" unset _gate fi unset _master_conf fi # ── Step 9d: Seed media shares on mirror ───────────────────────────────────────────────────── # arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but # have no files on disk. Without this rsync, every imported item looks missing and arrs # will immediately queue searches. --seed skips the empty-remote guard and does a clean push. echo "" echo "━━━ Step 9d — Media Share Seed ━━━" # Read the gate back off disk rather than trusting the value sourced at startup: Step 9c has # just rewritten master.conf, and each rsync.sh below is a separate process that will source # the file fresh. A closed gate here means every share would exit 0 having moved nothing, so # say so once and skip, instead of counting fourteen no-ops as fourteen successes. # # The trailing comment is cut before the value is read. master.conf writes this gate as # RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below # so stopping at `cut -d= -f2` and squeezing whitespace yields "true#Tier1—globalgate,…", # which is not "true" — and the seed would then skip itself on every single run, including # the ones where the gate is open. _rsync_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \ | cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]') if [[ "$SKIP_MEDIA_SEED" == true ]]; then warn "Skipping (--skip-media-seed)" elif [[ "$DRY_RUN" == false && "$_rsync_gate" != "true" ]]; then warn "RSYNC_ENABLED is '$_rsync_gate' — skipping media seed, rsync.sh would move nothing" warn "Arm it and re-run: Rsync/rsync.sh --seed" elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed" warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually" else echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while" _rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh" _seed_flags=(--seed) [[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run) [[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log) for _share in "${DAILY_SYNC_SHARES[@]}"; do echo " Seeding: $_share" if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then (( MEDIA_SEED_COUNT++ )) || true else warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed" fi done if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅" MEDIA_SEED_OK=true else warn "Media seed: no shares completed — check errors above" fi unset _rsync_script _seed_flags _share fi # ── Step 9e: Start webhook listener on mirror ───────────────────────────────────────────────── # Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is # already running — kick it now so events are captured immediately after onboard. echo "" echo "━━━ Step 9e — Webhook Listener (Mirror) ━━━" _listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh" if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then warn "Skipping (--skip-webhook-listener)" elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then warn "WEBHOOK_PORT=0 — webhook disabled, skipping" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start webhook listener on $MIRROR" WEBHOOK_LISTENER_OK=true elif timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "nohup bash '$_listener_script' > /var/log/varaverk/upgrade_webhook.log 2>&1 & echo started" \ 2>/dev/null | grep -q started; then echo "Webhook listener started on $MIRROR ✅" WEBHOOK_LISTENER_OK=true else warn "Could not start listener on $MIRROR — it will start automatically on next array restart" fi unset _listener_script # ── Step 10: Push master.conf to all listed hosts ───────────────────────────────────────────── # SSH is now established and all partners have the plugin installed. # Push the authoritative master.conf so every listed host is in sync immediately. echo "" echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━" if [[ "$ONBOARD_OK" == false ]]; then warn "Skipping — onboard did not complete" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push master.conf to all listed hosts" MASTER_PUSH_OK=true else push_output=$(platform_push_conf) push_rc=$? [[ -n "$push_output" ]] && echo "$push_output" platform_push_setup_state if [[ $push_rc -eq 0 ]]; then echo "master.conf sync complete ✅" MASTER_PUSH_OK=true else warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf" fi fi # ── Write Phase 2 completion state ──────────────────────────────────────────────────────────── [[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2 # ── Summary ─────────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)" echo " Mirror: $MIRROR ($MIRROR_IP)" [[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)" echo " Duration: $(format_duration $(( END - START )))" echo "" _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")" echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")" echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")" echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )" echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")" echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )" echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")" echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )" echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")" echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )" echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")" echo " Step 9c — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )" echo " Step 9d — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" \ || { [[ "$DRY_RUN" == false && "${_rsync_gate:-}" != "true" ]] && echo "skipped (rsync gate closed)" \ || echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")"; } )" echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")" echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )" echo "" if [[ "$ONBOARD_OK" == true ]]; then [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ echo "$ICON_DONE DONE — partnership established ✅" echo "Verify with: Partnership/partnership_manager.sh --status" else error "Setup incomplete — resolve errors above and re-run" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" [[ "$ONBOARD_OK" == false ]] && exit 1 exit 0