#!/bin/bash # ============================================================================================== # ============================= Partnership Onboard ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Runs once on both servers to establish a new partnership. Role is detected # automatically via detect_hosts() — no flags needed to declare which side you are. # Run on the mirror first (generates its SSH key), then on the owner to complete # setup remotely. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # MIRROR PATH (1 step) # Step 1: SSH key setup — generate keypair, copy to owner, update conf # Owner completes the rest remotely. Mirror is done. # # OWNER PATH (14 steps) # Step 1: SSH key setup — generate keypair, install on mirror, update conf # Step 1b: Docker network — ensure varaverk docker network exists on mirror # Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent) # Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing # Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror # Mariadb/Redis health-checked before Authelia deploys # Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing # Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror # Step 6: Stop mirror services — stop mirror's existing services containers before replacing # Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start # Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby # Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh) # Step 9b: Webhook setup — register download webhook in arrs on both servers # Step 9c: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed) # prevents arrs treating every file as missing after bootstrap # Step 9d: Webhook listener — start listener on mirror (runs continuously, no reboot needed) # Step 10: Conf push — push master.conf + setup state to all listed hosts # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Credentials never in SSH command strings # Auth stack containers hold API keys, DB passwords, etc. The deploy script is written # locally, SCPed to the remote, and executed there. Command-line args are never used # to pass credentials — they'd appear in `ps` output and shell history on both servers. # # XML templates are the single source of truth for deployed containers # The owner's templates-user/ XMLs define every container deployed on the mirror. # The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's # Docker Manager can manage the containers after onboard without additional config. # # Dependency ordering in the auth stack is owner-enforced # PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia. # The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy, # the script waits for the container to be healthy before continuing. This is a remote # health check — the container must be running (or report healthy) before the next # dependent is deployed. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Root check # All operations run as root — SSH key management, docker operations, conf updates. # # SSH timeout on all remote calls # Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a # slow or unreachable mirror. # # --dry-run shows exact actions without executing # Every step prints what it would do. SCP, deploy, plugin install, arr sync — # all dry-run safe. # # Step skip flags for partial re-runs # --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow # resuming after a partial failure without re-running completed steps. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_PARTNERSHIP_AUTH_STACK # XML filenames (from this server's templates-user/) to push and deploy on the # mirror as its auth stack. Order matters: database deps before Authelia. # Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK # # HOST*_PARTNERSHIP_REPLACE_CONTAINERS # Containers to stop on the mirror before deploying the auth stack. # Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf. # Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at # the same $SCRIPTS_ROOT path — convention: both servers use the same repo location). # Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop). # Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror) # # HOST*_PARTNERSHIP_ARR_STACK # XML filenames to push and deploy on the mirror as its arr stack. # Leave empty to skip arr stack deploy. # Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK # # HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS # Arr containers to stop on the mirror before deploying the arr stack. # Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's. # Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror) # # HOST*_PARTNERSHIP_SERVICES_STACK # XML filenames to push and deploy on the mirror as its shared services stack. # Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy. # Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK # # HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS # Services containers to stop on the mirror before deploying the services stack. # Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's. # Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # Partnership/partnership_onboard.sh # Full onboard — role detected automatically # # Partnership/partnership_onboard.sh --dry-run # Preview all steps without making changes # # Partnership/partnership_onboard.sh --log # Verbose per-step output # # Partnership/partnership_onboard.sh --skip-ssh # Skip SSH key setup (key already in place) # # Partnership/partnership_onboard.sh --skip-share-setup # Skip share creation on mirror (shares already exist) # # Partnership/partnership_onboard.sh --skip-auth-stack # Skip auth stack stop + deploy (Steps 3-4) # # Partnership/partnership_onboard.sh --skip-arr-stack # Skip arr stack stop + deploy (Steps 4-5) # # Partnership/partnership_onboard.sh --skip-services-stack # Skip services stack stop + deploy (Steps 6-7) # # Partnership/partnership_onboard.sh --skip-arr-sync # Skip arr library bootstrap (Step 9) # # Partnership/partnership_onboard.sh --skip-webhook-setup # Skip webhook registration in arrs (Step 9b) # # Partnership/partnership_onboard.sh --skip-media-seed # Skip initial media share rsync to mirror (Step 9c) # Use when mirror already has files or you want to seed manually # # Partnership/partnership_onboard.sh --skip-webhook-listener # Skip starting webhook listener on mirror (Step 9d) # Listener will start automatically on next array restart # # Partnership/partnership_onboard.sh --phase1-only # OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk. # Writes HOST2_PHASE1_DONE=true to varaverk_setup.db. # # Partnership/partnership_onboard.sh --phase2-only # OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically # by HOST2 after it completes its Mirror-path onboard. Can also be run manually. # Writes HOST2_PHASE2_DONE=true to varaverk_setup.db. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPTS_ROOT="$SCRIPT_DIR/.." SSH_TIMEOUT=15 source "$SCRIPTS_ROOT/load_config.sh" source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" # ── Parse flags ─────────────────────────────────────────────────────────────────────────────── SKIP_SSH=false SKIP_SHARE_SETUP=false SKIP_AUTH_STACK=false SKIP_ARR_STACK=false SKIP_SERVICES_STACK=false SKIP_ARR_SYNC=false SKIP_WEBHOOK_SETUP=false SKIP_MEDIA_SEED=false SKIP_WEBHOOK_LISTENER=false PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed) PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards) FILTERED_ARGS=() for arg in "$@"; do case "$arg" in --skip-ssh) SKIP_SSH=true ;; --skip-share-setup) SKIP_SHARE_SETUP=true ;; --skip-auth-stack) SKIP_AUTH_STACK=true ;; --skip-arr-stack) SKIP_ARR_STACK=true ;; --skip-services-stack) SKIP_SERVICES_STACK=true ;; --skip-arr-sync) SKIP_ARR_SYNC=true ;; --skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;; --skip-media-seed) SKIP_MEDIA_SEED=true ;; --skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;; --phase1-only) PHASE1_ONLY=true ;; --phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;; *) FILTERED_ARGS+=("$arg") ;; esac done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== [[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; } acquire_lock if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" ) OWNER="${!OWNER_ID}" MIRROR="${!MIRROR_ID}" # SSH_KEY (set by detect_hosts) is this server's own private key. # The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh. # HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's # conf is never present here. Always use SSH_KEY (local private key) for outbound SSH. MIRROR_SSH_KEY="$SSH_KEY" AM_OWNER=false AM_MIRROR=false [[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true [[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true EXTRA_FLAGS=() [[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run") [[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log") START=$(date +%s) # ── Helper: write phase completion flag to setup.db + push to remotes ───────────────────────── write_onboard_phase() { local target_id="$1" phase="$2" local key="${target_id}_PHASE${phase}_DONE" local state_file="$(platform_setup_db_path)" [[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; } if grep -q "^${key}=" "$state_file" 2>/dev/null; then sed -i "s|^${key}=.*|${key}=true|" "$state_file" else echo "${key}=true" >> "$state_file" fi platform_push_setup_state } echo "" echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )" echo " This: $MY_ID ($LOCAL_SERVER_NAME)" echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )" echo "" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made" # ============================================================================================== # ── HELPER: stop containers on the mirror by reading its own conf via SSH ──────────────────── # # SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers # use the same convention), and reads the named config array from the mirror's own conf. # HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf. # Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop). # # deploy_container_from_xml() already stops/removes containers with the same name as what's # being deployed. This step handles containers with DIFFERENT names that conflict. # ============================================================================================== stop_mirror_stack() { local config_var="$1" label="$2" local -a to_stop=() mapfile -t to_stop < <( timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null detect_hosts 2>/dev/null printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$' ) if [[ ${#to_stop[@]} -eq 0 ]]; then log "No $label containers to stop on $MIRROR — skipping" return 0 fi log "Stopping $label on $MIRROR: ${to_stop[*]}" for container in "${to_stop[@]}"; do if [[ "$DRY_RUN" == true ]]; then warn " DRY RUN — would stop + rm $container on $MIRROR" continue fi timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \ "docker stop '$container' 2>/dev/null docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \ grep -q removed && \ echo " $container removed ✅" || \ log " $container not found on $MIRROR — skipping" done } # ============================================================================================== # ── MIRROR PATH ─────────────────────────────────────────────────────────────────────────────── # ============================================================================================== if [[ "$AM_MIRROR" == true ]]; then echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━" echo "" echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2." echo "" if [[ "$SKIP_SSH" == true ]]; then warn "Skipping SSH setup (--skip-ssh)" elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then echo "SSH key ready ✅" else error "SSH key setup failed" exit 1 fi echo "" echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━" echo "" OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true) PHASE2_TRIGGERED=false if [[ -n "$OWNER_IP" ]]; then # Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror _probe_cmd=$(platform_scripts_dir_probe_cmd) OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \ "$_probe_cmd" 2>/dev/null | tr -d '[:space:]') OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2" PHASE2_TRIGGERED=true elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \ "nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \ 2>/dev/null | grep -q triggered; then echo "Phase 2 triggered on $OWNER ✅" log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log" PHASE2_TRIGGERED=true else warn "Could not auto-trigger Phase 2 on $OWNER" fi else warn "Cannot resolve $OWNER Tailscale IP" fi echo "" echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━" echo " SSH key: ready" echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )" if [[ "$PHASE2_TRIGGERED" == false ]]; then echo "" echo " Run manually on $OWNER:" echo " bash Partnership/partnership_onboard.sh --phase2-only" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── OWNER PATH ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== MIRROR_IP=$(resolve_tailscale_ip "$MIRROR") [[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; } log "Mirror: $MIRROR ($MIRROR_IP)" [[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)" [[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)" echo "" STEP_SSH_OK=false STEP_NETWORK_OK=false STEP_STOP_AUTH_OK=true STEP_AUTH_OK=true AUTH_DEPLOYED=0 AUTH_FAILED=0 STEP_STOP_ARR_OK=true STEP_ARR_OK=true ARR_DEPLOYED=0 ARR_FAILED=0 STEP_STOP_SERVICES_OK=true STEP_SERVICES_OK=true SERVICES_DEPLOYED=0 SERVICES_FAILED=0 ONBOARD_OK=false ARR_SYNC_OK=false WEBHOOK_SETUP_OK=false MEDIA_SEED_OK=false MEDIA_SEED_COUNT=0 WEBHOOK_LISTENER_OK=false MASTER_PUSH_OK=false # ── Step 1: SSH ─────────────────────────────────────────────────────────────────────────────── # Skipped when --phase2-only (SSH was already done in Phase 1). echo "━━━ Step 1 — SSH Key Setup ━━━" if [[ "$SKIP_SSH" == true ]]; then warn "Skipping (--skip-ssh)" STEP_SSH_OK=true elif [[ "$PHASE1_ONLY" == true ]]; then # Phase 1 in background: test if SSH already works first — avoids ssh-copy-id # hanging for a password prompt with no TTY. if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then echo "SSH to $MIRROR already works ✅ — skipping key install" STEP_SSH_OK=true else # Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background) if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then echo "SSH keys ready ✅" STEP_SSH_OK=true else # Soft-fail: generate key locally if not present, then tell user to install manually warn "Could not install key on $MIRROR automatically (no terminal for password prompt)" if [[ -f "$SSH_KEY" ]]; then log "Local key exists at: $SSH_KEY" else bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true fi if [[ -f "${SSH_KEY}.pub" ]]; then echo "" echo " Install this key on $MIRROR to complete SSH setup:" echo " ┌─────────────────────────────────────────────────────" cat "${SSH_KEY}.pub" | sed 's/^/ │ /' echo " └─────────────────────────────────────────────────────" echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}" echo " Then click 'Push Conf' in the Partnership tab." # Write key-ready flag so UI can show the manual-install state [[ "$DRY_RUN" == false ]] && { local kflag="${MIRROR_ID}_KEY_READY" local _setup_f="$(platform_setup_db_path)" grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \ && sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \ || echo "${kflag}=true" >> "$_setup_f" } fi STEP_SSH_OK=false fi fi elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then echo "SSH keys ready ✅" STEP_SSH_OK=true else error "SSH key setup failed — aborting" error "Re-run or use --skip-ssh if key is already set up" exit 1 fi # ── Phase 1 exit point ──────────────────────────────────────────────────────────────────────── # --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk. # HOST2's wizard will detect the pushed master.conf + state file and take the correct path. if [[ "$PHASE1_ONLY" == true ]]; then if [[ "$STEP_SSH_OK" == false ]]; then # SSH key not yet installed on HOST2 — can't push conf, but local setup still runs. # UI will show "key ready, install manually" state via HOST2_KEY_READY flag. echo "" echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━" bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \ warn "Local setup had issues — check partnership_manager.sh output above" END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━" echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠" echo " Conf push: skipped (needs SSH access to $MIRROR)" echo " HOST1 setup: done ✅" echo " Duration: $(format_duration $(( END - START )))" echo "" echo " ACTION NEEDED: install the key on $MIRROR:" echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}" echo " Then click 'Push Conf' in Partnership tab, or run:" echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi echo "" echo "━━━ Phase 1 — Conf Push ━━━" CONF_PUSH_OK=false if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push master.conf + state file to $MIRROR" CONF_PUSH_OK=true else push_output=$(platform_push_conf) push_rc=$? [[ -n "$push_output" ]] && echo "$push_output" platform_push_setup_state if [[ $push_rc -eq 0 ]]; then echo "Conf push complete ✅" CONF_PUSH_OK=true else warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf" fi fi # HOST1 local setup — runs immediately without needing HOST2 echo "" echo "━━━ Phase 1 — HOST1 Local Setup ━━━" bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \ warn "Local setup had issues — check partnership_manager.sh output above" [[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1 END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━" echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )" echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )" echo " HOST1 setup: done ✅" echo " Duration: $(format_duration $(( END - START )))" echo "" echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin." echo " The wizard will detect the pushed conf and take the correct path." echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here." echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ── Step 1b: Ensure custom Docker network exists on mirror ──────────────────────────────────── # Must run before any container deploy — docker create fails if the network is missing. echo "" echo "━━━ Step 1b — Docker Network (Mirror) ━━━" _net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run docker_network_connect.sh on $MIRROR" STEP_NETWORK_OK=true elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "bash '$_net_script'" 2>/dev/null; then echo "Docker network ready on $MIRROR ✅" STEP_NETWORK_OK=true else warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing" warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed" fi # ── Step 1c: Share setup ───────────────────────────────────────────────────────────────────── echo "" echo "━━━ Step 1c — Share Setup (Mirror) ━━━" if [[ "$SKIP_SHARE_SETUP" == true ]]; then warn "Skipping (--skip-share-setup)" elif [[ "$DRY_RUN" == true ]]; then bash "$SCRIPT_DIR/share_setup.sh" --dry-run else bash "$SCRIPT_DIR/share_setup.sh" fi # ── Step 2: Stop mirror's existing auth stack ───────────────────────────────────────────────── echo "" echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━" if [[ "$SKIP_AUTH_STACK" == true ]]; then warn "Skipping (--skip-auth-stack)" else stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack" fi # ── Step 4: Deploy auth stack on mirror ─────────────────────────────────────────────────────── echo "" echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━" if [[ "$SKIP_AUTH_STACK" == true ]]; then warn "Skipping (--skip-auth-stack)" elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy" warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf" STEP_AUTH_OK=false else deploy_xml_stack PARTNERSHIP_AUTH_STACK AUTH_DEPLOYED=$_STACK_DEPLOYED AUTH_FAILED=$_STACK_FAILED echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed" [[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false fi # ── Step 5: Stop mirror's existing arr stack ────────────────────────────────────────────────── echo "" echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━" if [[ "$SKIP_ARR_STACK" == true ]]; then warn "Skipping (--skip-arr-stack)" elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy" SKIP_ARR_STACK=true else stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack" fi # ── Step 5: Deploy arr stack on mirror ─────────────────────────────────────────────────────── echo "" echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━" if [[ "$SKIP_ARR_STACK" == true ]]; then warn "Skipping (--skip-arr-stack)" else deploy_xml_stack PARTNERSHIP_ARR_STACK ARR_DEPLOYED=$_STACK_DEPLOYED ARR_FAILED=$_STACK_FAILED echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed" [[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false fi # ── Step 6: Stop mirror's existing services stack ───────────────────────────────────────────── echo "" echo "━━━ Step 6 — Stop Mirror Services Stack ━━━" if [[ "$SKIP_SERVICES_STACK" == true ]]; then warn "Skipping (--skip-services-stack)" elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy" SKIP_SERVICES_STACK=true else stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack" fi # ── Step 7: Deploy services stack on mirror ─────────────────────────────────────────────────── echo "" echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━" if [[ "$SKIP_SERVICES_STACK" == true ]]; then warn "Skipping (--skip-services-stack)" else deploy_xml_stack PARTNERSHIP_SERVICES_STACK SERVICES_DEPLOYED=$_STACK_DEPLOYED SERVICES_FAILED=$_STACK_FAILED echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed" [[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false fi # ── Step 8: Partnership onboard ─────────────────────────────────────────────────────────────── echo "" echo "━━━ Step 8 — Partnership Onboard ━━━" if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then echo "Partnership onboard complete ✅" ONBOARD_OK=true else error "Partnership onboard failed" ONBOARD_OK=false fi # ── Step 9: Arr library bootstrap ───────────────────────────────────────────────────────────── echo "" echo "━━━ Step 9 — Arr Library Bootstrap ━━━" if [[ "$ONBOARD_OK" == false ]]; then warn "Skipping — onboard did not complete" elif [[ "$SKIP_ARR_SYNC" == true ]]; then warn "Skipping (--skip-arr-sync)" elif [[ ! -f "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ]]; then warn "arr_sync.sh not found — run Arrs_Stack/arr_sync.sh manually once arrs are live" elif bash "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then echo "Arr bootstrap complete ✅" ARR_SYNC_OK=true else warn "Arr sync had errors — partnership still valid" warn "Re-run Arrs_Stack/arr_sync.sh once all arr containers are live" fi # ── Step 9b: Webhook setup ──────────────────────────────────────────────────────────────────── # Register the download webhook in each arr on both servers. Arrs must be running. # webhook_setup.sh handles local + SSH to remote in one call. echo "" echo "━━━ Step 9b — Webhook Setup ━━━" _webhook_script="$SCRIPTS_ROOT/Tools/webhook_setup.sh" if [[ "$SKIP_WEBHOOK_SETUP" == true ]]; then warn "Skipping (--skip-webhook-setup)" elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then warn "WEBHOOK_PORT=0 — webhook disabled, skipping" elif [[ ! -f "$_webhook_script" ]]; then warn "Tools/webhook_setup.sh not found — run manually after onboard" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run webhook_setup.sh (local + remote)" WEBHOOK_SETUP_OK=true elif bash "$_webhook_script" "${EXTRA_FLAGS[@]}"; then echo "Webhook setup complete ✅" WEBHOOK_SETUP_OK=true else warn "Webhook setup had errors — run Tools/webhook_setup.sh manually once arrs are settled" fi unset _webhook_script # ── Step 9c: Seed media shares on mirror ───────────────────────────────────────────────────── # arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but # have no files on disk. Without this rsync, every imported item looks missing and arrs # will immediately queue searches. --seed skips the empty-remote guard and does a clean push. echo "" echo "━━━ Step 9c — Media Share Seed ━━━" if [[ "$SKIP_MEDIA_SEED" == true ]]; then warn "Skipping (--skip-media-seed)" elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed" warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually" else echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while" _rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh" _seed_flags=(--seed) [[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run) [[ "$LOG_MODE" == true ]] && _seed_flags+=(--log) for _share in "${DAILY_SYNC_SHARES[@]}"; do echo " Seeding: $_share" if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then (( MEDIA_SEED_COUNT++ )) || true else warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed" fi done if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅" MEDIA_SEED_OK=true else warn "Media seed: no shares completed — check errors above" fi unset _rsync_script _seed_flags _share fi # ── Step 9d: Start webhook listener on mirror ───────────────────────────────────────────────── # Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is # already running — kick it now so events are captured immediately after onboard. echo "" echo "━━━ Step 9d — Webhook Listener (Mirror) ━━━" _listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh" if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then warn "Skipping (--skip-webhook-listener)" elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then warn "WEBHOOK_PORT=0 — webhook disabled, skipping" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start webhook listener on $MIRROR" WEBHOOK_LISTENER_OK=true elif timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \ "nohup bash '$_listener_script' > /var/log/varaverk/upgrade_webhook.log 2>&1 & echo started" \ 2>/dev/null | grep -q started; then echo "Webhook listener started on $MIRROR ✅" WEBHOOK_LISTENER_OK=true else warn "Could not start listener on $MIRROR — it will start automatically on next array restart" fi unset _listener_script # ── Step 10: Push master.conf to all listed hosts ───────────────────────────────────────────── # SSH is now established and all partners have the plugin installed. # Push the authoritative master.conf so every listed host is in sync immediately. echo "" echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━" if [[ "$ONBOARD_OK" == false ]]; then warn "Skipping — onboard did not complete" elif [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would push master.conf to all listed hosts" MASTER_PUSH_OK=true else push_output=$(platform_push_conf) push_rc=$? [[ -n "$push_output" ]] && echo "$push_output" platform_push_setup_state if [[ $push_rc -eq 0 ]]; then echo "master.conf sync complete ✅" MASTER_PUSH_OK=true else warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf" fi fi # ── Write Phase 2 completion state ──────────────────────────────────────────────────────────── [[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2 # ── Summary ─────────────────────────────────────────────────────────────────────────────────── END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)" echo " Mirror: $MIRROR ($MIRROR_IP)" [[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)" echo " Duration: $(format_duration $(( END - START )))" echo "" _ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; } _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")" echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")" echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")" echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )" echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")" echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )" echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")" echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )" echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")" echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )" echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")" echo " Step 9c — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" || echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")" )" echo " Step 9d — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")" echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )" echo "" if [[ "$ONBOARD_OK" == true ]]; then [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \ echo "$ICON_DONE DONE — partnership established ✅" echo "Verify with: Partnership/partnership_manager.sh --status" else error "Setup incomplete — resolve errors above and re-run" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" [[ "$ONBOARD_OK" == false ]] && exit 1 exit 0