1095 lines
56 KiB
Bash
Executable File
1095 lines
56 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Partnership Onboard ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Runs once on both servers to establish a new partnership. Role is detected
|
|
# automatically via detect_hosts() — no flags needed to declare which side you are.
|
|
# Run on the mirror first (generates its SSH key), then on the owner to complete
|
|
# setup remotely.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# MIRROR PATH (1 step)
|
|
# Step 1: SSH key setup — generate keypair, copy to owner, update conf
|
|
# Owner completes the rest remotely. Mirror is done.
|
|
#
|
|
# OWNER PATH (14 steps)
|
|
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
|
# Step 1b: Docker network — ensure varaverk docker network exists on mirror
|
|
# Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent)
|
|
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
|
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
|
# Mariadb/Redis health-checked before Authelia deploys
|
|
# Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing
|
|
# Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror
|
|
# Step 6: Stop mirror services — stop mirror's existing services containers before replacing
|
|
# Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start
|
|
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
|
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
|
# Step 9b: Webhook setup — register download webhook in arrs on both servers
|
|
# Step 9c: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the
|
|
# template ships closed so a fresh node cannot sync early.
|
|
# Ahead of the seed because the seed is itself an rsync and
|
|
# Tier 1 stops every rsync, including that one
|
|
# Step 9d: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed)
|
|
# prevents arrs treating every file as missing after bootstrap
|
|
# Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
|
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
|
# Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it
|
|
# would discover are actually deployed there
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Credentials never in SSH command strings
|
|
# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
|
|
# locally, SCPed to the remote, and executed there. Command-line args are never used
|
|
# to pass credentials — they'd appear in `ps` output and shell history on both servers.
|
|
#
|
|
# XML templates are the single source of truth for deployed containers
|
|
# The owner's templates-user/ XMLs define every container deployed on the mirror.
|
|
# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
|
|
# Docker Manager can manage the containers after onboard without additional config.
|
|
#
|
|
# Dependency ordering in the auth stack is owner-enforced
|
|
# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
|
|
# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
|
|
# the script waits for the container to be healthy before continuing. This is a remote
|
|
# health check — the container must be running (or report healthy) before the next
|
|
# dependent is deployed.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check
|
|
# All operations run as root — SSH key management, docker operations, conf updates.
|
|
#
|
|
# SSH timeout on all remote calls
|
|
# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
|
|
# slow or unreachable mirror.
|
|
#
|
|
# --dry-run shows exact actions without executing
|
|
# Every step prints what it would do. SCP, deploy, plugin install, arr sync —
|
|
# all dry-run safe.
|
|
#
|
|
# Step skip flags for partial re-runs
|
|
# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
|
|
# resuming after a partial failure without re-running completed steps.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_PARTNERSHIP_AUTH_STACK
|
|
# XML filenames (from this server's templates-user/) to push and deploy on the
|
|
# mirror as its auth stack. Order matters: database deps before Authelia.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_REPLACE_CONTAINERS
|
|
# Containers to stop on the mirror before deploying the auth stack.
|
|
# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
|
|
# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
|
|
# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
|
|
# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
|
|
# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# HOST*_PARTNERSHIP_ARR_STACK
|
|
# XML filenames to push and deploy on the mirror as its arr stack.
|
|
# Leave empty to skip arr stack deploy.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
|
|
# Arr containers to stop on the mirror before deploying the arr stack.
|
|
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# HOST*_PARTNERSHIP_SERVICES_STACK
|
|
# XML filenames to push and deploy on the mirror as its shared services stack.
|
|
# Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS
|
|
# Services containers to stop on the mirror before deploying the services stack.
|
|
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# Partnership/partnership_onboard.sh
|
|
# Full onboard — role detected automatically
|
|
#
|
|
# Partnership/partnership_onboard.sh --dry-run
|
|
# Preview all steps without making changes
|
|
#
|
|
# Partnership/partnership_onboard.sh --log
|
|
# Verbose per-step output
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-ssh
|
|
# Skip SSH key setup (key already in place)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-share-setup
|
|
# Skip share creation on mirror (shares already exist)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-auth-stack
|
|
# Skip auth stack stop + deploy (Steps 3-4)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-arr-stack
|
|
# Skip arr stack stop + deploy (Steps 4-5)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-services-stack
|
|
# Skip services stack stop + deploy (Steps 6-7)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-arr-sync
|
|
# Skip arr library bootstrap (Step 9)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-webhook-setup
|
|
# Skip webhook registration in arrs (Step 9b)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-media-seed
|
|
# Skip initial media share rsync to mirror (Step 9d)
|
|
# Use when mirror already has files or you want to seed manually
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-webhook-listener
|
|
# Skip starting webhook listener on mirror (Step 9e)
|
|
# Listener will start automatically on next array restart
|
|
#
|
|
# Partnership/partnership_onboard.sh --no-arm
|
|
# Leave RSYNC_ENABLED / CONF_SYNC_ENABLED / ARR_SYNC_ENABLED as they are (Step 9c).
|
|
# Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose
|
|
# shares are not populated yet.
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase1-only
|
|
# OWNER only: SSH key exchange, conf push, docker network, partner conf cache.
|
|
# Safe to run before HOST2 has Varaverk — all of it needs docker and SSH, not the plugin.
|
|
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase2-only
|
|
# OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
|
|
# by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
|
|
# Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
SSH_TIMEOUT=15
|
|
|
|
source "$SCRIPTS_ROOT/load_config.sh"
|
|
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
|
|
|
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
|
SKIP_SSH=false
|
|
SKIP_SHARE_SETUP=false
|
|
SKIP_AUTH_STACK=false
|
|
SKIP_ARR_STACK=false
|
|
SKIP_SERVICES_STACK=false
|
|
SKIP_ARR_SYNC=false
|
|
SKIP_WEBHOOK_SETUP=false
|
|
SKIP_MEDIA_SEED=false
|
|
SKIP_WEBHOOK_LISTENER=false
|
|
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
|
|
# MIRROR: SSH key install only, no owner notification
|
|
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
|
SKIP_ARM=false # leave the sync gates as they are — see Step 9c
|
|
FILTERED_ARGS=()
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--skip-ssh) SKIP_SSH=true ;;
|
|
--skip-share-setup) SKIP_SHARE_SETUP=true ;;
|
|
--skip-auth-stack) SKIP_AUTH_STACK=true ;;
|
|
--skip-arr-stack) SKIP_ARR_STACK=true ;;
|
|
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
|
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
|
--skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;;
|
|
--skip-media-seed) SKIP_MEDIA_SEED=true ;;
|
|
--skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;;
|
|
--phase1-only) PHASE1_ONLY=true ;;
|
|
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
|
--no-arm) SKIP_ARM=true ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
partnership_resolve_roles
|
|
|
|
EXTRA_FLAGS=()
|
|
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
|
[[ "$ENABLE_LOGGING" == true ]] && EXTRA_FLAGS+=("--log")
|
|
|
|
START=$(date +%s)
|
|
|
|
# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
|
|
write_onboard_phase() {
|
|
local target_id="$1" phase="$2"
|
|
local key="${target_id}_PHASE${phase}_DONE"
|
|
local state_file="$(platform_setup_db_path)"
|
|
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
|
set_state_var "$state_file" "$key" "true"
|
|
platform_push_setup_state
|
|
}
|
|
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
|
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
|
echo ""
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
|
|
#
|
|
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
|
|
# use the same convention), and reads the named config array from the mirror's own conf.
|
|
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
|
|
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
|
|
#
|
|
# deploy_container_from_xml() already stops/removes containers with the same name as what's
|
|
# being deployed. This step handles containers with DIFFERENT names that conflict.
|
|
# ==============================================================================================
|
|
stop_mirror_stack() {
|
|
local config_var="$1" label="$2"
|
|
local -a to_stop=()
|
|
|
|
mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$')
|
|
|
|
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
|
log "No $label containers to stop on $MIRROR — skipping"
|
|
return 0
|
|
fi
|
|
|
|
log "Stopping $label on $MIRROR: ${to_stop[*]}"
|
|
for container in "${to_stop[@]}"; do
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would stop + rm $container on $MIRROR"
|
|
continue
|
|
fi
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"docker stop '$container' 2>/dev/null
|
|
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
|
grep -q removed && \
|
|
echo " $container removed ✅" || \
|
|
log " $container not found on $MIRROR — skipping"
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
if [[ "$AM_MIRROR" == true ]]; then
|
|
echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
|
|
echo ""
|
|
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
|
echo ""
|
|
|
|
if [[ "$SKIP_SSH" == true ]]; then
|
|
warn "Skipping SSH setup (--skip-ssh)"
|
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "SSH key ready ✅"
|
|
else
|
|
error "SSH key setup failed"
|
|
exit 1
|
|
fi
|
|
|
|
# Stop after the key when asked. ssh_setup.sh runs ssh-copy-id, which prompts for the
|
|
# owner's root password on a first install — answerable in a terminal, never from the
|
|
# WebGUI button, which is why the mirror's panel sends the operator to a terminal for
|
|
# exactly this step and nothing more. The flag was parsed but only ever honoured on the
|
|
# owner path, so a mirror asked for phase 1 silently ran the whole thing.
|
|
if [[ "$PHASE1_ONLY" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MIRROR PHASE 1 COMPLETE ━━━━━"
|
|
echo " SSH key: ready"
|
|
echo " Next: press ▶ Onboard on the Partnership tab to notify $OWNER"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
|
echo ""
|
|
|
|
OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
|
PHASE2_TRIGGERED=false
|
|
|
|
if [[ -n "$OWNER_IP" ]]; then
|
|
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
|
|
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2 via run_job.sh"
|
|
PHASE2_TRIGGERED=true
|
|
else
|
|
# Launched through run_job.sh, the same path cron and api/run.php use, so Phase 2
|
|
# gets a stat file, a job log and a Scheduler entry on the owner. It used to be a
|
|
# bare `nohup … > /tmp/vv_phase2_onboard.log`, which ran fine and left the owner's
|
|
# entire half of onboarding invisible to its own WebGUI — no job record, nothing
|
|
# under /var/log/varaverk, nothing for the operator to look at when asking why
|
|
# pressing Onboard here appeared to do nothing.
|
|
#
|
|
# setsid, not bare nohup: the job must lead its own process group so api/stop.php
|
|
# can signal the whole tree. api/run.php carries the same note for the same reason.
|
|
#
|
|
# Reported triggered only after the stat file proves run_job.sh actually started.
|
|
# The old `& echo triggered` printed unconditionally — it would have claimed success
|
|
# for a path that does not exist on the owner, which is exactly the failure mode a
|
|
# mirror in a different storage mode hits.
|
|
# Its own timeout, not SSH_TIMEOUT: the remote waits for the runner to prove itself,
|
|
# and 15s would cut that short and report a healthy launch as a failure.
|
|
_phase2_out=$(timeout 40 ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
|
"bash -s -- $(printf '%q' "$OWNER_SCRIPTS_DIR")" 2>/dev/null <<'PHASE2_TRIGGER'
|
|
sd="$1"
|
|
runner="$sd/Plugin/unraid/run_job.sh"
|
|
script="$sd/Partnership/partnership_onboard.sh"
|
|
stat_file="/var/log/varaverk/Partnership/partnership_onboard.json"
|
|
|
|
[ -f "$runner" ] || { echo "missing-runner:$runner"; exit 1; }
|
|
[ -f "$script" ] || { echo "missing-script:$script"; exit 1; }
|
|
|
|
# Absolute, not "newer than the file we saw a moment ago". A previous run's stat file rewritten
|
|
# inside the same second would compare equal and read as a failed launch.
|
|
t0=$(date +%s)
|
|
|
|
setsid nohup bash "$runner" "Partnership/partnership_onboard.sh" "$script" \
|
|
--manual --phase2-only >/dev/null 2>&1 </dev/null &
|
|
|
|
# run_job.sh writes the stat file before it execs the script, so a stat file dated at or after
|
|
# the moment we launched is proof the runner is alive — rather than proof the ssh call returned.
|
|
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
|
|
sleep 1
|
|
[ -f "$stat_file" ] || continue
|
|
now=$(date -r "$stat_file" +%s 2>/dev/null || echo 0)
|
|
[ "$now" -ge "$t0" ] && { echo triggered; exit 0; }
|
|
done
|
|
echo start-failed
|
|
exit 1
|
|
PHASE2_TRIGGER
|
|
)
|
|
# Failure patterns are matched first, and they echo a path back. An owner whose
|
|
# SCRIPTS_DIR happened to contain the word "triggered" would otherwise satisfy a
|
|
# leading *triggered* glob and report success for a launch that never happened.
|
|
case "$_phase2_out" in
|
|
missing-runner:*)
|
|
error "Phase 2 not started — no run_job.sh at ${_phase2_out#missing-runner:} on $OWNER"
|
|
;;
|
|
missing-script:*)
|
|
error "Phase 2 not started — no partnership_onboard.sh at ${_phase2_out#missing-script:} on $OWNER"
|
|
;;
|
|
start-failed)
|
|
error "Phase 2 launch on $OWNER did not produce a job record — check run_job.sh there"
|
|
;;
|
|
triggered)
|
|
echo "Phase 2 triggered on $OWNER ✅"
|
|
log "Watch on $OWNER: Scheduler tab, or tail -f /var/log/varaverk/Partnership/partnership_onboard.log"
|
|
PHASE2_TRIGGERED=true
|
|
;;
|
|
*)
|
|
warn "Could not auto-trigger Phase 2 on $OWNER"
|
|
;;
|
|
esac
|
|
fi
|
|
else
|
|
warn "Cannot resolve $OWNER Tailscale IP"
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
|
|
echo " SSH key: ready"
|
|
echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
|
|
if [[ "$PHASE2_TRIGGERED" == false ]]; then
|
|
echo ""
|
|
echo " Run manually on $OWNER:"
|
|
echo " bash Partnership/partnership_onboard.sh --phase2-only"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
|
log "Mirror: $MIRROR ($MIRROR_IP)"
|
|
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
|
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
|
echo ""
|
|
|
|
STEP_SSH_OK=false
|
|
STEP_NETWORK_OK=false
|
|
LOCAL_SETUP_OK=true # partnership_manager --local-only; the summary claimed done ✅ regardless
|
|
PHASE1_NET_OK=false # Phase 1 only — network created on the mirror before any deploy
|
|
PHASE1_CACHE_OK=false # Phase 1 only — our conf pushed into the mirror's RAM cache
|
|
STEP_STOP_AUTH_OK=true
|
|
STEP_AUTH_OK=true
|
|
AUTH_DEPLOYED=0
|
|
AUTH_FAILED=0
|
|
STEP_STOP_ARR_OK=true
|
|
STEP_ARR_OK=true
|
|
ARR_DEPLOYED=0
|
|
ARR_FAILED=0
|
|
STEP_STOP_SERVICES_OK=true
|
|
STEP_SERVICES_OK=true
|
|
SERVICES_DEPLOYED=0
|
|
SERVICES_FAILED=0
|
|
ONBOARD_OK=false
|
|
ARR_SYNC_OK=false
|
|
WEBHOOK_SETUP_OK=false
|
|
MEDIA_SEED_OK=false
|
|
MEDIA_SEED_COUNT=0
|
|
WEBHOOK_LISTENER_OK=false
|
|
MASTER_PUSH_OK=false
|
|
|
|
# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
|
|
# Skipped when --phase2-only (SSH was already done in Phase 1).
|
|
echo "━━━ Step 1 — SSH Key Setup ━━━"
|
|
|
|
if [[ "$SKIP_SSH" == true ]]; then
|
|
warn "Skipping (--skip-ssh)"
|
|
STEP_SSH_OK=true
|
|
elif [[ "$PHASE1_ONLY" == true ]]; then
|
|
# Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
|
|
# hanging for a password prompt with no TTY.
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
|
|
echo "SSH to $MIRROR already works ✅ — skipping key install"
|
|
STEP_SSH_OK=true
|
|
else
|
|
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
|
|
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "SSH keys ready ✅"
|
|
STEP_SSH_OK=true
|
|
else
|
|
# Soft-fail: generate key locally if not present, then tell user to install manually
|
|
warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
|
|
if [[ -f "$SSH_KEY" ]]; then
|
|
log "Local key exists at: $SSH_KEY"
|
|
else
|
|
bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
|
|
fi
|
|
if [[ -f "${SSH_KEY}.pub" ]]; then
|
|
echo ""
|
|
echo " Install this key on $MIRROR to complete SSH setup:"
|
|
echo " ┌─────────────────────────────────────────────────────"
|
|
cat "${SSH_KEY}.pub" | sed 's/^/ │ /'
|
|
echo " └─────────────────────────────────────────────────────"
|
|
echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
|
echo " Then click 'Push Conf' in the Partnership tab."
|
|
# Write key-ready flag so UI can show the manual-install state
|
|
[[ "$DRY_RUN" == false ]] && {
|
|
kflag="${MIRROR_ID}_KEY_READY"
|
|
_setup_f="$(platform_setup_db_path)"
|
|
set_state_var "$_setup_f" "$kflag" "true"
|
|
}
|
|
fi
|
|
STEP_SSH_OK=false
|
|
fi
|
|
fi
|
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "SSH keys ready ✅"
|
|
STEP_SSH_OK=true
|
|
else
|
|
error "SSH key setup failed — aborting"
|
|
error "Re-run or use --skip-ssh if key is already set up"
|
|
exit 1
|
|
fi
|
|
|
|
# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
|
|
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
|
|
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
|
|
if [[ "$PHASE1_ONLY" == true ]]; then
|
|
if [[ "$STEP_SSH_OK" == false ]]; then
|
|
# SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
|
|
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
|
echo ""
|
|
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
|
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
|
LOCAL_SETUP_OK=false
|
|
warn "Local setup had issues — check partnership_manager.sh output above"
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
|
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
|
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
|
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " ACTION NEEDED: install the key on $MIRROR:"
|
|
echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
|
echo " Then click 'Push Conf' in Partnership tab, or run:"
|
|
echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ Phase 1 — Conf Push ━━━"
|
|
|
|
CONF_PUSH_OK=false
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would push master.conf + state file to $MIRROR"
|
|
CONF_PUSH_OK=true
|
|
else
|
|
push_output=$(platform_push_conf)
|
|
push_rc=$?
|
|
[[ -n "$push_output" ]] && echo "$push_output"
|
|
platform_push_setup_state
|
|
if [[ $push_rc -eq 0 ]]; then
|
|
echo "Conf push complete ✅"
|
|
CONF_PUSH_OK=true
|
|
else
|
|
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 1 — Docker network on the mirror ────────────────────────────────────────────────
|
|
# Here, not only in Step 1b, because a --phase1-only run exits above and never reaches it.
|
|
# The mirror needs docker, not Varaverk, so this works before the plugin is installed — and
|
|
# creating the network now means it is in place long before the first container is deployed
|
|
# against it. Deploying against a missing network is what left twelve containers stuck in
|
|
# `Created`, so the earliest safe moment is the right one.
|
|
echo ""
|
|
echo "━━━ Phase 1 — Docker Network ($MIRROR) ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would create any networks the stack templates reference on $MIRROR"
|
|
PHASE1_NET_OK=true
|
|
elif ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
PHASE1_NET_OK=true
|
|
else
|
|
warn "Network prep incomplete on $MIRROR — Step 1b retries this during Phase 2"
|
|
fi
|
|
|
|
# ── Phase 1 — Partner conf cache ──────────────────────────────────────────────────────────
|
|
# CONF_SYNC_ENABLED is armed here rather than waiting for Step 9c. It is the safe one of the
|
|
# three gates — it moves no data, it copies each side's host*.conf into the other's tmpfs so
|
|
# partner vars resolve — and the moment SSH works is the moment that should start. Leaving it
|
|
# until Phase 2 meant conf_sync.sh, which sources the conf fresh in its own process, exited
|
|
# on a closed gate every time it was called before then.
|
|
#
|
|
# Push always; pull only if the mirror actually has a conf yet. Before HOST2 installs Varaverk
|
|
# there is nothing to pull, and an unconditional pull would count a failure and notify about
|
|
# a condition that is simply "HOST2 is not installed". The pull lands on the re-run after the
|
|
# install — the same --phase1-only --skip-ssh the operator uses to push conf again.
|
|
echo ""
|
|
echo "━━━ Phase 1 — Partner Conf Cache ━━━"
|
|
_conf_sync_script="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would arm CONF_SYNC_ENABLED and cache confs with $MIRROR"
|
|
PHASE1_CACHE_OK=true
|
|
elif [[ ! -f "$_conf_sync_script" ]]; then
|
|
warn "conf_sync.sh not found — skipping partner conf cache"
|
|
else
|
|
set_conf_bool CONF_SYNC_ENABLED "true" "$SCRIPTS_ROOT/Configurations/master.conf" \
|
|
|| warn "Could not arm CONF_SYNC_ENABLED — cache step may no-op"
|
|
|
|
if bash "$_conf_sync_script" --push-only; then
|
|
PHASE1_CACHE_OK=true
|
|
else
|
|
warn "Could not push our conf to $MIRROR"
|
|
fi
|
|
|
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '${_mirror_sd}/Configurations/${MIRROR_ID,,}.conf' ]" 2>/dev/null; then
|
|
bash "$_conf_sync_script" --pull-only || warn "Could not pull ${MIRROR_ID,,}.conf from $MIRROR"
|
|
else
|
|
echo " $MIRROR has no ${MIRROR_ID,,}.conf yet — it will cache once Varaverk is installed there"
|
|
fi
|
|
unset _mirror_sd
|
|
fi
|
|
unset _conf_sync_script
|
|
|
|
# HOST1 local setup — runs immediately without needing HOST2
|
|
echo ""
|
|
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
|
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
|
LOCAL_SETUP_OK=false
|
|
warn "Local setup had issues — check partnership_manager.sh output above"
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
|
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
|
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
|
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
|
echo " Network: $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )"
|
|
echo " Conf cache: $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )"
|
|
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
|
# The push no longer waits for a Varaverk install on the far side. It resolves the partner's
|
|
# conf directory across both layouts and creates the internal one if neither exists, so the
|
|
# conf arrives BEFORE the plugin — which is the order that makes it useful. The .plg only
|
|
# seeds master.conf from the template when none is present, so what lands here survives the
|
|
# install and the wizard reads its identity straight out of it.
|
|
if [[ "$CONF_PUSH_OK" == true ]]; then
|
|
echo " master.conf is on $MIRROR — the wizard will find it and take the partner path,"
|
|
echo " already knowing $MY_ID and $MIRROR_ID. If the operator picks flash storage there,"
|
|
echo " storage_migrate.sh moves the conf to appdata with the rest of the install."
|
|
else
|
|
echo " master.conf was NOT delivered. Phase 1 seeds it into a bare host, so this is a"
|
|
echo " real failure, not the pre-install state — check SSH and that /boot is writable"
|
|
echo " on $MIRROR. Retry with:"
|
|
echo " • bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
|
echo " • or push from Scheduler → master.conf → Save Conf"
|
|
echo " • or, once the plugin is installed, 'Pull from HOST1' on HOST2's Setup tab"
|
|
fi
|
|
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Step 1b: Ensure custom Docker network exists on mirror ────────────────────────────────────
|
|
# Must run before any container deploy — docker create fails if the network is missing.
|
|
echo ""
|
|
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
|
|
|
|
# Two halves, and the first is the one that matters for a fresh mirror.
|
|
#
|
|
# ensure_stack_networks_on_remote reads the networks out of the XMLs this onboard is about to
|
|
# push and creates any that are missing on the mirror. It does not consult the mirror's conf,
|
|
# because on a fresh node that array is the template default — a single commented-out entry —
|
|
# and an empty list is indistinguishable from "no networks needed". The result was every
|
|
# container in both stacks created against a network that did not exist.
|
|
#
|
|
# docker_network_connect.sh still runs afterwards: it is what *connects* the mirror's own
|
|
# listed containers to its own listed networks, which is a different job and remains the
|
|
# mirror's to declare.
|
|
if ! ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
warn "One or more stack networks could not be prepared on $MIRROR — deploys below may fail"
|
|
fi
|
|
|
|
_net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
|
|
STEP_NETWORK_OK=true
|
|
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"bash '$_net_script'" 2>/dev/null; then
|
|
echo "Docker network ready on $MIRROR ✅"
|
|
STEP_NETWORK_OK=true
|
|
else
|
|
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
|
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
|
fi
|
|
|
|
# ── Step 1c: Share setup ─────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 1c — Share Setup (Mirror) ━━━"
|
|
|
|
if [[ "$SKIP_SHARE_SETUP" == true ]]; then
|
|
warn "Skipping (--skip-share-setup)"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
bash "$SCRIPT_DIR/share_setup.sh" --dry-run
|
|
else
|
|
bash "$SCRIPT_DIR/share_setup.sh"
|
|
fi
|
|
|
|
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
|
|
|
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
|
warn "Skipping (--skip-auth-stack)"
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
|
|
fi
|
|
|
|
# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
|
warn "Skipping (--skip-auth-stack)"
|
|
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
|
|
warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
|
|
warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
|
|
STEP_AUTH_OK=false
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_AUTH_STACK
|
|
AUTH_DEPLOYED=$_STACK_DEPLOYED
|
|
AUTH_FAILED=$_STACK_FAILED
|
|
echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
|
|
[[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
|
|
fi
|
|
|
|
# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━"
|
|
|
|
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
|
warn "Skipping (--skip-arr-stack)"
|
|
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
|
|
log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
|
|
SKIP_ARR_STACK=true
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
|
|
fi
|
|
|
|
# ── Step 5: Deploy arr stack on mirror ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
|
warn "Skipping (--skip-arr-stack)"
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_ARR_STACK
|
|
ARR_DEPLOYED=$_STACK_DEPLOYED
|
|
ARR_FAILED=$_STACK_FAILED
|
|
echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
|
|
[[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
|
|
fi
|
|
|
|
# ── Step 6: Stop mirror's existing services stack ─────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 6 — Stop Mirror Services Stack ━━━"
|
|
|
|
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
|
warn "Skipping (--skip-services-stack)"
|
|
elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then
|
|
log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy"
|
|
SKIP_SERVICES_STACK=true
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack"
|
|
fi
|
|
|
|
# ── Step 7: Deploy services stack on mirror ───────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
|
warn "Skipping (--skip-services-stack)"
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_SERVICES_STACK
|
|
SERVICES_DEPLOYED=$_STACK_DEPLOYED
|
|
SERVICES_FAILED=$_STACK_FAILED
|
|
echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed"
|
|
[[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false
|
|
fi
|
|
|
|
# ── Step 8: Partnership onboard ───────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 8 — Partnership Onboard ━━━"
|
|
|
|
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
|
|
echo "Partnership onboard complete ✅"
|
|
ONBOARD_OK=true
|
|
else
|
|
error "Partnership onboard failed"
|
|
ONBOARD_OK=false
|
|
fi
|
|
|
|
# ── Step 9: Arr library bootstrap ─────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 9 — Arr Library Bootstrap ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete"
|
|
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
|
|
warn "Skipping (--skip-arr-sync)"
|
|
elif [[ ! -f "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ]]; then
|
|
warn "arr_sync.sh not found — run Arrs_Stack/arr_sync.sh manually once arrs are live"
|
|
elif bash "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "Arr bootstrap complete ✅"
|
|
ARR_SYNC_OK=true
|
|
else
|
|
warn "Arr sync had errors — partnership still valid"
|
|
warn "Re-run Arrs_Stack/arr_sync.sh once all arr containers are live"
|
|
fi
|
|
|
|
# ── Step 9b: Webhook setup ────────────────────────────────────────────────────────────────────
|
|
# Register the download webhook in each arr on both servers. Arrs must be running.
|
|
# webhook_setup.sh handles local + SSH to remote in one call.
|
|
echo ""
|
|
echo "━━━ Step 9b — Webhook Setup ━━━"
|
|
|
|
_webhook_script="$SCRIPTS_ROOT/Tools/webhook_setup.sh"
|
|
if [[ "$SKIP_WEBHOOK_SETUP" == true ]]; then
|
|
warn "Skipping (--skip-webhook-setup)"
|
|
elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then
|
|
warn "WEBHOOK_PORT=0 — webhook disabled, skipping"
|
|
elif [[ ! -f "$_webhook_script" ]]; then
|
|
warn "Tools/webhook_setup.sh not found — run manually after onboard"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run webhook_setup.sh (local + remote)"
|
|
WEBHOOK_SETUP_OK=true
|
|
elif bash "$_webhook_script" "${EXTRA_FLAGS[@]}"; then
|
|
echo "Webhook setup complete ✅"
|
|
WEBHOOK_SETUP_OK=true
|
|
else
|
|
warn "Webhook setup had errors — run Tools/webhook_setup.sh manually once arrs are settled"
|
|
fi
|
|
unset _webhook_script
|
|
|
|
# ── Step 9c: Arm the sync gates ───────────────────────────────────────────────────────────────
|
|
# master.conf.template ships a fresh node inert — RSYNC_ENABLED, CONF_SYNC_ENABLED and
|
|
# ARR_SYNC_ENABLED all false — because a node that has just been seeded has empty shares and no
|
|
# verified partner, and those two facts are what make unattended syncing safe to do.
|
|
#
|
|
# A successful onboard is the event that makes them true. Without this step the defaults were a
|
|
# one-way door: the node stayed inert for ever and somebody had to remember to hand-edit three
|
|
# toggles, on the machine where forgetting looks exactly like everything working.
|
|
#
|
|
# Ahead of the media seed, which it used to follow. The seed calls Rsync/rsync.sh, and Tier 1
|
|
# stops *every* rsync — so with the gates still closed all of it exited cleanly having moved
|
|
# nothing, and, because a clean exit is exit 0, the seed counted each share as a success and
|
|
# reported "14/14 shares ✅" over an empty mirror.
|
|
#
|
|
# Still ahead of Step 10 on purpose. The push carries the owner's master.conf to every listed
|
|
# host, so arming before it means both sides come up agreeing about whether sync is on; arming
|
|
# after the push would leave the mirror a version behind until the next conf save.
|
|
#
|
|
# Owner only — the mirror receives these values in the push rather than deciding for itself.
|
|
ARM_OK=true
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 9c — Arm Sync Gates ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete, leaving the gates closed"
|
|
ARM_OK=false
|
|
elif [[ "$SKIP_ARM" == true ]]; then
|
|
echo " --no-arm — leaving the sync gates as they are"
|
|
elif [[ "$AM_OWNER" != true ]]; then
|
|
echo " mirror — the owner's push decides these"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would set RSYNC_ENABLED, CONF_SYNC_ENABLED, ARR_SYNC_ENABLED to true"
|
|
else
|
|
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
|
if [[ ! -f "$_master_conf" ]]; then
|
|
warn "master.conf not found at $_master_conf — gates left closed"
|
|
ARM_OK=false
|
|
else
|
|
cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)"
|
|
for _gate in RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED; do
|
|
set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false
|
|
done
|
|
# Tier 1 is now open, so say what that actually turned on rather than leaving it implied.
|
|
echo " rsync tier 2 gates were left as configured — check them before the first run"
|
|
unset _gate
|
|
fi
|
|
unset _master_conf
|
|
fi
|
|
|
|
# ── Step 9d: Seed media shares on mirror ─────────────────────────────────────────────────────
|
|
# arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but
|
|
# have no files on disk. Without this rsync, every imported item looks missing and arrs
|
|
# will immediately queue searches. --seed skips the empty-remote guard and does a clean push.
|
|
echo ""
|
|
echo "━━━ Step 9d — Media Share Seed ━━━"
|
|
|
|
# Read the gate back off disk rather than trusting the value sourced at startup: Step 9c has
|
|
# just rewritten master.conf, and each rsync.sh below is a separate process that will source
|
|
# the file fresh. A closed gate here means every share would exit 0 having moved nothing, so
|
|
# say so once and skip, instead of counting fourteen no-ops as fourteen successes.
|
|
#
|
|
# The trailing comment is cut before the value is read. master.conf writes this gate as
|
|
# RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
|
# so stopping at `cut -d= -f2` and squeezing whitespace yields "true#Tier1—globalgate,…",
|
|
# which is not "true" — and the seed would then skip itself on every single run, including
|
|
# the ones where the gate is open.
|
|
_rsync_gate=$(grep -m1 -E '^[[:space:]]*RSYNC_ENABLED=' "$SCRIPTS_ROOT/Configurations/master.conf" 2>/dev/null \
|
|
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
|
|
|
if [[ "$SKIP_MEDIA_SEED" == true ]]; then
|
|
warn "Skipping (--skip-media-seed)"
|
|
elif [[ "$DRY_RUN" == false && "$_rsync_gate" != "true" ]]; then
|
|
warn "RSYNC_ENABLED is '$_rsync_gate' — skipping media seed, rsync.sh would move nothing"
|
|
warn "Arm it and re-run: Rsync/rsync.sh <share> --seed"
|
|
elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
|
|
warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed"
|
|
warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually"
|
|
else
|
|
echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while"
|
|
_rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh"
|
|
_seed_flags=(--seed)
|
|
[[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run)
|
|
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
|
for _share in "${DAILY_SYNC_SHARES[@]}"; do
|
|
echo " Seeding: $_share"
|
|
if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then
|
|
(( MEDIA_SEED_COUNT++ )) || true
|
|
else
|
|
warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
|
|
fi
|
|
done
|
|
if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then
|
|
echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅"
|
|
MEDIA_SEED_OK=true
|
|
else
|
|
warn "Media seed: no shares completed — check errors above"
|
|
fi
|
|
unset _rsync_script _seed_flags _share
|
|
fi
|
|
|
|
# ── Step 9e: Start webhook listener on mirror ─────────────────────────────────────────────────
|
|
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
|
|
# already running — kick it now so events are captured immediately after onboard.
|
|
echo ""
|
|
echo "━━━ Step 9e — Webhook Listener (Mirror) ━━━"
|
|
|
|
_listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh"
|
|
if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then
|
|
warn "Skipping (--skip-webhook-listener)"
|
|
elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then
|
|
warn "WEBHOOK_PORT=0 — webhook disabled, skipping"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would start webhook listener on $MIRROR"
|
|
WEBHOOK_LISTENER_OK=true
|
|
elif timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"nohup bash '$_listener_script' > /var/log/varaverk/upgrade_webhook.log 2>&1 & echo started" \
|
|
2>/dev/null | grep -q started; then
|
|
echo "Webhook listener started on $MIRROR ✅"
|
|
WEBHOOK_LISTENER_OK=true
|
|
else
|
|
warn "Could not start listener on $MIRROR — it will start automatically on next array restart"
|
|
fi
|
|
unset _listener_script
|
|
|
|
# ── Step 10: Push master.conf to all listed hosts ─────────────────────────────────────────────
|
|
# SSH is now established and all partners have the plugin installed.
|
|
# Push the authoritative master.conf so every listed host is in sync immediately.
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would push master.conf to all listed hosts"
|
|
MASTER_PUSH_OK=true
|
|
else
|
|
push_output=$(platform_push_conf)
|
|
push_rc=$?
|
|
[[ -n "$push_output" ]] && echo "$push_output"
|
|
platform_push_setup_state
|
|
if [[ $push_rc -eq 0 ]]; then
|
|
echo "master.conf sync complete ✅"
|
|
MASTER_PUSH_OK=true
|
|
else
|
|
warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
|
fi
|
|
fi
|
|
|
|
# ── Step 11: Service discovery on the mirror ──────────────────────────────────────────────────
|
|
# Deliberately last. conf_populate.sh fills host*.conf from what it can actually find running —
|
|
# arr API keys, container names, URLs — and until Step 3 and Step 5 deployed the auth and arr
|
|
# stacks there was nothing on the mirror to find. The wizard runs it during first-run setup,
|
|
# which on a fresh mirror is precisely the moment the machine is still empty, so everything it
|
|
# could have discovered was discovered as absent.
|
|
#
|
|
# No --overwrite: it only fills blanks, so anything the operator set by hand survives. --no-push
|
|
# because Step 10 above has just pushed conf; letting discovery push again would race it.
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 11 — Service Discovery ($MIRROR) ━━━"
|
|
|
|
POPULATE_OK=false
|
|
# MIRROR_IP, not MIRROR_REACHABLE — the latter is partnership_offboard.sh's variable and does not
|
|
# exist in this script, so the test was always true against an empty string and Step 11 reported
|
|
# "skipped (unreachable)" on a mirror it had just deployed twelve containers to.
|
|
if [[ -z "${MIRROR_IP:-}" ]]; then
|
|
warn "$MIRROR has no resolved IP — skipping discovery, run Deployment/conf_populate.sh there later"
|
|
POPULATE_OK=skipped
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run conf_populate.sh --no-push on $MIRROR"
|
|
POPULATE_OK=true
|
|
else
|
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
|
_pop_script="${_mirror_sd}/Deployment/conf_populate.sh"
|
|
if timeout 180 ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '$_pop_script' ] || { echo missing; exit 127; }; bash '$_pop_script' --no-push" 2>/dev/null; then
|
|
echo "Discovery complete on $MIRROR ✅"
|
|
POPULATE_OK=true
|
|
else
|
|
warn "Discovery failed on $MIRROR — run $_pop_script there by hand"
|
|
fi
|
|
unset _mirror_sd _pop_script
|
|
fi
|
|
|
|
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
|
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
|
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
|
[[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
|
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
|
|
|
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
|
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
|
|
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
|
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
|
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
|
echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
|
|
echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")"
|
|
echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )"
|
|
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
|
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
|
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
|
|
echo " Step 9c — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )"
|
|
echo " Step 9d — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" \
|
|
|| { [[ "$DRY_RUN" == false && "${_rsync_gate:-}" != "true" ]] && echo "skipped (rsync gate closed)" \
|
|
|| echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")"; } )"
|
|
echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
|
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
|
echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )"
|
|
echo ""
|
|
|
|
if [[ "$ONBOARD_OK" == true ]]; then
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
|
echo "$ICON_DONE DONE — partnership established ✅"
|
|
echo "Verify with: Partnership/partnership_manager.sh --status"
|
|
else
|
|
error "Setup incomplete — resolve errors above and re-run"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
[[ "$ONBOARD_OK" == false ]] && exit 1
|
|
exit 0
|