Gmer4Lfe
0447fa6f86
Comments asserting a partner is down, being rebuilt or not yet installed are true on the day they are written and wrong afterwards, and nothing prompts anyone to revisit them. Behaviour is described generically; current state belongs in operator memory, not in tracked files.
1380 lines
73 KiB
Bash
Executable File
1380 lines
73 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Partnership Onboard ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Runs once on both servers to establish a new partnership. Role is detected
|
|
# automatically via detect_hosts() — no flags needed to declare which side you are.
|
|
# Run on the mirror first (generates its SSH key), then on the owner to complete
|
|
# setup remotely.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# MIRROR PATH (1 step)
|
|
# Step 1: SSH key setup — generate keypair, copy to owner, update conf
|
|
# Owner completes the rest remotely. Mirror is done.
|
|
#
|
|
# OWNER PATH (14 steps)
|
|
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
|
# Step 1b: Docker network — ensure varaverk docker network exists on mirror
|
|
# Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent)
|
|
# Step 1c2:Permit mirror — drop the mirror from the partnership blocklist a previous
|
|
# offboard wrote, or Step 1e's rsync is refused
|
|
# Step 1d: Sync gates — Tier 1 + CONF_SYNC + ARR_SYNC on, every Tier 2 rsync gate
|
|
# off. Here, not at the end, because Step 1e is an rsync
|
|
# Step 1e: Auth appdata — rsync PARTNERSHIP_PROVISION_SHARES to the mirror BEFORE the
|
|
# containers that read it exist. The only rsync an onboard does
|
|
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
|
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
|
# Mariadb/Redis health-checked before Authelia deploys
|
|
# Step 4: Stop mirror arr — stop mirror's existing arr containers before replacing
|
|
# Step 5: Deploy arr stack — push arr XMLs, pull images, create + start on mirror
|
|
# Step 6: Stop mirror services — stop mirror's existing services containers before replacing
|
|
# Step 7: Deploy services stack — push Emby/Jellyfin/Seerr XMLs, pull images, create + start
|
|
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
|
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
|
# Step 9b: Webhook setup — register download webhook in arrs on both servers
|
|
# Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
|
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
|
# Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it
|
|
# would discover are actually deployed there
|
|
# Step 12: Container grouping — reproduce this host's folder layout on the mirror (Arrs
|
|
# Stack, Networking, Databases…) for the containers deployed
|
|
# there; only unfiled or PARTNERSHIP_FALLBACK_ONLY entries go
|
|
# to "<OwnerShort>-Fallback". Icon resolved here and passed
|
|
# over: the mirror has no Emby key
|
|
#
|
|
# Phase 3 (media seed) is NOT part of the above. Onboard ends at Phase 2 — connected, running,
|
|
# and saying so. Seeding is a separate operator-triggered phase; see --phase3-only. With
|
|
# MEDIA_SEED_ENABLED=false there is no Phase 3 and the model is two phases.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Credentials never in SSH command strings
|
|
# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
|
|
# locally, SCPed to the remote, and executed there. Command-line args are never used
|
|
# to pass credentials — they'd appear in `ps` output and shell history on both servers.
|
|
#
|
|
# XML templates are the single source of truth for deployed containers
|
|
# The owner's templates-user/ XMLs define every container deployed on the mirror.
|
|
# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
|
|
# Docker Manager can manage the containers after onboard without additional config.
|
|
#
|
|
# Dependency ordering in the auth stack is owner-enforced
|
|
# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
|
|
# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
|
|
# the script waits for the container to be healthy before continuing. This is a remote
|
|
# health check — the container must be running (or report healthy) before the next
|
|
# dependent is deployed.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check
|
|
# All operations run as root — SSH key management, docker operations, conf updates.
|
|
#
|
|
# SSH timeout on all remote calls
|
|
# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
|
|
# slow or unreachable mirror.
|
|
#
|
|
# --dry-run shows exact actions without executing
|
|
# Every step prints what it would do. SCP, deploy, plugin install, arr sync —
|
|
# all dry-run safe.
|
|
#
|
|
# Step skip flags for partial re-runs
|
|
# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
|
|
# resuming after a partial failure without re-running completed steps.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_PARTNERSHIP_AUTH_STACK
|
|
# XML filenames (from this server's templates-user/) to push and deploy on the
|
|
# mirror as its auth stack. Order matters: database deps before Authelia.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_REPLACE_CONTAINERS
|
|
# Containers to stop on the mirror before deploying the auth stack.
|
|
# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
|
|
# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
|
|
# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
|
|
# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
|
|
# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# HOST*_PARTNERSHIP_ARR_STACK
|
|
# XML filenames to push and deploy on the mirror as its arr stack.
|
|
# Leave empty to skip arr stack deploy.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
|
|
# Arr containers to stop on the mirror before deploying the arr stack.
|
|
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# HOST*_PARTNERSHIP_SERVICES_STACK
|
|
# XML filenames to push and deploy on the mirror as its shared services stack.
|
|
# Includes Emby, Jellyfin, Seerr, SeerrFin. Leave empty to skip services stack deploy.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_SERVICES_REPLACE_CONTAINERS
|
|
# Services containers to stop on the mirror before deploying the services stack.
|
|
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_SERVICES_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# Partnership/partnership_onboard.sh
|
|
# Full onboard — role detected automatically
|
|
#
|
|
# Partnership/partnership_onboard.sh --dry-run
|
|
# Preview all steps without making changes
|
|
#
|
|
# Partnership/partnership_onboard.sh --log
|
|
# Verbose per-step output
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-ssh
|
|
# Skip SSH key setup (key already in place)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-share-setup
|
|
# Skip share creation on mirror (shares already exist)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-auth-stack
|
|
# Skip auth stack stop + deploy (Steps 3-4)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-arr-stack
|
|
# Skip arr stack stop + deploy (Steps 4-5)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-services-stack
|
|
# Skip services stack stop + deploy (Steps 6-7)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-arr-sync
|
|
# Skip arr library bootstrap (Step 9)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-webhook-setup
|
|
# Skip webhook registration in arrs (Step 9b)
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase3-only
|
|
# OWNER: media seed only. Refuses unless Phase 2 is done and MEDIA_SEED_ENABLED is true.
|
|
# Sets the seeding gate posture (Tier 1 open, every Tier 2 gate closed) and dispatches
|
|
# Rsync/media_seed.sh detached. Returns in seconds; the seed runs for weeks.
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-appdata-provision
|
|
# Skip the pre-container auth appdata rsync (Step 1e)
|
|
# The mirror's auth containers then start against whatever is already there
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-webhook-listener
|
|
# Skip starting webhook listener on mirror (Step 9e)
|
|
# Listener will start automatically on next array restart
|
|
#
|
|
# Partnership/partnership_onboard.sh --no-arm
|
|
# Leave every sync gate as it is (Step 1d). Step 1e then has no Tier 1 and cannot provision.
|
|
# Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose
|
|
# shares are not populated yet.
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase1-only
|
|
# OWNER only: SSH key exchange, conf push, docker network, partner conf cache.
|
|
# Safe to run before HOST2 has Varaverk — all of it needs docker and SSH, not the plugin.
|
|
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase2-only
|
|
# OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
|
|
# by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
|
|
# Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
SSH_TIMEOUT=15
|
|
|
|
source "$SCRIPTS_ROOT/load_config.sh"
|
|
source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
|
|
|
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
|
SKIP_SSH=false
|
|
SKIP_SHARE_SETUP=false
|
|
SKIP_AUTH_STACK=false
|
|
SKIP_ARR_STACK=false
|
|
SKIP_SERVICES_STACK=false
|
|
SKIP_ARR_SYNC=false
|
|
SKIP_WEBHOOK_SETUP=false
|
|
SKIP_WEBHOOK_LISTENER=false
|
|
PHASE1_ONLY=false # OWNER: SSH + conf push only (mirror not yet installed)
|
|
# MIRROR: SSH key install only, no owner notification
|
|
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
|
PHASE3_ONLY=false # OWNER: media seed only — a separate decision, see the PHASE 3 block
|
|
SKIP_ARM=false # leave the sync gates as they are — see Step 1d
|
|
SKIP_APPDATA_PROVISION=false # skip the pre-container auth appdata sync — see Step 1e
|
|
FILTERED_ARGS=()
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--skip-ssh) SKIP_SSH=true ;;
|
|
--skip-share-setup) SKIP_SHARE_SETUP=true ;;
|
|
--skip-auth-stack) SKIP_AUTH_STACK=true ;;
|
|
--skip-arr-stack) SKIP_ARR_STACK=true ;;
|
|
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
|
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
|
--skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;;
|
|
--skip-appdata-provision) SKIP_APPDATA_PROVISION=true ;;
|
|
--skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;;
|
|
--phase1-only) PHASE1_ONLY=true ;;
|
|
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
|
--phase3-only) PHASE3_ONLY=true; SKIP_SSH=true ;;
|
|
--no-arm) SKIP_ARM=true ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
partnership_resolve_roles
|
|
|
|
EXTRA_FLAGS=()
|
|
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
|
[[ "$ENABLE_LOGGING" == true ]] && EXTRA_FLAGS+=("--log")
|
|
|
|
START=$(date +%s)
|
|
|
|
# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
|
|
write_onboard_phase() {
|
|
local target_id="$1" phase="$2"
|
|
local key="${target_id}_PHASE${phase}_DONE"
|
|
local state_file="$(platform_setup_db_path)"
|
|
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
|
set_state_var "$state_file" "$key" "true"
|
|
# The push now reports failure, so it is warned about rather than returned. This function's
|
|
# contract is "the phase is recorded here" — the flag is on local disk either way, and an
|
|
# unreachable partner must not make a completed phase look like it did not happen.
|
|
platform_push_setup_state \
|
|
|| warn "Phase flag ${key} written locally but not delivered to the partner"
|
|
return 0
|
|
}
|
|
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
|
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
|
echo ""
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
|
|
#
|
|
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
|
|
# use the same convention), and reads the named config array from the mirror's own conf.
|
|
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
|
|
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
|
|
#
|
|
# deploy_container_from_xml() already stops/removes containers with the same name as what's
|
|
# being deployed. This step handles containers with DIFFERENT names that conflict.
|
|
# ==============================================================================================
|
|
stop_mirror_stack() {
|
|
local config_var="$1" label="$2"
|
|
local -a to_stop=()
|
|
|
|
mapfile -t to_stop < <(read_remote_conf_array "$MIRROR_IP" "$config_var" | grep -v '^$')
|
|
|
|
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
|
log "No $label containers to stop on $MIRROR — skipping"
|
|
return 0
|
|
fi
|
|
|
|
log "Stopping $label on $MIRROR: ${to_stop[*]}"
|
|
for container in "${to_stop[@]}"; do
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would stop + rm $container on $MIRROR"
|
|
continue
|
|
fi
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"docker stop '$container' 2>/dev/null
|
|
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
|
grep -q removed && \
|
|
echo " $container removed ✅" || \
|
|
log " $container not found on $MIRROR — skipping"
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
if [[ "$AM_MIRROR" == true ]]; then
|
|
echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
|
|
echo ""
|
|
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
|
echo ""
|
|
|
|
# Tested before attempted, the same guard the owner's phase 1 has had all along.
|
|
#
|
|
# The key step is a TERMINAL step by design — ssh_setup.sh runs ssh-copy-id, which prompts for
|
|
# the owner's root password on a first install. Once the operator has done that in a terminal,
|
|
# pressing ▶ Onboard ran the whole thing again: another ssh-copy-id, this time from the WebGUI
|
|
# with no TTY to answer the prompt, which fails and aborts the run at Step 1 — so the button
|
|
# whose entire job is Step 2 could never reach it.
|
|
#
|
|
# Working SSH is the actual precondition, not "have we run the setup script". If it already
|
|
# works there is nothing to install, whichever route installed it.
|
|
OWNER_IP_PRE=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
|
if [[ "$SKIP_SSH" == true ]]; then
|
|
warn "Skipping SSH setup (--skip-ssh)"
|
|
elif [[ -n "$OWNER_IP_PRE" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$OWNER_IP_PRE" exit 0 2>/dev/null; then
|
|
echo "SSH to $OWNER already works ✅ — key already installed, skipping setup"
|
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "SSH key ready ✅"
|
|
else
|
|
error "SSH key setup failed"
|
|
error "Install the key from a terminal on this host — ssh-copy-id needs $OWNER's password,"
|
|
error "and a WebGUI button has no way to answer that prompt:"
|
|
error " bash $SCRIPTS_ROOT/Partnership/partnership_onboard.sh --phase1-only"
|
|
exit 1
|
|
fi
|
|
unset OWNER_IP_PRE
|
|
|
|
# Stop after the key when asked. ssh_setup.sh runs ssh-copy-id, which prompts for the
|
|
# owner's root password on a first install — answerable in a terminal, never from the
|
|
# WebGUI button, which is why the mirror's panel sends the operator to a terminal for
|
|
# exactly this step and nothing more. The flag was parsed but only ever honoured on the
|
|
# owner path, so a mirror asked for phase 1 silently ran the whole thing.
|
|
if [[ "$PHASE1_ONLY" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MIRROR PHASE 1 COMPLETE ━━━━━"
|
|
echo " SSH key: ready"
|
|
echo " Next: press ▶ Onboard on the Partnership tab to notify $OWNER"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
|
echo ""
|
|
|
|
OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
|
PHASE2_TRIGGERED=false
|
|
|
|
if [[ -n "$OWNER_IP" ]]; then
|
|
# Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
|
|
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2 via run_job.sh"
|
|
PHASE2_TRIGGERED=true
|
|
else
|
|
# Launched through run_job.sh, the same path cron and api/run.php use, so Phase 2
|
|
# gets a stat file, a job log and a Scheduler entry on the owner. It used to be a
|
|
# bare `nohup … > /tmp/vv_phase2_onboard.log`, which ran fine and left the owner's
|
|
# entire half of onboarding invisible to its own WebGUI — no job record, nothing
|
|
# under /var/log/varaverk, nothing for the operator to look at when asking why
|
|
# pressing Onboard here appeared to do nothing.
|
|
#
|
|
# setsid, not bare nohup: the job must lead its own process group so api/stop.php
|
|
# can signal the whole tree. api/run.php carries the same note for the same reason.
|
|
#
|
|
# Reported triggered only after the stat file proves run_job.sh actually started.
|
|
# The old `& echo triggered` printed unconditionally — it would have claimed success
|
|
# for a path that does not exist on the owner, which is exactly the failure mode a
|
|
# mirror in a different storage mode hits.
|
|
# Its own timeout, not SSH_TIMEOUT: the remote waits for the runner to prove itself,
|
|
# and 15s would cut that short and report a healthy launch as a failure.
|
|
_phase2_out=$(timeout 40 ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
|
"bash -s -- $(printf '%q' "$OWNER_SCRIPTS_DIR")" 2>/dev/null <<'PHASE2_TRIGGER'
|
|
sd="$1"
|
|
runner="$sd/Plugin/unraid/run_job.sh"
|
|
script="$sd/Partnership/partnership_onboard.sh"
|
|
stat_file="/var/log/varaverk/Partnership/partnership_onboard.json"
|
|
|
|
[ -f "$runner" ] || { echo "missing-runner:$runner"; exit 1; }
|
|
[ -f "$script" ] || { echo "missing-script:$script"; exit 1; }
|
|
|
|
# Absolute, not "newer than the file we saw a moment ago". A previous run's stat file rewritten
|
|
# inside the same second would compare equal and read as a failed launch.
|
|
t0=$(date +%s)
|
|
|
|
setsid nohup bash "$runner" "Partnership/partnership_onboard.sh" "$script" \
|
|
--manual --phase2-only >/dev/null 2>&1 </dev/null &
|
|
|
|
# run_job.sh writes the stat file before it execs the script, so a stat file dated at or after
|
|
# the moment we launched is proof the runner is alive — rather than proof the ssh call returned.
|
|
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
|
|
sleep 1
|
|
[ -f "$stat_file" ] || continue
|
|
now=$(date -r "$stat_file" +%s 2>/dev/null || echo 0)
|
|
[ "$now" -ge "$t0" ] && { echo triggered; exit 0; }
|
|
done
|
|
echo start-failed
|
|
exit 1
|
|
PHASE2_TRIGGER
|
|
)
|
|
# Failure patterns are matched first, and they echo a path back. An owner whose
|
|
# SCRIPTS_DIR happened to contain the word "triggered" would otherwise satisfy a
|
|
# leading *triggered* glob and report success for a launch that never happened.
|
|
case "$_phase2_out" in
|
|
missing-runner:*)
|
|
error "Phase 2 not started — no run_job.sh at ${_phase2_out#missing-runner:} on $OWNER"
|
|
;;
|
|
missing-script:*)
|
|
error "Phase 2 not started — no partnership_onboard.sh at ${_phase2_out#missing-script:} on $OWNER"
|
|
;;
|
|
start-failed)
|
|
error "Phase 2 launch on $OWNER did not produce a job record — check run_job.sh there"
|
|
;;
|
|
triggered)
|
|
echo "Phase 2 triggered on $OWNER ✅"
|
|
log "Watch on $OWNER: Scheduler tab, or tail -f /var/log/varaverk/Partnership/partnership_onboard.log"
|
|
PHASE2_TRIGGERED=true
|
|
;;
|
|
*)
|
|
warn "Could not auto-trigger Phase 2 on $OWNER"
|
|
;;
|
|
esac
|
|
fi
|
|
else
|
|
warn "Cannot resolve $OWNER Tailscale IP"
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
|
|
echo " SSH key: ready"
|
|
echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
|
|
if [[ "$PHASE2_TRIGGERED" == false ]]; then
|
|
echo ""
|
|
echo " Run manually on $OWNER:"
|
|
echo " bash Partnership/partnership_onboard.sh --phase2-only"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
|
log "Mirror: $MIRROR ($MIRROR_IP)"
|
|
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
|
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
|
[[ "$PHASE3_ONLY" == true ]] && log "Mode: Phase 3 only (media seed)"
|
|
echo ""
|
|
|
|
# ==============================================================================================
|
|
# ── PHASE 3: MEDIA SEED ───────────────────────────────────────────────────────────────────────
|
|
#
|
|
# The partnership is finished at the end of Phase 2 — connected, running, and saying so. Phase 3
|
|
# is not part of it. It is the separate decision to fill the partner's disks by rsync, and it is
|
|
# operator-triggered because a first seed of a full library is weeks of transfer and tens of
|
|
# terabytes of somebody else's free space.
|
|
#
|
|
# The seed used to be a step inside Phase 2. Inline it held the onboard open for the entire
|
|
# transfer, which meant the phase-2 flag, the conf push, discovery and container grouping all
|
|
# waited on it, and both hosts described an unfinished onboard over a partnership that was
|
|
# already live. Detaching it fixed the blocking but left the seed starting on its own, which is
|
|
# still the wrong default: the machine decided to move 28 TB because an onboard succeeded.
|
|
#
|
|
# With MEDIA_SEED_ENABLED=false there is no Phase 3 at all and the model is two phases. That is
|
|
# the point of the toggle — not a seed that runs and does nothing, an absent phase.
|
|
# ==============================================================================================
|
|
if [[ "$PHASE3_ONLY" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SYNC PHASE 3 — MEDIA SEED — $MY_ID → $MIRROR ━━━━━"
|
|
echo ""
|
|
|
|
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
|
_seed_gate=$(grep -m1 -E '^[[:space:]]*MEDIA_SEED_ENABLED=' "$_master_conf" 2>/dev/null \
|
|
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
|
|
|
if [[ -n "$_seed_gate" && "$_seed_gate" != "true" ]]; then
|
|
error "MEDIA_SEED_ENABLED is '$_seed_gate' — there is no Phase 3 on this host"
|
|
error " Onboarding is two phases while seeding is off. Arm the toggle in master.conf"
|
|
error " if you want the partner's library filled by rsync."
|
|
exit 1
|
|
fi
|
|
|
|
_setup_db="$(platform_setup_db_path)"
|
|
if ! grep -q "^${MIRROR_ID}_PHASE2_DONE=true" "$_setup_db" 2>/dev/null; then
|
|
error "$MIRROR has not completed Phase 2 — seeding a partner that is not onboarded"
|
|
error " would push media at a host with no containers to serve it. Finish Phase 2 first."
|
|
exit 1
|
|
fi
|
|
unset _setup_db
|
|
|
|
# Seeding posture, set before the transfer rather than after it. Tier 1 has to be open or
|
|
# rsync.sh refuses every share; every Tier 2 gate is closed so the scheduled orchestrators
|
|
# are not competing for the same link and the same disks for the next several weeks. This is
|
|
# also the posture the seed leaves behind — global on, per-orchestrator off — so finishing
|
|
# Phase 3 does not quietly hand the schedule back.
|
|
echo "━━━ Gate posture ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would set RSYNC_ENABLED true and every Tier 2 rsync gate false"
|
|
else
|
|
cp -a "$_master_conf" "${_master_conf}.bak-phase3-$(date +%Y%m%d-%H%M%S)"
|
|
set_conf_bool RSYNC_ENABLED "true" "$_master_conf" \
|
|
|| { error "Could not open Tier 1 — the seed would move nothing"; exit 1; }
|
|
for _gate in CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED \
|
|
WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED; do
|
|
set_conf_bool "$_gate" "false" "$_master_conf" \
|
|
|| warn "Could not close $_gate — a scheduled sync may run beside the seed"
|
|
done
|
|
unset _gate
|
|
echo " Tier 1 open, every Tier 2 gate closed ✅"
|
|
fi
|
|
unset _master_conf _seed_gate
|
|
|
|
echo ""
|
|
echo "━━━ Dispatch ━━━"
|
|
_seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh"
|
|
_runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would dispatch Rsync/media_seed.sh"
|
|
exit 0
|
|
fi
|
|
if [[ ! -f "$_seed_script" || ! -f "$_runner" ]]; then
|
|
error "Seed runner not found at $_seed_script"
|
|
exit 1
|
|
fi
|
|
|
|
# setsid, not nohup + &: the seed must outlive this script and must not share its process
|
|
# group, or stopping Phase 3 takes the transfer down with it.
|
|
_seed_flags=()
|
|
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
|
setsid bash "$_runner" "Rsync/media_seed.sh" "$_seed_script" "${_seed_flags[@]}" \
|
|
>/dev/null 2>&1 < /dev/null &
|
|
disown 2>/dev/null || true
|
|
|
|
# Report the record, not the launch. run_job.sh writes its stat file before running the
|
|
# script, so a fresh "running" record is the difference between dispatched and attempted.
|
|
_seed_stat="/var/log/varaverk/Rsync/media_seed.json"
|
|
_dispatched=false
|
|
for _i in $(seq 1 10); do
|
|
if [[ -f "$_seed_stat" ]] \
|
|
&& grep -q '"status":"running"' "$_seed_stat" 2>/dev/null \
|
|
&& (( $(date +%s) - $(stat -c %Y "$_seed_stat") < 60 )); then
|
|
_dispatched=true
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PHASE 3 SUMMARY ━━━━━"
|
|
if [[ "$_dispatched" == true ]]; then
|
|
echo " Seed: running as job Rsync/media_seed.sh ✅"
|
|
echo " Shares: ${#DAILY_SYNC_SHARES[@]}"
|
|
echo " Gates: Tier 1 open · every Tier 2 closed"
|
|
echo ""
|
|
echo " A first full seed runs for days. Watch it on the Partnership tab; the"
|
|
echo " partnership itself has been live since Phase 2 and does not depend on this."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
error " Seed did not start — check /var/log/varaverk/Rsync/media_seed.log"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 1
|
|
fi
|
|
|
|
STEP_SSH_OK=false
|
|
STEP_NETWORK_OK=false
|
|
LOCAL_SETUP_OK=true # partnership_manager --local-only; the summary claimed done ✅ regardless
|
|
PHASE1_NET_OK=false # Phase 1 only — network created on the mirror before any deploy
|
|
PHASE1_CACHE_OK=false # Phase 1 only — our conf pushed into the mirror's RAM cache
|
|
STEP_STOP_AUTH_OK=true
|
|
STEP_AUTH_OK=true
|
|
AUTH_DEPLOYED=0
|
|
AUTH_FAILED=0
|
|
STEP_STOP_ARR_OK=true
|
|
STEP_ARR_OK=true
|
|
ARR_DEPLOYED=0
|
|
ARR_FAILED=0
|
|
STEP_STOP_SERVICES_OK=true
|
|
STEP_SERVICES_OK=true
|
|
SERVICES_DEPLOYED=0
|
|
SERVICES_FAILED=0
|
|
ONBOARD_OK=false
|
|
ARR_SYNC_OK=false
|
|
WEBHOOK_SETUP_OK=false
|
|
WEBHOOK_LISTENER_OK=false
|
|
MASTER_PUSH_OK=false
|
|
|
|
# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
|
|
# Skipped when --phase2-only (SSH was already done in Phase 1).
|
|
echo "━━━ Step 1 — SSH Key Setup ━━━"
|
|
|
|
if [[ "$SKIP_SSH" == true ]]; then
|
|
warn "Skipping (--skip-ssh)"
|
|
STEP_SSH_OK=true
|
|
elif [[ "$PHASE1_ONLY" == true ]]; then
|
|
# Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
|
|
# hanging for a password prompt with no TTY.
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
|
|
echo "SSH to $MIRROR already works ✅ — skipping key install"
|
|
STEP_SSH_OK=true
|
|
else
|
|
# Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
|
|
if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "SSH keys ready ✅"
|
|
STEP_SSH_OK=true
|
|
else
|
|
# Soft-fail: generate key locally if not present, then tell user to install manually
|
|
warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
|
|
if [[ -f "$SSH_KEY" ]]; then
|
|
log "Local key exists at: $SSH_KEY"
|
|
else
|
|
bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
|
|
fi
|
|
if [[ -f "${SSH_KEY}.pub" ]]; then
|
|
echo ""
|
|
echo " Install this key on $MIRROR to complete SSH setup:"
|
|
echo " ┌─────────────────────────────────────────────────────"
|
|
cat "${SSH_KEY}.pub" | sed 's/^/ │ /'
|
|
echo " └─────────────────────────────────────────────────────"
|
|
echo " Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
|
echo " Then click 'Push Conf' in the Partnership tab."
|
|
# Write key-ready flag so UI can show the manual-install state
|
|
[[ "$DRY_RUN" == false ]] && {
|
|
kflag="${MIRROR_ID}_KEY_READY"
|
|
_setup_f="$(platform_setup_db_path)"
|
|
set_state_var "$_setup_f" "$kflag" "true"
|
|
}
|
|
fi
|
|
STEP_SSH_OK=false
|
|
fi
|
|
fi
|
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "SSH keys ready ✅"
|
|
STEP_SSH_OK=true
|
|
else
|
|
error "SSH key setup failed — aborting"
|
|
error "Re-run or use --skip-ssh if key is already set up"
|
|
exit 1
|
|
fi
|
|
|
|
# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
|
|
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
|
|
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
|
|
if [[ "$PHASE1_ONLY" == true ]]; then
|
|
if [[ "$STEP_SSH_OK" == false ]]; then
|
|
# SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
|
|
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
|
echo ""
|
|
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
|
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
|
LOCAL_SETUP_OK=false
|
|
warn "Local setup had issues — check partnership_manager.sh output above"
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
|
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
|
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
|
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " ACTION NEEDED: install the key on $MIRROR:"
|
|
echo " ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
|
|
echo " Then click 'Push Conf' in Partnership tab, or run:"
|
|
echo " bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ Phase 1 — Conf Push ━━━"
|
|
|
|
CONF_PUSH_OK=false
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would push master.conf + state file to $MIRROR"
|
|
CONF_PUSH_OK=true
|
|
else
|
|
push_output=$(platform_push_conf)
|
|
push_rc=$?
|
|
[[ -n "$push_output" ]] && echo "$push_output"
|
|
platform_push_setup_state
|
|
if [[ $push_rc -eq 0 ]]; then
|
|
echo "Conf push complete ✅"
|
|
CONF_PUSH_OK=true
|
|
else
|
|
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
|
fi
|
|
fi
|
|
|
|
# ── Phase 1 — Docker network on the mirror ────────────────────────────────────────────────
|
|
# Here, not only in Step 1b, because a --phase1-only run exits above and never reaches it.
|
|
# The mirror needs docker, not Varaverk, so this works before the plugin is installed — and
|
|
# creating the network now means it is in place long before the first container is deployed
|
|
# against it. Deploying against a missing network is what left twelve containers stuck in
|
|
# `Created`, so the earliest safe moment is the right one.
|
|
echo ""
|
|
echo "━━━ Phase 1 — Docker Network ($MIRROR) ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would create any networks the stack templates reference on $MIRROR"
|
|
PHASE1_NET_OK=true
|
|
elif ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
PHASE1_NET_OK=true
|
|
else
|
|
warn "Network prep incomplete on $MIRROR — Step 1b retries this during Phase 2"
|
|
fi
|
|
|
|
# ── Phase 1 — Partner conf cache ──────────────────────────────────────────────────────────
|
|
# CONF_SYNC_ENABLED is armed here rather than waiting for Step 1d. It is the safe one of the
|
|
# three gates — it moves no data, it copies each side's host*.conf into the other's tmpfs so
|
|
# partner vars resolve — and the moment SSH works is the moment that should start. Leaving it
|
|
# until Phase 2 meant conf_sync.sh, which sources the conf fresh in its own process, exited
|
|
# on a closed gate every time it was called before then.
|
|
#
|
|
# Push always; pull only if the mirror actually has a conf yet. Before HOST2 installs Varaverk
|
|
# there is nothing to pull, and an unconditional pull would count a failure and notify about
|
|
# a condition that is simply "the mirror is not installed". The pull lands on the re-run after the
|
|
# install — the same --phase1-only --skip-ssh the operator uses to push conf again.
|
|
echo ""
|
|
echo "━━━ Phase 1 — Partner Conf Cache ━━━"
|
|
_conf_sync_script="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would arm CONF_SYNC_ENABLED and cache confs with $MIRROR"
|
|
PHASE1_CACHE_OK=true
|
|
elif [[ ! -f "$_conf_sync_script" ]]; then
|
|
warn "conf_sync.sh not found — skipping partner conf cache"
|
|
else
|
|
set_conf_bool CONF_SYNC_ENABLED "true" "$SCRIPTS_ROOT/Configurations/master.conf" \
|
|
|| warn "Could not arm CONF_SYNC_ENABLED — cache step may no-op"
|
|
|
|
if bash "$_conf_sync_script" --push-only; then
|
|
PHASE1_CACHE_OK=true
|
|
else
|
|
warn "Could not push our conf to $MIRROR"
|
|
fi
|
|
|
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '${_mirror_sd}/Configurations/${MIRROR_ID,,}.conf' ]" 2>/dev/null; then
|
|
bash "$_conf_sync_script" --pull-only || warn "Could not pull ${MIRROR_ID,,}.conf from $MIRROR"
|
|
else
|
|
echo " $MIRROR has no ${MIRROR_ID,,}.conf yet — it will cache once Varaverk is installed there"
|
|
fi
|
|
unset _mirror_sd
|
|
fi
|
|
unset _conf_sync_script
|
|
|
|
# HOST1 local setup — runs immediately without needing HOST2
|
|
echo ""
|
|
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
|
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
|
LOCAL_SETUP_OK=false
|
|
warn "Local setup had issues — check partnership_manager.sh output above"
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
|
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
|
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
|
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
|
echo " Network: $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )"
|
|
echo " Conf cache: $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )"
|
|
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
|
# The push no longer waits for a Varaverk install on the far side. It resolves the partner's
|
|
# conf directory across both layouts and creates the internal one if neither exists, so the
|
|
# conf arrives BEFORE the plugin — which is the order that makes it useful. The .plg only
|
|
# seeds master.conf from the template when none is present, so what lands here survives the
|
|
# install and the wizard reads its identity straight out of it.
|
|
if [[ "$CONF_PUSH_OK" == true ]]; then
|
|
echo " master.conf is on $MIRROR — the wizard will find it and take the partner path,"
|
|
echo " already knowing $MY_ID and $MIRROR_ID. If the operator picks flash storage there,"
|
|
echo " storage_migrate.sh moves the conf to appdata with the rest of the install."
|
|
else
|
|
echo " master.conf was NOT delivered. Phase 1 seeds it into a bare host, so this is a"
|
|
echo " real failure, not the pre-install state — check SSH and that /boot is writable"
|
|
echo " on $MIRROR. Retry with:"
|
|
echo " • bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
|
echo " • or push from Scheduler → master.conf → Save Conf"
|
|
echo " • or, once the plugin is installed, 'Pull from HOST1' on HOST2's Setup tab"
|
|
fi
|
|
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Step 1b: Ensure custom Docker network exists on mirror ────────────────────────────────────
|
|
# Must run before any container deploy — docker create fails if the network is missing.
|
|
echo ""
|
|
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
|
|
|
|
# Two halves, and the first is the one that matters for a fresh mirror.
|
|
#
|
|
# ensure_stack_networks_on_remote reads the networks out of the XMLs this onboard is about to
|
|
# push and creates any that are missing on the mirror. It does not consult the mirror's conf,
|
|
# because on a fresh node that array is the template default — a single commented-out entry —
|
|
# and an empty list is indistinguishable from "no networks needed". The result was every
|
|
# container in both stacks created against a network that did not exist.
|
|
#
|
|
# docker_network_connect.sh still runs afterwards: it is what *connects* the mirror's own
|
|
# listed containers to its own listed networks, which is a different job and remains the
|
|
# mirror's to declare.
|
|
if ! ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
warn "One or more stack networks could not be prepared on $MIRROR — deploys below may fail"
|
|
fi
|
|
|
|
# Resolve the path on the mirror, not here. This used SCRIPTS_ROOT — the OWNER's install path —
|
|
# to name a script it then runs over SSH on the MIRROR. That only holds while both hosts install
|
|
# to the same place, and an appdata-mode mirror is at /mnt/user/appdata/Varaverk, so bash was
|
|
# handed a path that does not exist there and Step 1b failed on every appdata install. The
|
|
# warning even printed the owner's path and told the operator to go check it on the mirror.
|
|
#
|
|
# Same fix as Steps 11 and 12, which already resolve the remote layout this way.
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
|
|
STEP_NETWORK_OK=true
|
|
else
|
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
|
_net_script="${_mirror_sd}/Docker_Essentials/docker_network_connect.sh"
|
|
if timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '$_net_script' ] || { echo missing; exit 127; }; bash '$_net_script'" 2>/dev/null; then
|
|
echo "Docker network ready on $MIRROR ✅"
|
|
STEP_NETWORK_OK=true
|
|
else
|
|
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
|
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
|
fi
|
|
unset _mirror_sd
|
|
fi
|
|
|
|
# ── Step 1c: Share setup ─────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 1c — Share Setup (Mirror) ━━━"
|
|
|
|
if [[ "$SKIP_SHARE_SETUP" == true ]]; then
|
|
warn "Skipping (--skip-share-setup)"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
bash "$SCRIPT_DIR/share_setup.sh" --dry-run
|
|
else
|
|
bash "$SCRIPT_DIR/share_setup.sh"
|
|
fi
|
|
|
|
# ── Step 1c2: Permit the mirror ───────────────────────────────────────────────────────────────
|
|
# A previous offboard blocklists the mirror, and rsync.sh refuses any host on that list outright
|
|
# — deliberately, so a stale cron cannot keep syncing to a partner that has been removed.
|
|
#
|
|
# The blocklist was cleared inside Step 8, which was fine while every rsync in the onboard came
|
|
# after it. Step 1e does not: it runs before the containers are deployed, which is the whole
|
|
# point of it. On a re-onboard that ordering means the provisioning sync is refused, the auth
|
|
# stack comes up against empty directories, and the step built to prevent exactly that reports
|
|
# a failure nobody can explain from its own output.
|
|
#
|
|
# Un-blocking here is not a loosening. The offboard's own summary says "re-onboard to permit
|
|
# access again" — an onboard is the event that permits it, and this is where the onboard starts
|
|
# needing it. Step 8 still runs its own clear; removing an absent entry is a no-op.
|
|
echo ""
|
|
echo "━━━ Step 1c2 — Permit $MIRROR ━━━"
|
|
|
|
UNBLOCK_OK=true
|
|
_blocklist="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}"
|
|
if [[ ! -f "$_blocklist" ]] || ! grep -q "^${MIRROR}|" "$_blocklist" 2>/dev/null; then
|
|
echo " $MIRROR is not blocklisted ✅"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would remove $MIRROR from the partnership blocklist"
|
|
else
|
|
sed -i "/^${MIRROR}|/d" "$_blocklist" 2>/dev/null
|
|
if grep -q "^${MIRROR}|" "$_blocklist" 2>/dev/null; then
|
|
error "Could not remove $MIRROR from $_blocklist — rsync.sh will refuse it"
|
|
UNBLOCK_OK=false
|
|
else
|
|
echo " Removed $MIRROR from the blocklist ✅"
|
|
fi
|
|
fi
|
|
unset _blocklist
|
|
|
|
# ── Step 1d: Sync gates ───────────────────────────────────────────────────────────────────────
|
|
# This was Step 9c, after the containers were already deployed. It has to run here instead,
|
|
# because Step 1e below is itself an rsync and Tier 1 stops every rsync — arming afterwards
|
|
# meant the provisioning sync exited cleanly having moved nothing.
|
|
#
|
|
# master.conf.template ships a fresh node inert. A successful Phase 1 is what makes Tier 1,
|
|
# CONF_SYNC and ARR_SYNC true; without this the defaults were a one-way door and somebody had
|
|
# to remember to hand-edit three toggles on the machine where forgetting looks exactly like
|
|
# everything working.
|
|
#
|
|
# Tier 2 is now set false rather than "left as configured". Tier 1 opening is what makes the
|
|
# scheduled jobs live, and a node that was onboarded ten seconds ago is not a node anyone has
|
|
# checked yet — free space, share layout, what the partner actually holds. Arriving connected
|
|
# and idle is the useful state; the operator arms each orchestrator when they mean to.
|
|
# MEDIA_SEED_ENABLED is not touched here at all — see Step 13.
|
|
#
|
|
# Owner only — the mirror receives these values in the Step 10 push rather than deciding.
|
|
ARM_OK=true
|
|
_VV_ARM_ON=(RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED)
|
|
_VV_ARM_OFF=(CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED
|
|
WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED)
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 1d — Sync Gates ━━━"
|
|
|
|
if [[ "$SKIP_ARM" == true ]]; then
|
|
echo " --no-arm — leaving the sync gates as they are"
|
|
warn " Step 1e needs Tier 1 open; with it closed the auth appdata will not be provisioned"
|
|
elif [[ "$AM_OWNER" != true ]]; then
|
|
echo " mirror — the owner's push decides these"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would set ${_VV_ARM_ON[*]} true and ${_VV_ARM_OFF[*]} false"
|
|
else
|
|
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
|
if [[ ! -f "$_master_conf" ]]; then
|
|
warn "master.conf not found at $_master_conf — gates left closed"
|
|
ARM_OK=false
|
|
else
|
|
cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)"
|
|
for _gate in "${_VV_ARM_ON[@]}"; do set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false; done
|
|
for _gate in "${_VV_ARM_OFF[@]}"; do set_conf_bool "$_gate" "false" "$_master_conf" || ARM_OK=false; done
|
|
unset _gate
|
|
echo " Tier 1 open: ${_VV_ARM_ON[*]}"
|
|
echo " Tier 2 off: every scheduled rsync — arm them yourself when you are ready"
|
|
fi
|
|
unset _master_conf
|
|
fi
|
|
|
|
# ── Step 1e: Provision the auth stack's appdata ───────────────────────────────────────────────
|
|
# The one rsync an onboard actually needs, and it has to land before Step 3 creates the
|
|
# containers that read it. Deploying first meant Authelia, Lldap, NPM and both databases came up
|
|
# on the mirror against empty directories and initialised themselves from nothing — so the
|
|
# mirror's auth stack was a fresh install wearing the owner's container names.
|
|
#
|
|
# The critical-data profile, not critical-fallback. critical-data stops the auth containers on
|
|
# both sides for the duration, which costs the owner its auth for about a minute at 588 MB, and
|
|
# buys a consistent copy. critical-fallback is the dirty variant that keeps auth running, and a
|
|
# dirty copy is exactly what broke Redis and MariaDB on the mirror before: the manifest and the
|
|
# binlog index were copied without the files they name, and MariaDB hides that — the container
|
|
# reads Up while mysqld_safe restarts the dead database every few seconds.
|
|
#
|
|
# Media is not seeded here or anywhere in Phase 2. This share is 588 MB and the containers do
|
|
# not start correctly without it; a media library is terabytes and nothing waits on it.
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Step 1e — Auth Appdata Provision ━━━"
|
|
|
|
APPDATA_PROVISION_OK=false
|
|
if [[ "$SKIP_APPDATA_PROVISION" == true ]]; then
|
|
warn "Skipping (--skip-appdata-provision) — the mirror's auth stack will start empty"
|
|
elif [[ "${#PARTNERSHIP_PROVISION_SHARES[@]}" -eq 0 ]]; then
|
|
warn "PARTNERSHIP_PROVISION_SHARES is empty — nothing to provision"
|
|
warn " Set it in host${MY_ID: -1}.conf, or the mirror's auth stack starts from nothing"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would sync ${PARTNERSHIP_PROVISION_SHARES[*]} to $MIRROR"
|
|
APPDATA_PROVISION_OK=true
|
|
else
|
|
_prov_rc=0
|
|
for _share in "${PARTNERSHIP_PROVISION_SHARES[@]}"; do
|
|
echo " Provisioning: $_share"
|
|
_prov_flags=()
|
|
[[ "$ENABLE_LOGGING" == true ]] && _prov_flags+=(--log)
|
|
if ! bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_prov_flags[@]}"; then
|
|
warn " Failed: $_share"
|
|
_prov_rc=1
|
|
fi
|
|
done
|
|
unset _share _prov_flags
|
|
if [[ "$_prov_rc" -eq 0 ]]; then
|
|
echo "Auth appdata provisioned to $MIRROR ✅"
|
|
APPDATA_PROVISION_OK=true
|
|
else
|
|
warn "Auth appdata NOT fully provisioned — the containers below will start against"
|
|
warn " whatever is already on $MIRROR, which on a fresh node is nothing"
|
|
fi
|
|
unset _prov_rc
|
|
fi
|
|
|
|
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
|
|
|
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
|
warn "Skipping (--skip-auth-stack)"
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
|
|
fi
|
|
|
|
# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 3 — Deploy Auth Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
|
warn "Skipping (--skip-auth-stack)"
|
|
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
|
|
warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
|
|
warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
|
|
STEP_AUTH_OK=false
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_AUTH_STACK
|
|
AUTH_DEPLOYED=$_STACK_DEPLOYED
|
|
AUTH_FAILED=$_STACK_FAILED
|
|
echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
|
|
[[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
|
|
fi
|
|
|
|
# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 4 — Stop Mirror Arr Stack ━━━"
|
|
|
|
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
|
warn "Skipping (--skip-arr-stack)"
|
|
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
|
|
log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
|
|
SKIP_ARR_STACK=true
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
|
|
fi
|
|
|
|
# ── Step 5: Deploy arr stack on mirror ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 5 — Deploy Arr Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
|
warn "Skipping (--skip-arr-stack)"
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_ARR_STACK
|
|
ARR_DEPLOYED=$_STACK_DEPLOYED
|
|
ARR_FAILED=$_STACK_FAILED
|
|
echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
|
|
[[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
|
|
fi
|
|
|
|
# ── Step 6: Stop mirror's existing services stack ─────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 6 — Stop Mirror Services Stack ━━━"
|
|
|
|
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
|
warn "Skipping (--skip-services-stack)"
|
|
elif [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -eq 0 ]]; then
|
|
log "PARTNERSHIP_SERVICES_STACK not configured — skipping services stack deploy"
|
|
SKIP_SERVICES_STACK=true
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" "services stack"
|
|
fi
|
|
|
|
# ── Step 7: Deploy services stack on mirror ───────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 7 — Deploy Services Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_SERVICES_STACK" == true ]]; then
|
|
warn "Skipping (--skip-services-stack)"
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_SERVICES_STACK
|
|
SERVICES_DEPLOYED=$_STACK_DEPLOYED
|
|
SERVICES_FAILED=$_STACK_FAILED
|
|
echo "Services stack: $SERVICES_DEPLOYED deployed, $SERVICES_FAILED failed"
|
|
[[ "$SERVICES_FAILED" -gt 0 ]] && STEP_SERVICES_OK=false
|
|
fi
|
|
|
|
# ── Step 8: Partnership onboard ───────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 8 — Partnership Onboard ━━━"
|
|
|
|
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
|
|
echo "Partnership onboard complete ✅"
|
|
ONBOARD_OK=true
|
|
else
|
|
error "Partnership onboard failed"
|
|
ONBOARD_OK=false
|
|
fi
|
|
|
|
# ── Step 9: Arr library bootstrap ─────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 9 — Arr Library Bootstrap ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete"
|
|
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
|
|
warn "Skipping (--skip-arr-sync)"
|
|
elif [[ ! -f "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ]]; then
|
|
warn "arr_sync.sh not found — run Arrs_Stack/arr_sync.sh manually once arrs are live"
|
|
elif bash "$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "Arr bootstrap complete ✅"
|
|
ARR_SYNC_OK=true
|
|
else
|
|
warn "Arr sync had errors — partnership still valid"
|
|
warn "Re-run Arrs_Stack/arr_sync.sh once all arr containers are live"
|
|
fi
|
|
|
|
# ── Step 9b: Webhook setup ────────────────────────────────────────────────────────────────────
|
|
# Register the download webhook in each arr on both servers. Arrs must be running.
|
|
# webhook_setup.sh handles local + SSH to remote in one call.
|
|
echo ""
|
|
echo "━━━ Step 9b — Webhook Setup ━━━"
|
|
|
|
_webhook_script="$SCRIPTS_ROOT/Tools/webhook_setup.sh"
|
|
if [[ "$SKIP_WEBHOOK_SETUP" == true ]]; then
|
|
warn "Skipping (--skip-webhook-setup)"
|
|
elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then
|
|
warn "WEBHOOK_PORT=0 — webhook disabled, skipping"
|
|
elif [[ ! -f "$_webhook_script" ]]; then
|
|
warn "Tools/webhook_setup.sh not found — run manually after onboard"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run webhook_setup.sh (local + remote)"
|
|
WEBHOOK_SETUP_OK=true
|
|
elif bash "$_webhook_script" "${EXTRA_FLAGS[@]}"; then
|
|
echo "Webhook setup complete ✅"
|
|
WEBHOOK_SETUP_OK=true
|
|
else
|
|
warn "Webhook setup had errors — run Tools/webhook_setup.sh manually once arrs are settled"
|
|
fi
|
|
unset _webhook_script
|
|
|
|
# ── Step 9e: Start webhook listener on mirror ─────────────────────────────────────────────────
|
|
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
|
|
# already running — kick it now so events are captured immediately after onboard.
|
|
echo ""
|
|
echo "━━━ Step 9e — Webhook Listener (Mirror) ━━━"
|
|
|
|
_listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh"
|
|
if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then
|
|
warn "Skipping (--skip-webhook-listener)"
|
|
elif [[ "${WEBHOOK_PORT:-0}" -eq 0 ]]; then
|
|
warn "WEBHOOK_PORT=0 — webhook disabled, skipping"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would start webhook listener on $MIRROR"
|
|
WEBHOOK_LISTENER_OK=true
|
|
elif timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"nohup bash '$_listener_script' > /var/log/varaverk/upgrade_webhook.log 2>&1 & echo started" \
|
|
2>/dev/null | grep -q started; then
|
|
echo "Webhook listener started on $MIRROR ✅"
|
|
WEBHOOK_LISTENER_OK=true
|
|
else
|
|
warn "Could not start listener on $MIRROR — it will start automatically on next array restart"
|
|
fi
|
|
unset _listener_script
|
|
|
|
# ── Step 10: Push master.conf to all listed hosts ─────────────────────────────────────────────
|
|
# SSH is now established and all partners have the plugin installed.
|
|
# Push the authoritative master.conf so every listed host is in sync immediately.
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 10 — master.conf Push ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would push master.conf to all listed hosts"
|
|
MASTER_PUSH_OK=true
|
|
else
|
|
push_output=$(platform_push_conf)
|
|
push_rc=$?
|
|
[[ -n "$push_output" ]] && echo "$push_output"
|
|
platform_push_setup_state
|
|
if [[ $push_rc -eq 0 ]]; then
|
|
echo "master.conf sync complete ✅"
|
|
MASTER_PUSH_OK=true
|
|
else
|
|
warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
|
fi
|
|
fi
|
|
|
|
# ── Step 11: Service discovery on the mirror ──────────────────────────────────────────────────
|
|
# Deliberately last. conf_populate.sh fills host*.conf from what it can actually find running —
|
|
# arr API keys, container names, URLs — and until Step 3 and Step 5 deployed the auth and arr
|
|
# stacks there was nothing on the mirror to find. The wizard runs it during first-run setup,
|
|
# which on a fresh mirror is precisely the moment the machine is still empty, so everything it
|
|
# could have discovered was discovered as absent.
|
|
#
|
|
# No --overwrite: it only fills blanks, so anything the operator set by hand survives. --no-push
|
|
# because Step 10 above has just pushed conf; letting discovery push again would race it.
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 11 — Service Discovery ($MIRROR) ━━━"
|
|
|
|
POPULATE_OK=false
|
|
# MIRROR_IP, not MIRROR_REACHABLE — the latter is partnership_offboard.sh's variable and does not
|
|
# exist in this script, so the test was always true against an empty string and Step 11 reported
|
|
# "skipped (unreachable)" on a mirror it had just deployed twelve containers to.
|
|
if [[ -z "${MIRROR_IP:-}" ]]; then
|
|
warn "$MIRROR has no resolved IP — skipping discovery, run Deployment/conf_populate.sh there later"
|
|
POPULATE_OK=skipped
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run conf_populate.sh --no-push on $MIRROR"
|
|
POPULATE_OK=true
|
|
else
|
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
|
_pop_script="${_mirror_sd}/Deployment/conf_populate.sh"
|
|
if timeout 180 ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '$_pop_script' ] || { echo missing; exit 127; }; bash '$_pop_script' --no-push" 2>/dev/null; then
|
|
echo "Discovery complete on $MIRROR ✅"
|
|
POPULATE_OK=true
|
|
else
|
|
warn "Discovery failed on $MIRROR — run $_pop_script there by hand"
|
|
fi
|
|
unset _mirror_sd _pop_script
|
|
fi
|
|
|
|
# ── Step 12: Group our containers on the mirror ───────────────────────────────────────────────
|
|
# The mirror now runs a dozen containers that are ours, scattered among its own. This files them
|
|
# under one folder named after us — "<OwnerShort>-Fallback" — matching the convention the owner
|
|
# already keeps for the mirror's containers.
|
|
#
|
|
# The icon is resolved HERE and passed over, not looked up there. It comes from the closest Emby
|
|
# user to our own name, and the mirror has neither our Emby key nor necessarily an Emby at all —
|
|
# so a lookup on that side would find nothing and the folder would come up blank.
|
|
#
|
|
# Not fatal in any direction: folder.view3 absent on the mirror is a clean skip, and a folder
|
|
# without a picture is still a folder.
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 12 — Container Grouping ($MIRROR) ━━━"
|
|
|
|
FOLDER_OK=false
|
|
if [[ -z "${MIRROR_IP:-}" ]]; then
|
|
warn "$MIRROR has no resolved IP — skipping container grouping"
|
|
FOLDER_OK=skipped
|
|
else
|
|
mapfile -t _deployed < <(deployed_stack_container_names)
|
|
_deployed_csv=$(IFS=,; echo "${_deployed[*]}")
|
|
if [[ -z "$_deployed_csv" ]]; then
|
|
log "No stack templates resolved to container names — nothing to group"
|
|
FOLDER_OK=skipped
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would create ${MY_ID}-named fallback folder on $MIRROR with: $_deployed_csv"
|
|
FOLDER_OK=true
|
|
else
|
|
_ff_local="$SCRIPTS_ROOT/Plugin/$PLATFORM/Tools/fallback_folder.php"
|
|
_mf_local="$SCRIPTS_ROOT/Plugin/$PLATFORM/Tools/mirror_folders.php"
|
|
_icon=$(php "$_ff_local" --host="$MY_ID" --icon-only 2>/dev/null || true)
|
|
[[ -z "$_icon" ]] && log "No icon resolved for $MY_ID — folder will be created without one"
|
|
|
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
|
_ff_remote="${_mirror_sd}/Plugin/${PLATFORM}/Tools/fallback_folder.php"
|
|
_mf_remote="${_mirror_sd}/Plugin/${PLATFORM}/Tools/mirror_folders.php"
|
|
|
|
# ── 12a: reproduce our own folder layout on the mirror ────────────────────────────────
|
|
# The deployed stacks run on the mirror continuously — they are not failover coverage —
|
|
# so they belong on the same shelves they occupy here: Sonarr in "Arrs Stack", NPM and
|
|
# Lldap in "Networking", the databases in "Databases". Filing all of them under
|
|
# "<Owner>-Fallback", which is what this step used to do, records whose they are and
|
|
# nothing about what they do.
|
|
#
|
|
# The plan is computed here, where the owner's folder.view3 layout lives, and applied
|
|
# there. Whatever the layout does not account for comes back as "unfiled" and is what
|
|
# the fallback folder is actually for.
|
|
_unfiled="$_deployed_csv"
|
|
if [[ -f "$_mf_local" ]]; then
|
|
# Comma-joined: the flag takes a CSV, and "${arr[*]}" would join on spaces — which
|
|
# also happen to appear inside folder names, so the plan must never be reflowed.
|
|
_fbonly=$(IFS=,; printf '%s' "${PARTNERSHIP_FALLBACK_ONLY[*]:-}")
|
|
_plan=$(php "$_mf_local" --export --containers="$_deployed_csv" \
|
|
--fallback-only="$_fbonly" 2>/dev/null)
|
|
if [[ -n "$_plan" ]]; then
|
|
if printf '%s' "$_plan" | timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '$_mf_remote' ] || exit 127; php '$_mf_remote' --import" 2>/dev/null; then
|
|
_unfiled=$(printf '%s' "$_plan" | php -r \
|
|
'echo implode(",", json_decode(stream_get_contents(STDIN),true)["unfiled"] ?? []);' 2>/dev/null)
|
|
else
|
|
warn "Could not mirror the folder layout to $MIRROR — falling back to one folder"
|
|
fi
|
|
fi
|
|
unset _plan _fbonly
|
|
fi
|
|
|
|
# ── 12b: the fallback folder gets only what is genuinely fallback ─────────────────────
|
|
# Unfiled containers plus anything named in PARTNERSHIP_FALLBACK_ONLY. Empty is the
|
|
# normal, correct outcome when every deployed container has a home in the layout.
|
|
if [[ -z "$_unfiled" ]]; then
|
|
echo " Folder layout mirrored — nothing left for ${MY_ID}-Fallback ✅"
|
|
FOLDER_OK=true
|
|
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
|
root@"$MIRROR_IP" \
|
|
"[ -f '$_ff_remote' ] || { echo missing; exit 127; }
|
|
php '$_ff_remote' --host=$(printf '%q' "$MY_ID") \
|
|
--containers=$(printf '%q' "$_unfiled") \
|
|
--icon=$(printf '%q' "$_icon")" 2>/dev/null; then
|
|
FOLDER_OK=true
|
|
else
|
|
warn "Could not group containers on $MIRROR — run $_ff_remote there by hand"
|
|
fi
|
|
unset _ff_local _ff_remote _mf_local _mf_remote _mirror_sd _icon _unfiled
|
|
fi
|
|
unset _deployed _deployed_csv
|
|
fi
|
|
|
|
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
|
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
|
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
|
[[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
|
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
|
|
|
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
|
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
|
|
echo " Step 1c2— Permit mirror: $(_ok "$UNBLOCK_OK")"
|
|
echo " Step 1d — Sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$AM_OWNER" != true ]] && echo "mirror — set by owner" || _ok "$ARM_OK"; } )"
|
|
echo " Step 1e — Auth appdata: $( [[ "$SKIP_APPDATA_PROVISION" == true ]] && echo "skipped" || _ok "$APPDATA_PROVISION_OK" )"
|
|
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
|
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
|
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
|
echo " Step 5 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
|
|
echo " Step 6 — Stop services: $(_skip "$SKIP_SERVICES_STACK" "$STEP_STOP_SERVICES_OK")"
|
|
echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo "skipped" || echo "${SERVICES_DEPLOYED} deployed, ${SERVICES_FAILED} failed" )"
|
|
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
|
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
|
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
|
|
echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
|
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
|
echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )"
|
|
echo " Step 12 — Grouping: $( [[ "$FOLDER_OK" == skipped ]] && echo "skipped" || _ok "$FOLDER_OK" )"
|
|
echo ""
|
|
|
|
if [[ "$ONBOARD_OK" == true ]]; then
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
|
echo "$ICON_DONE DONE — partnership established ✅"
|
|
echo "Verify with: Partnership/partnership_manager.sh --status"
|
|
else
|
|
error "Setup incomplete — resolve errors above and re-run"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
[[ "$ONBOARD_OK" == false ]] && exit 1
|
|
exit 0
|