Gmer4Lfe
a5755921ab
partnership_onboard.sh: - --phase1-only (OWNER): SSH key exchange + conf push only; safe to run before HOST2 has Varaverk installed; writes HOST2_PHASE1_DONE to setup.db - --phase2-only (OWNER): skips SSH, runs Steps 2-9 (containers, arr stack, onboard, arr sync, conf push); writes HOST2_PHASE2_DONE to setup.db - Mirror path: after SSH setup, SSHes OWNER and fires --phase2-only in background (nohup); reads OWNER's SCRIPTS_DIR from varaverk.cfg first; falls back to manual instruction if SSH fails partnership.php (include): vv_pt_nodes() reads setup.db and exposes onboard_phase (0/1/2) per remote node partnership.php (page): phase-aware Actions section per partner host; node cards show phase badge (not provisioned / awaiting onboard / done); Phase 1 and Phase 2 buttons call run.php with extra_args monitor.php: partner card logo (top-right), CPU thread count, RAM total GB, unRAID version row in remote stats grid
809 lines
37 KiB
Bash
Executable File
809 lines
37 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Partnership Onboard ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Runs once on both servers to establish a new partnership. Role is detected
|
|
# automatically via detect_hosts() — no flags needed to declare which side you are.
|
|
# Run on the mirror first (generates its SSH key), then on the owner to complete
|
|
# setup remotely.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# MIRROR PATH (1 step)
|
|
# Step 1: SSH key setup — generate keypair, copy to owner, update conf
|
|
# Owner completes the rest remotely. Mirror is done.
|
|
#
|
|
# OWNER PATH (8 steps)
|
|
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
|
# Step 2: Plugin install — FolderView3 and required plugins on mirror
|
|
# Step 3: Stop mirror auth — stop mirror's existing auth containers before replacing
|
|
# Step 4: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
|
# Mariadb/Redis health-checked before Authelia deploys
|
|
# Step 5: Stop mirror arr — stop mirror's existing arr containers before replacing
|
|
# Step 6: Deploy arr stack — push arr XMLs, pull images, create + start on mirror
|
|
# Step 7: Partnership onboard — configure WebUIs → owner IP, write state, FolderView3, Emby
|
|
# Step 8: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
|
# Step 9: Conf push — push master.conf + setup state to all listed hosts
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Credentials never in SSH command strings
|
|
# Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
|
|
# locally, SCPed to the remote, and executed there. Command-line args are never used
|
|
# to pass credentials — they'd appear in `ps` output and shell history on both servers.
|
|
#
|
|
# XML templates are the single source of truth for deployed containers
|
|
# The owner's templates-user/ XMLs define every container deployed on the mirror.
|
|
# The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
|
|
# Docker Manager can manage the containers after onboard without additional config.
|
|
#
|
|
# Dependency ordering in the auth stack is owner-enforced
|
|
# PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
|
|
# The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
|
|
# the script waits for the container to be healthy before continuing. This is a remote
|
|
# health check — the container must be running (or report healthy) before the next
|
|
# dependent is deployed.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check
|
|
# All operations run as root — SSH key management, docker operations, conf updates.
|
|
#
|
|
# SSH timeout on all remote calls
|
|
# Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
|
|
# slow or unreachable mirror.
|
|
#
|
|
# --dry-run shows exact actions without executing
|
|
# Every step prints what it would do. SCP, deploy, plugin install, arr sync —
|
|
# all dry-run safe.
|
|
#
|
|
# Step skip flags for partial re-runs
|
|
# --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
|
|
# resuming after a partial failure without re-running completed steps.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_PARTNERSHIP_AUTH_STACK
|
|
# XML filenames (from this server's templates-user/) to push and deploy on the
|
|
# mirror as its auth stack. Order matters: database deps before Authelia.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_REPLACE_CONTAINERS
|
|
# Containers to stop on the mirror before deploying the auth stack.
|
|
# Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
|
|
# Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
|
|
# the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
|
|
# Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
|
|
# Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# HOST*_PARTNERSHIP_ARR_STACK
|
|
# XML filenames to push and deploy on the mirror as its arr stack.
|
|
# Leave empty to skip arr stack deploy.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
|
|
#
|
|
# HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
|
|
# Arr containers to stop on the mirror before deploying the arr stack.
|
|
# Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
|
|
# Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# Partnership/partnership_onboard.sh
|
|
# Full onboard — role detected automatically
|
|
#
|
|
# Partnership/partnership_onboard.sh --dry-run
|
|
# Preview all steps without making changes
|
|
#
|
|
# Partnership/partnership_onboard.sh --log
|
|
# Verbose per-step output
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-ssh
|
|
# Skip SSH key setup (key already in place)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-auth-stack
|
|
# Skip auth stack stop + deploy (Steps 3-4)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-arr-stack
|
|
# Skip arr stack stop + deploy (Steps 5-6)
|
|
#
|
|
# Partnership/partnership_onboard.sh --skip-arr-sync
|
|
# Skip arr library bootstrap (Step 8)
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase1-only
|
|
# OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
|
|
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
|
#
|
|
# Partnership/partnership_onboard.sh --phase2-only
|
|
# OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
|
|
# by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
|
|
# Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
|
|
SSH_TIMEOUT=15
|
|
|
|
source "$SCRIPTS_ROOT/load_config.sh"
|
|
|
|
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
|
SKIP_SSH=false
|
|
SKIP_AUTH_STACK=false
|
|
SKIP_ARR_STACK=false
|
|
SKIP_ARR_SYNC=false
|
|
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
|
|
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
|
FILTERED_ARGS=()
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--skip-ssh) SKIP_SSH=true ;;
|
|
--skip-auth-stack) SKIP_AUTH_STACK=true ;;
|
|
--skip-arr-stack) SKIP_ARR_STACK=true ;;
|
|
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
|
--phase1-only) PHASE1_ONLY=true ;;
|
|
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
|
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
|
OWNER="${!OWNER_ID}"
|
|
MIRROR="${!MIRROR_ID}"
|
|
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
|
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
|
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
|
|
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
|
|
MIRROR_SSH_KEY="$SSH_KEY"
|
|
|
|
AM_OWNER=false
|
|
AM_MIRROR=false
|
|
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
|
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
|
|
|
EXTRA_FLAGS=()
|
|
[[ "$DRY_RUN" == true ]] && EXTRA_FLAGS+=("--dry-run")
|
|
[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log")
|
|
|
|
START=$(date +%s)
|
|
|
|
# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
|
|
write_onboard_phase() {
|
|
local target_id="$1" phase="$2"
|
|
local key="${target_id}_PHASE${phase}_DONE"
|
|
local state_file="/boot/config/varaverk_setup.db"
|
|
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
|
if grep -q "^${key}=" "$state_file" 2>/dev/null; then
|
|
sed -i "s|^${key}=.*|${key}=true|" "$state_file"
|
|
else
|
|
echo "${key}=true" >> "$state_file"
|
|
fi
|
|
command -v php &>/dev/null && \
|
|
php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null || true
|
|
}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo ""
|
|
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
|
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
|
echo ""
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER: deploy a container from a local Unraid template XML to a remote host ─────────────
|
|
#
|
|
# Parses Port / Path / Variable Config entries from the XML, SCPs the template and a
|
|
# self-contained deploy script to the remote, executes it, then cleans up both sides.
|
|
# Credentials are never passed as SSH command-line args — they stay in the SCPed script.
|
|
# ==============================================================================================
|
|
deploy_container_from_xml() {
|
|
local xml_file="$1" remote_ip="$2" ssh_key="$3"
|
|
local xml_name
|
|
xml_name=$(basename "$xml_file")
|
|
|
|
# Extract top-level fields
|
|
local name repo network extra privileged
|
|
name=$( awk 'match($0,/<Name>([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file")
|
|
repo=$( awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
|
|
network=$( awk 'match($0,/<Network>([^<]+)<\/Network>/, a){print a[1];exit}' "$xml_file")
|
|
extra=$( awk 'match($0,/<ExtraParams>([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file")
|
|
privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")
|
|
|
|
if [[ -z "$name" || -z "$repo" ]]; then
|
|
warn " Cannot parse Name/Repository from $xml_name — skipping"
|
|
return 1
|
|
fi
|
|
|
|
log "Deploying $name..."
|
|
|
|
# SCP the XML so Unraid Docker Manager recognises and can manage the container
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
|
|
warn " SCP failed for $xml_name — skipping $name"
|
|
return 1
|
|
}
|
|
else
|
|
warn " DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/"
|
|
fi
|
|
|
|
# Build a self-contained deploy script locally.
|
|
# Writing to a temp file keeps credentials out of SSH command strings.
|
|
local tmp_script
|
|
tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh)
|
|
chmod 600 "$tmp_script"
|
|
|
|
{
|
|
echo "#!/bin/bash"
|
|
echo "set -e"
|
|
echo ""
|
|
printf "docker pull %q 2>/dev/null || true\n" "$repo"
|
|
printf "docker stop %q 2>/dev/null || true\n" "$name"
|
|
printf "docker rm %q 2>/dev/null || true\n" "$name"
|
|
echo ""
|
|
printf "docker create --name %q --restart=unless-stopped" "$name"
|
|
[[ -n "$network" ]] && printf " --network=%q" "$network"
|
|
[[ "$privileged" == "true" ]] && printf " --privileged"
|
|
[[ -n "$extra" ]] && printf " %s" "$extra"
|
|
|
|
# Port mappings → -p host:container/proto
|
|
awk '/Type="Port"/ {
|
|
match($0, /Target="([^"]+)"/, t)
|
|
match($0, /Mode="([^"]+)"/, m)
|
|
match($0, />([^<]+)<\/Config>/, v)
|
|
if (t[1] != "" && v[1] != "") {
|
|
proto = (m[1] == "udp") ? "udp" : "tcp"
|
|
printf " -p %s:%s/%s", v[1], t[1], proto
|
|
}
|
|
}' "$xml_file"
|
|
|
|
# Volume mappings → -v 'host:container:mode'
|
|
awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ {
|
|
match($0, /Target="([^"]+)"/, t)
|
|
match($0, /Mode="([^"]+)"/, m)
|
|
match($0, />([^<]+)<\/Config>/, v)
|
|
if (t[1] != "" && v[1] != "") {
|
|
mode = (m[1] == "ro") ? "ro" : "rw"
|
|
printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q
|
|
}
|
|
}' "$xml_file"
|
|
|
|
# Environment variables → -e 'KEY=VALUE' (single-quoted to protect $ and special chars)
|
|
awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ {
|
|
match($0, /Target="([^"]+)"/, t)
|
|
match($0, />([^<]+)<\/Config>/, v)
|
|
if (t[1] != "" && v[1] != "") {
|
|
printf " -e %s%s=%s%s", q, t[1], v[1], q
|
|
}
|
|
}' "$xml_file"
|
|
|
|
printf " %q\n" "$repo"
|
|
echo ""
|
|
printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name"
|
|
} > "$tmp_script"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would deploy $name on $MIRROR"
|
|
rm -f "$tmp_script"
|
|
return 0
|
|
fi
|
|
|
|
# SCP deploy script → remote, execute, clean up both sides
|
|
local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
|
|
|
|
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
"$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
|
|
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
|
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
|
|
grep -q "deployed:${name}"; then
|
|
log " $name deployed ✅"
|
|
rm -f "$tmp_script"
|
|
return 0
|
|
else
|
|
warn " $name deployment failed — check $MIRROR manually"
|
|
rm -f "$tmp_script"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER: wait for a container on the remote to be healthy/running ─────────────────────────
|
|
#
|
|
# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined;
|
|
# falls back to the running state for containers with no healthcheck. Non-fatal after timeout
|
|
# — Authelia may take time to fully initialize but the deploy itself succeeded.
|
|
# ==============================================================================================
|
|
wait_for_container_healthy() {
|
|
local name="$1" remote_ip="$2" ssh_key="$3"
|
|
local max_wait=60 interval=5 elapsed=0
|
|
|
|
[[ "$DRY_RUN" == true ]] && return 0
|
|
|
|
log " Waiting for $name to be ready..."
|
|
while (( elapsed < max_wait )); do
|
|
local status
|
|
status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null)
|
|
r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null)
|
|
echo \${h:-\$r}" 2>/dev/null)
|
|
|
|
case "$status" in
|
|
healthy|true)
|
|
log " $name ready ✅"
|
|
return 0
|
|
;;
|
|
starting|unhealthy|false|"")
|
|
sleep "$interval"
|
|
(( elapsed += interval ))
|
|
;;
|
|
*)
|
|
sleep "$interval"
|
|
(( elapsed += interval ))
|
|
;;
|
|
esac
|
|
done
|
|
|
|
warn " $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)"
|
|
return 0
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
|
|
#
|
|
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
|
|
# use the same convention), and reads the named config array from the mirror's own conf.
|
|
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
|
|
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
|
|
#
|
|
# deploy_container_from_xml() already stops/removes containers with the same name as what's
|
|
# being deployed. This step handles containers with DIFFERENT names that conflict.
|
|
# ==============================================================================================
|
|
stop_mirror_stack() {
|
|
local config_var="$1" label="$2"
|
|
local -a to_stop=()
|
|
|
|
mapfile -t to_stop < <(
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
|
detect_hosts 2>/dev/null
|
|
printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
|
|
)
|
|
|
|
if [[ ${#to_stop[@]} -eq 0 ]]; then
|
|
log "No $label containers to stop on $MIRROR — skipping"
|
|
return 0
|
|
fi
|
|
|
|
log "Stopping $label on $MIRROR: ${to_stop[*]}"
|
|
for container in "${to_stop[@]}"; do
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would stop + rm $container on $MIRROR"
|
|
continue
|
|
fi
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
|
|
"docker stop '$container' 2>/dev/null
|
|
docker rm '$container' 2>/dev/null && echo removed" 2>/dev/null | \
|
|
grep -q removed && \
|
|
log " $container removed ✅" || \
|
|
log " $container not found on $MIRROR — skipping"
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER: deploy a stack of XMLs to the mirror, health-checking db deps between batches ────
|
|
#
|
|
# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout.
|
|
# This avoids the process-substitution capture problem: warn() writes to stdout, so any
|
|
# read -r X Y < <(func) would capture warn output as the count values.
|
|
# ==============================================================================================
|
|
_STACK_DEPLOYED=0
|
|
_STACK_FAILED=0
|
|
|
|
deploy_xml_stack() {
|
|
local -n xml_array_ref="$1"
|
|
_STACK_DEPLOYED=0
|
|
_STACK_FAILED=0
|
|
|
|
for xml_name in "${xml_array_ref[@]}"; do
|
|
local xml_file="${TEMPLATES_DIR}/${xml_name}"
|
|
if [[ ! -f "$xml_file" ]]; then
|
|
warn "$xml_name not found in $TEMPLATES_DIR — skipping"
|
|
(( _STACK_FAILED++ ))
|
|
continue
|
|
fi
|
|
|
|
# Extract container name to use for health-wait matching
|
|
local cname
|
|
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
|
|
|
if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
(( _STACK_DEPLOYED++ ))
|
|
# Health-check database deps before continuing — they must be ready before
|
|
# Authelia/app containers that depend on them can start cleanly.
|
|
if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then
|
|
wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
|
fi
|
|
else
|
|
(( _STACK_FAILED++ ))
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
if [[ "$AM_MIRROR" == true ]]; then
|
|
echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
|
|
echo ""
|
|
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
|
echo ""
|
|
|
|
if [[ "$SKIP_SSH" == true ]]; then
|
|
warn "Skipping SSH setup (--skip-ssh)"
|
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
log "SSH key ready ✅"
|
|
else
|
|
error "SSH key setup failed"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
|
echo ""
|
|
|
|
OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
|
PHASE2_TRIGGERED=false
|
|
|
|
if [[ -n "$OWNER_IP" ]]; then
|
|
# Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
|
|
OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
|
'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
|
|
OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/mnt/user/appdata/Varaverk}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
|
PHASE2_TRIGGERED=true
|
|
elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
|
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
|
|
2>/dev/null | grep -q triggered; then
|
|
log "Phase 2 triggered on $OWNER ✅"
|
|
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
|
|
PHASE2_TRIGGERED=true
|
|
else
|
|
warn "Could not auto-trigger Phase 2 on $OWNER"
|
|
fi
|
|
else
|
|
warn "Cannot resolve $OWNER Tailscale IP"
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
|
|
echo " SSH key: ready"
|
|
echo " Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
|
|
if [[ "$PHASE2_TRIGGERED" == false ]]; then
|
|
echo ""
|
|
echo " Run manually on $OWNER:"
|
|
echo " bash Partnership/partnership_onboard.sh --phase2-only"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
|
|
log "Mirror: $MIRROR ($MIRROR_IP)"
|
|
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
|
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
|
echo ""
|
|
|
|
STEP_SSH_OK=false
|
|
STEP_PLUGINS_OK=true
|
|
STEP_STOP_AUTH_OK=true
|
|
STEP_AUTH_OK=true
|
|
AUTH_DEPLOYED=0
|
|
AUTH_FAILED=0
|
|
STEP_STOP_ARR_OK=true
|
|
STEP_ARR_OK=true
|
|
ARR_DEPLOYED=0
|
|
ARR_FAILED=0
|
|
ONBOARD_OK=false
|
|
ARR_SYNC_OK=false
|
|
MASTER_PUSH_OK=false
|
|
|
|
# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
|
|
# Skipped when --phase2-only (SSH was already done in Phase 1).
|
|
echo "━━━ Step 1 — SSH Key Setup ━━━"
|
|
|
|
if [[ "$SKIP_SSH" == true ]]; then
|
|
warn "Skipping (--skip-ssh / --phase2-only)"
|
|
STEP_SSH_OK=true
|
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
|
log "SSH keys ready ✅"
|
|
STEP_SSH_OK=true
|
|
else
|
|
error "SSH key setup failed — aborting"
|
|
error "Re-run or use --skip-ssh if key is already set up"
|
|
exit 1
|
|
fi
|
|
|
|
# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
|
|
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
|
|
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
|
|
if [[ "$PHASE1_ONLY" == true ]]; then
|
|
echo ""
|
|
echo "━━━ Phase 1 — Conf Push ━━━"
|
|
|
|
CONF_PUSH_OK=false
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would push master.conf + state file to $MIRROR"
|
|
CONF_PUSH_OK=true
|
|
elif ! command -v php &>/dev/null; then
|
|
warn "php not available — push master.conf manually via Scheduler → master.conf → Save Conf"
|
|
else
|
|
push_output=$(php -r "
|
|
require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
|
|
\$results = vv_push_master_conf();
|
|
vv_push_setup_state();
|
|
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
|
|
\$failed = 0;
|
|
foreach (\$results as \$r) {
|
|
echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
|
|
if (!\$r['ok']) \$failed++;
|
|
}
|
|
exit(\$failed > 0 ? 1 : 0);
|
|
" 2>/dev/null)
|
|
push_rc=$?
|
|
echo "$push_output"
|
|
if [[ $push_rc -eq 0 ]]; then
|
|
log "Conf push complete ✅"
|
|
CONF_PUSH_OK=true
|
|
else
|
|
warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
|
fi
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
|
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
|
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
|
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " HOST2 ($MIRROR) can now install the Varaverk plugin."
|
|
echo " The wizard will detect the pushed conf and take the correct path."
|
|
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Step 2: Plugins ───────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 2 — Plugin Install on Mirror ━━━"
|
|
|
|
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && [[ -n "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then
|
|
FV3_PRESENT=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
"test -d /usr/local/emhttp/plugins/folder.view3 && echo yes" 2>/dev/null)
|
|
|
|
if [[ "$FV3_PRESENT" == "yes" ]]; then
|
|
log "FolderView3 already installed on $MIRROR ✅"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would install FolderView3 on $MIRROR"
|
|
else
|
|
log "Installing FolderView3 on $MIRROR..."
|
|
timeout 60 ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
|
|
"plugin install '$PARTNERSHIP_FOLDERVIEW3_URL' 2>/dev/null && echo installed" \
|
|
2>/dev/null | grep -q installed && \
|
|
log "FolderView3 installed ✅" || {
|
|
warn "FolderView3 install failed — install manually from Community Applications"
|
|
STEP_PLUGINS_OK=false
|
|
}
|
|
fi
|
|
else
|
|
log "FolderView3 not configured — skipping"
|
|
fi
|
|
|
|
# ── Step 3: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 3 — Stop Mirror Auth Stack ━━━"
|
|
|
|
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
|
warn "Skipping (--skip-auth-stack)"
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
|
|
fi
|
|
|
|
# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 4 — Deploy Auth Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_AUTH_STACK" == true ]]; then
|
|
warn "Skipping (--skip-auth-stack)"
|
|
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
|
|
warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
|
|
warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
|
|
STEP_AUTH_OK=false
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_AUTH_STACK
|
|
AUTH_DEPLOYED=$_STACK_DEPLOYED
|
|
AUTH_FAILED=$_STACK_FAILED
|
|
echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
|
|
[[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
|
|
fi
|
|
|
|
# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 5 — Stop Mirror Arr Stack ━━━"
|
|
|
|
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
|
warn "Skipping (--skip-arr-stack)"
|
|
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
|
|
log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
|
|
SKIP_ARR_STACK=true
|
|
else
|
|
stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
|
|
fi
|
|
|
|
# ── Step 6: Deploy arr stack on mirror ───────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 6 — Deploy Arr Stack on Mirror ━━━"
|
|
|
|
if [[ "$SKIP_ARR_STACK" == true ]]; then
|
|
warn "Skipping (--skip-arr-stack)"
|
|
else
|
|
deploy_xml_stack PARTNERSHIP_ARR_STACK
|
|
ARR_DEPLOYED=$_STACK_DEPLOYED
|
|
ARR_FAILED=$_STACK_FAILED
|
|
echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
|
|
[[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
|
|
fi
|
|
|
|
# ── Step 7: Partnership onboard ───────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 7 — Partnership Onboard ━━━"
|
|
|
|
if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
|
|
echo "Partnership onboard complete ✅"
|
|
ONBOARD_OK=true
|
|
else
|
|
error "Partnership onboard failed"
|
|
ONBOARD_OK=false
|
|
fi
|
|
|
|
# ── Step 8: Arr library bootstrap ─────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ Step 8 — Arr Library Bootstrap ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete"
|
|
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
|
|
warn "Skipping (--skip-arr-sync)"
|
|
elif [[ ! -f "$SCRIPTS_ROOT/Media/arr_sync.sh" ]]; then
|
|
warn "arr_sync.sh not found — run Media/arr_sync.sh manually once arrs are live"
|
|
elif bash "$SCRIPTS_ROOT/Media/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
|
|
echo "Arr bootstrap complete ✅"
|
|
ARR_SYNC_OK=true
|
|
else
|
|
warn "Arr sync had errors — partnership still valid"
|
|
warn "Re-run Media/arr_sync.sh once all arr containers are live"
|
|
fi
|
|
|
|
# ── Step 9: Push master.conf to all listed hosts ──────────────────────────────────────────────
|
|
# SSH is now established and all partners have the plugin installed.
|
|
# Push the authoritative master.conf so every listed host is in sync immediately.
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Step 9 — master.conf Push ━━━"
|
|
|
|
if [[ "$ONBOARD_OK" == false ]]; then
|
|
warn "Skipping — onboard did not complete"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would push master.conf to all listed hosts"
|
|
MASTER_PUSH_OK=true
|
|
elif ! command -v php &>/dev/null; then
|
|
warn "php not available — push master.conf manually via Scheduler → master.conf → Save Conf"
|
|
else
|
|
push_output=$(php -r "
|
|
require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
|
|
\$results = vv_push_master_conf();
|
|
vv_push_setup_state();
|
|
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
|
|
\$failed = 0;
|
|
foreach (\$results as \$r) {
|
|
echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
|
|
if (!\$r['ok']) \$failed++;
|
|
}
|
|
exit(\$failed > 0 ? 1 : 0);
|
|
" 2>/dev/null)
|
|
push_rc=$?
|
|
echo "$push_output"
|
|
if [[ $push_rc -eq 0 ]]; then
|
|
echo "master.conf sync complete ✅"
|
|
MASTER_PUSH_OK=true
|
|
else
|
|
warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
|
|
fi
|
|
fi
|
|
|
|
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
|
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
|
END=$(date +%s)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
|
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
|
[[ "$PHASE2_ONLY" == true ]] && echo " Mode: Phase 2 (triggered by HOST2 notification)"
|
|
echo " Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
|
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
|
|
|
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
|
echo " Step 2 — Plugins: $(_ok "$STEP_PLUGINS_OK")"
|
|
echo " Step 3 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
|
echo " Step 4 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
|
echo " Step 5 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
|
echo " Step 6 — Arr stack: $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
|
|
echo " Step 7 — Onboard: $(_ok "$ONBOARD_OK")"
|
|
echo " Step 8 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
|
echo " Step 9 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
|
echo ""
|
|
|
|
if [[ "$ONBOARD_OK" == true ]]; then
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
|
echo "$ICON_DONE DONE — partnership established ✅"
|
|
echo "Verify with: Partnership/partnership_manager.sh --status"
|
|
else
|
|
error "Setup incomplete — resolve errors above and re-run"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
[[ "$ONBOARD_OK" == false ]] && exit 1
|
|
exit 0
|