1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Partnership Onboard ============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Runs once on both servers to establish a new partnership. Role is detected
9	# automatically via detect_hosts() — no flags needed to declare which side you are.
10	# Run on the mirror first (generates its SSH key), then on the owner to complete
11	# setup remotely.
12	#
13	# ==============================================================================================
14	# OPERATIONAL MODEL
15	# ==============================================================================================
16	#
17	# MIRROR PATH (1 step)
18	#   Step 1: SSH key setup — generate keypair, copy to owner, update conf
19	#           Owner completes the rest remotely. Mirror is done.
20	#
21	# OWNER PATH (8 steps)
22	#   Step 1: SSH key setup        — generate keypair, install on mirror, update conf
23	#   Step 2: Plugin install       — FolderView3 and required plugins on mirror
24	#   Step 3: Stop mirror auth     — stop mirror's existing auth containers before replacing
25	#   Step 4: Deploy auth stack    — push XMLs, pull images, create + start on mirror
26	#                                  Mariadb/Redis health-checked before Authelia deploys
27	#   Step 5: Stop mirror arr      — stop mirror's existing arr containers before replacing
28	#   Step 6: Deploy arr stack     — push arr XMLs, pull images, create + start on mirror
29	#   Step 7: Partnership onboard  — configure WebUIs → owner IP, write state, FolderView3, Emby
30	#   Step 8: Arr bootstrap        — bidirectional library sync (arr_sync.sh)
31	#   Step 9: Conf push            — push master.conf + setup state to all listed hosts
32	#
33	# ==============================================================================================
34	# DESIGN PRINCIPLES
35	# ==============================================================================================
36	#
37	# Credentials never in SSH command strings
38	#   Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
39	#   locally, SCPed to the remote, and executed there. Command-line args are never used
40	#   to pass credentials — they'd appear in `ps` output and shell history on both servers.
41	#
42	# XML templates are the single source of truth for deployed containers
43	#   The owner's templates-user/ XMLs define every container deployed on the mirror.
44	#   The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
45	#   Docker Manager can manage the containers after onboard without additional config.
46	#
47	# Dependency ordering in the auth stack is owner-enforced
48	#   PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
49	#   The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
50	#   the script waits for the container to be healthy before continuing. This is a remote
51	#   health check — the container must be running (or report healthy) before the next
52	#   dependent is deployed.
53	#
54	# ==============================================================================================
55	# OPERATIONAL SAFEGUARDS
56	# ==============================================================================================
57	#
58	# Root check
59	#   All operations run as root — SSH key management, docker operations, conf updates.
60	#
61	# SSH timeout on all remote calls
62	#   Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
63	#   slow or unreachable mirror.
64	#
65	# --dry-run shows exact actions without executing
66	#   Every step prints what it would do. SCP, deploy, plugin install, arr sync —
67	#   all dry-run safe.
68	#
69	# Step skip flags for partial re-runs
70	#   --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
71	#   resuming after a partial failure without re-running completed steps.
72	#
73	# ==============================================================================================
74	# CONFIGURATION
75	# ==============================================================================================
76	#
77	# host*.conf
78	#
79	#   HOST*_PARTNERSHIP_AUTH_STACK
80	#       XML filenames (from this server's templates-user/) to push and deploy on the
81	#       mirror as its auth stack. Order matters: database deps before Authelia.
82	#       Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
83	#
84	#   HOST*_PARTNERSHIP_REPLACE_CONTAINERS
85	#       Containers to stop on the mirror before deploying the auth stack.
86	#       Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
87	#       Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
88	#       the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
89	#       Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
90	#       Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
91	#
92	#   HOST*_PARTNERSHIP_ARR_STACK
93	#       XML filenames to push and deploy on the mirror as its arr stack.
94	#       Leave empty to skip arr stack deploy.
95	#       Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
96	#
97	#   HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
98	#       Arr containers to stop on the mirror before deploying the arr stack.
99	#       Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
100	#       Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
101	#
102	# ==============================================================================================
103	# RUNTIME MODES
104	# ==============================================================================================
105	#
106	# Partnership/partnership_onboard.sh
107	#     Full onboard — role detected automatically
108	#
109	# Partnership/partnership_onboard.sh --dry-run
110	#     Preview all steps without making changes
111	#
112	# Partnership/partnership_onboard.sh --log
113	#     Verbose per-step output
114	#
115	# Partnership/partnership_onboard.sh --skip-ssh
116	#     Skip SSH key setup (key already in place)
117	#
118	# Partnership/partnership_onboard.sh --skip-auth-stack
119	#     Skip auth stack stop + deploy (Steps 3-4)
120	#
121	# Partnership/partnership_onboard.sh --skip-arr-stack
122	#     Skip arr stack stop + deploy (Steps 5-6)
123	#
124	# Partnership/partnership_onboard.sh --skip-arr-sync
125	#     Skip arr library bootstrap (Step 8)
126	#
127	# Partnership/partnership_onboard.sh --phase1-only
128	#     OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
129	#     Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
130	#
131	# Partnership/partnership_onboard.sh --phase2-only
132	#     OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
133	#     by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
134	#     Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
135	#
136	# ==============================================================================================
137
138	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
139	SCRIPTS_ROOT="$SCRIPT_DIR/.."
140	TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
141	SSH_TIMEOUT=15
142
143	source "$SCRIPTS_ROOT/load_config.sh"
144
145	# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
146	SKIP_SSH=false
147	SKIP_AUTH_STACK=false
148	SKIP_ARR_STACK=false
149	SKIP_ARR_SYNC=false
150	PHASE1_ONLY=false   # OWNER: SSH + conf push only (HOST2 not yet installed)
151	PHASE2_ONLY=false   # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
152	FILTERED_ARGS=()
153
154	for arg in "$@"; do
155	    case "$arg" in
156	        --skip-ssh)        SKIP_SSH=true ;;
157	        --skip-auth-stack) SKIP_AUTH_STACK=true ;;
158	        --skip-arr-stack)  SKIP_ARR_STACK=true ;;
159	        --skip-arr-sync)   SKIP_ARR_SYNC=true ;;
160	        --phase1-only)     PHASE1_ONLY=true ;;
161	        --phase2-only)     PHASE2_ONLY=true; SKIP_SSH=true ;;
162	        *)                 FILTERED_ARGS+=("$arg") ;;
163	    esac
164	done
165
166	parse_args "${FILTERED_ARGS[@]}"
167
168	# ==============================================================================================
169	# ━━━ Setup ━━━
170	# ==============================================================================================
171	[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
172
173	acquire_lock
174
175	if ! command -v docker &>/dev/null; then
176	    error "Docker command not found"
177	    exit 1
178	fi
179
180	detect_hosts
181
182	OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
183	MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
184	OWNER="${!OWNER_ID}"
185	MIRROR="${!MIRROR_ID}"
186	# SSH_KEY (set by detect_hosts) is this server's own private key.
187	# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
188	# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
189	# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
190	MIRROR_SSH_KEY="$SSH_KEY"
191
192	AM_OWNER=false
193	AM_MIRROR=false
194	[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
195	[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
196
197	EXTRA_FLAGS=()
198	[[ "$DRY_RUN"  == true ]] && EXTRA_FLAGS+=("--dry-run")
199	[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log")
200
201	START=$(date +%s)
202
203	# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
204	write_onboard_phase() {
205	    local target_id="$1" phase="$2"
206	    local key="${target_id}_PHASE${phase}_DONE"
207	    local state_file="${VARAVERK_SETUP_FILE:-${STATE_DIR:-/boot/config}/varaverk_setup.db}"
208	    [[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
209	    if grep -q "^${key}=" "$state_file" 2>/dev/null; then
210	        sed -i "s|^${key}=.*|${key}=true|" "$state_file"
211	    else
212	        echo "${key}=true" >> "$state_file"
213	    fi
214	    command -v php &>/dev/null && \
215	        php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null || true
216	}
217
218	echo ""
219	echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
220	echo ""
221	echo "  Role:    $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
222	echo "  This:    $MY_ID ($LOCAL_SERVER_NAME)"
223	echo "  Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
224	echo ""
225	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
226
227	# ==============================================================================================
228	# ── HELPER: deploy a container from a local Unraid template XML to a remote host ─────────────
229	#
230	# Parses Port / Path / Variable Config entries from the XML, SCPs the template and a
231	# self-contained deploy script to the remote, executes it, then cleans up both sides.
232	# Credentials are never passed as SSH command-line args — they stay in the SCPed script.
233	# ==============================================================================================
234	deploy_container_from_xml() {
235	    local xml_file="$1" remote_ip="$2" ssh_key="$3"
236	    local xml_name
237	    xml_name=$(basename "$xml_file")
238
239	    # Extract top-level fields
240	    local name repo network extra privileged
241	    name=$(       awk 'match($0,/<Name>([^<]+)<\/Name>/,        a){print a[1];exit}' "$xml_file")
242	    repo=$(       awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
243	    network=$(    awk 'match($0,/<Network>([^<]+)<\/Network>/,   a){print a[1];exit}' "$xml_file")
244	    extra=$(      awk 'match($0,/<ExtraParams>([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file")
245	    privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")
246
247	    if [[ -z "$name" || -z "$repo" ]]; then
248	        warn "  Cannot parse Name/Repository from $xml_name — skipping"
249	        return 1
250	    fi
251
252	    log "Deploying $name..."
253
254	    # SCP the XML so Unraid Docker Manager recognises and can manage the container
255	    if [[ "$DRY_RUN" == false ]]; then
256	        timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
257	            "$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
258	            warn "  SCP failed for $xml_name — skipping $name"
259	            return 1
260	        }
261	    else
262	        warn "  DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/"
263	    fi
264
265	    # Build a self-contained deploy script locally.
266	    # Writing to a temp file keeps credentials out of SSH command strings.
267	    local tmp_script
268	    tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh)
269	    chmod 600 "$tmp_script"
270
271	    {
272	        echo "#!/bin/bash"
273	        echo "set -e"
274	        echo ""
275	        printf "docker pull %q 2>/dev/null || true\n" "$repo"
276	        printf "docker stop %q 2>/dev/null || true\n" "$name"
277	        printf "docker rm   %q 2>/dev/null || true\n" "$name"
278	        echo ""
279	        printf "docker create --name %q --restart=unless-stopped" "$name"
280	        [[ -n "$network"         ]] && printf " --network=%q"  "$network"
281	        [[ "$privileged" == "true" ]] && printf " --privileged"
282	        [[ -n "$extra"           ]] && printf " %s" "$extra"
283
284	        # Port mappings → -p host:container/proto
285	        awk '/Type="Port"/ {
286	            match($0, /Target="([^"]+)"/, t)
287	            match($0, /Mode="([^"]+)"/,   m)
288	            match($0, />([^<]+)<\/Config>/, v)
289	            if (t[1] != "" && v[1] != "") {
290	                proto = (m[1] == "udp") ? "udp" : "tcp"
291	                printf " -p %s:%s/%s", v[1], t[1], proto
292	            }
293	        }' "$xml_file"
294
295	        # Volume mappings → -v 'host:container:mode'
296	        awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ {
297	            match($0, /Target="([^"]+)"/, t)
298	            match($0, /Mode="([^"]+)"/,   m)
299	            match($0, />([^<]+)<\/Config>/, v)
300	            if (t[1] != "" && v[1] != "") {
301	                mode = (m[1] == "ro") ? "ro" : "rw"
302	                printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q
303	            }
304	        }' "$xml_file"
305
306	        # Environment variables → -e 'KEY=VALUE'  (single-quoted to protect $ and special chars)
307	        awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ {
308	            match($0, /Target="([^"]+)"/, t)
309	            match($0, />([^<]+)<\/Config>/, v)
310	            if (t[1] != "" && v[1] != "") {
311	                printf " -e %s%s=%s%s", q, t[1], v[1], q
312	            }
313	        }' "$xml_file"
314
315	        printf " %q\n" "$repo"
316	        echo ""
317	        printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name"
318	    } > "$tmp_script"
319
320	    if [[ "$DRY_RUN" == true ]]; then
321	        warn "  DRY RUN — would deploy $name on $MIRROR"
322	        rm -f "$tmp_script"
323	        return 0
324	    fi
325
326	    # SCP deploy script → remote, execute, clean up both sides
327	    local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
328
329	    if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
330	        "$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
331	       timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
332	        "bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
333	        grep -q "deployed:${name}"; then
334	        log "  $name deployed ✅"
335	        rm -f "$tmp_script"
336	        return 0
337	    else
338	        warn "  $name deployment failed — check $MIRROR manually"
339	        rm -f "$tmp_script"
340	        return 1
341	    fi
342	}
343
344	# ==============================================================================================
345	# ── HELPER: wait for a container on the remote to be healthy/running ─────────────────────────
346	#
347	# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined;
348	# falls back to the running state for containers with no healthcheck. Non-fatal after timeout
349	# — Authelia may take time to fully initialize but the deploy itself succeeded.
350	# ==============================================================================================
351	wait_for_container_healthy() {
352	    local name="$1" remote_ip="$2" ssh_key="$3"
353	    local max_wait=60 interval=5 elapsed=0
354
355	    [[ "$DRY_RUN" == true ]] && return 0
356
357	    log "  Waiting for $name to be ready..."
358	    while (( elapsed < max_wait )); do
359	        local status
360	        status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
361	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
362	            "h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null)
363	             r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null)
364	             echo \${h:-\$r}" 2>/dev/null)
365
366	        case "$status" in
367	            healthy|true)
368	                log "  $name ready ✅"
369	                return 0
370	                ;;
371	            starting|unhealthy|false|"")
372	                sleep "$interval"
373	                (( elapsed += interval ))
374	                ;;
375	            *)
376	                sleep "$interval"
377	                (( elapsed += interval ))
378	                ;;
379	        esac
380	    done
381
382	    warn "  $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)"
383	    return 0
384	}
385
386	# ==============================================================================================
387	# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
388	#
389	# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
390	# use the same convention), and reads the named config array from the mirror's own conf.
391	# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
392	# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
393	#
394	# deploy_container_from_xml() already stops/removes containers with the same name as what's
395	# being deployed. This step handles containers with DIFFERENT names that conflict.
396	# ==============================================================================================
397	stop_mirror_stack() {
398	    local config_var="$1" label="$2"
399	    local -a to_stop=()
400
401	    mapfile -t to_stop < <(
402	        timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
403	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
404	            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
405	             detect_hosts 2>/dev/null
406	             printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
407	    )
408
409	    if [[ ${#to_stop[@]} -eq 0 ]]; then
410	        log "No $label containers to stop on $MIRROR — skipping"
411	        return 0
412	    fi
413
414	    log "Stopping $label on $MIRROR: ${to_stop[*]}"
415	    for container in "${to_stop[@]}"; do
416	        if [[ "$DRY_RUN" == true ]]; then
417	            warn "  DRY RUN — would stop + rm $container on $MIRROR"
418	            continue
419	        fi
420	        timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
421	            -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
422	            "docker stop '$container' 2>/dev/null
423	             docker rm   '$container' 2>/dev/null && echo removed" 2>/dev/null | \
424	            grep -q removed && \
425	            log "  $container removed ✅" || \
426	            log "  $container not found on $MIRROR — skipping"
427	    done
428	}
429
430	# ==============================================================================================
431	# ── HELPER: deploy a stack of XMLs to the mirror, health-checking db deps between batches ────
432	#
433	# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout.
434	# This avoids the process-substitution capture problem: warn() writes to stdout, so any
435	# read -r X Y < <(func) would capture warn output as the count values.
436	# ==============================================================================================
437	_STACK_DEPLOYED=0
438	_STACK_FAILED=0
439
440	deploy_xml_stack() {
441	    local -n xml_array_ref="$1"
442	    _STACK_DEPLOYED=0
443	    _STACK_FAILED=0
444
445	    for xml_name in "${xml_array_ref[@]}"; do
446	        local xml_file="${TEMPLATES_DIR}/${xml_name}"
447	        if [[ ! -f "$xml_file" ]]; then
448	            warn "$xml_name not found in $TEMPLATES_DIR — skipping"
449	            (( _STACK_FAILED++ ))
450	            continue
451	        fi
452
453	        # Extract container name to use for health-wait matching
454	        local cname
455	        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
456
457	        if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
458	            (( _STACK_DEPLOYED++ ))
459	            # Health-check database deps before continuing — they must be ready before
460	            # Authelia/app containers that depend on them can start cleanly.
461	            if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then
462	                wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY"
463	            fi
464	        else
465	            (( _STACK_FAILED++ ))
466	        fi
467	    done
468	}
469
470	# ==============================================================================================
471	# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
472	# ==============================================================================================
473	if [[ "$AM_MIRROR" == true ]]; then
474	    echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
475	    echo ""
476	    echo "  Mirror sets up SSH keys, then notifies Owner to run Phase 2."
477	    echo ""
478
479	    if [[ "$SKIP_SSH" == true ]]; then
480	        warn "Skipping SSH setup (--skip-ssh)"
481	    elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
482	        log "SSH key ready ✅"
483	    else
484	        error "SSH key setup failed"
485	        exit 1
486	    fi
487
488	    echo ""
489	    echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
490	    echo ""
491
492	    OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
493	    PHASE2_TRIGGERED=false
494
495	    if [[ -n "$OWNER_IP" ]]; then
496	        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
497	        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
498	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
499	            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
500	        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"
501
502	        if [[ "$DRY_RUN" == true ]]; then
503	            warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
504	            PHASE2_TRIGGERED=true
505	        elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
506	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
507	            "nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
508	            2>/dev/null | grep -q triggered; then
509	            log "Phase 2 triggered on $OWNER ✅"
510	            log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
511	            PHASE2_TRIGGERED=true
512	        else
513	            warn "Could not auto-trigger Phase 2 on $OWNER"
514	        fi
515	    else
516	        warn "Cannot resolve $OWNER Tailscale IP"
517	    fi
518
519	    echo ""
520	    echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
521	    echo "  SSH key:        ready"
522	    echo "  Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
523	    if [[ "$PHASE2_TRIGGERED" == false ]]; then
524	        echo ""
525	        echo "  Run manually on $OWNER:"
526	        echo "    bash Partnership/partnership_onboard.sh --phase2-only"
527	    fi
528	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
529	    exit 0
530	fi
531
532	# ==============================================================================================
533	# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
534	# ==============================================================================================
535
536	MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
537	[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
538	log "Mirror: $MIRROR ($MIRROR_IP)"
539	[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
540	[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
541	echo ""
542
543	STEP_SSH_OK=false
544	STEP_PLUGINS_OK=true
545	STEP_STOP_AUTH_OK=true
546	STEP_AUTH_OK=true
547	AUTH_DEPLOYED=0
548	AUTH_FAILED=0
549	STEP_STOP_ARR_OK=true
550	STEP_ARR_OK=true
551	ARR_DEPLOYED=0
552	ARR_FAILED=0
553	ONBOARD_OK=false
554	ARR_SYNC_OK=false
555	MASTER_PUSH_OK=false
556
557	# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
558	# Skipped when --phase2-only (SSH was already done in Phase 1).
559	echo "━━━ Step 1 — SSH Key Setup ━━━"
560
561	if [[ "$SKIP_SSH" == true ]]; then
562	    warn "Skipping (--skip-ssh)"
563	    STEP_SSH_OK=true
564	elif [[ "$PHASE1_ONLY" == true ]]; then
565	    # Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
566	    # hanging for a password prompt with no TTY.
567	    if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
568	        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
569	        log "SSH to $MIRROR already works ✅ — skipping key install"
570	        STEP_SSH_OK=true
571	    else
572	        # Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
573	        if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
574	            log "SSH keys ready ✅"
575	            STEP_SSH_OK=true
576	        else
577	            # Soft-fail: generate key locally if not present, then tell user to install manually
578	            warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
579	            if [[ -f "$SSH_KEY" ]]; then
580	                log "Local key exists at: $SSH_KEY"
581	            else
582	                bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
583	            fi
584	            if [[ -f "${SSH_KEY}.pub" ]]; then
585	                echo ""
586	                echo "  Install this key on $MIRROR to complete SSH setup:"
587	                echo "  ┌─────────────────────────────────────────────────────"
588	                cat "${SSH_KEY}.pub" | sed 's/^/  │ /'
589	                echo "  └─────────────────────────────────────────────────────"
590	                echo "  Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
591	                echo "  Then click 'Push Conf' in the Partnership tab."
592	                # Write key-ready flag so UI can show the manual-install state
593	                [[ "$DRY_RUN" == false ]] && {
594	                    local kflag="${MIRROR_ID}_KEY_READY"
595	                    local _setup_f="${VARAVERK_SETUP_FILE:-${STATE_DIR:-/boot/config}/varaverk_setup.db}"
596	                    grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
597	                        && sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
598	                        || echo "${kflag}=true" >> "$_setup_f"
599	                }
600	            fi
601	            STEP_SSH_OK=false
602	        fi
603	    fi
604	elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
605	    log "SSH keys ready ✅"
606	    STEP_SSH_OK=true
607	else
608	    error "SSH key setup failed — aborting"
609	    error "Re-run or use --skip-ssh if key is already set up"
610	    exit 1
611	fi
612
613	# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
614	# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
615	# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
616	if [[ "$PHASE1_ONLY" == true ]]; then
617	    if [[ "$STEP_SSH_OK" == false ]]; then
618	        # SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
619	        # UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
620	        echo ""
621	        echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
622	        bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
623	            warn "Local setup had issues — FolderView3 may need manual setup"
624
625	        END=$(date +%s)
626	        echo ""
627	        echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
628	        echo "  SSH keys:    key generated ✅ — NOT yet installed on $MIRROR ⚠"
629	        echo "  Conf push:   skipped (needs SSH access to $MIRROR)"
630	        echo "  HOST1 setup: done ✅"
631	        echo "  Duration:    $(format_duration $(( END - START )))"
632	        echo ""
633	        echo "  ACTION NEEDED: install the key on $MIRROR:"
634	        echo "    ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
635	        echo "  Then click 'Push Conf' in Partnership tab, or run:"
636	        echo "    bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
637	        echo "━━━━━━━━━━━━━━━━━━━━━━━"
638	        exit 0
639	    fi
640
641	    echo ""
642	    echo "━━━ Phase 1 — Conf Push ━━━"
643
644	    CONF_PUSH_OK=false
645	    if [[ "$DRY_RUN" == true ]]; then
646	        warn "DRY RUN — would push master.conf + state file to $MIRROR"
647	        CONF_PUSH_OK=true
648	    elif ! command -v php &>/dev/null; then
649	        warn "php not available — push master.conf manually via Scheduler → master.conf → Save Conf"
650	    else
651	        push_output=$(php -r "
652	require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
653	\$results = vv_push_master_conf();
654	vv_push_setup_state();
655	if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
656	\$failed = 0;
657	foreach (\$results as \$r) {
658	    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
659	    if (!\$r['ok']) \$failed++;
660	}
661	exit(\$failed > 0 ? 1 : 0);
662	" 2>/dev/null)
663	        push_rc=$?
664	        echo "$push_output"
665	        if [[ $push_rc -eq 0 ]]; then
666	            log "Conf push complete ✅"
667	            CONF_PUSH_OK=true
668	        else
669	            warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
670	        fi
671	    fi
672
673	    # HOST1 local setup — runs immediately without needing HOST2
674	    echo ""
675	    echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
676	    bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
677	        warn "Local setup had issues — FolderView3 may need manual setup"
678
679	    [[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
680
681	    END=$(date +%s)
682	    echo ""
683	    echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
684	    echo "  SSH keys:     $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
685	    echo "  Conf push:    $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
686	    echo "  HOST1 setup:  done ✅"
687	    echo "  Duration:     $(format_duration $(( END - START )))"
688	    echo ""
689	    echo "  HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
690	    echo "  The wizard will detect the pushed conf and take the correct path."
691	    echo "  When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
692	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
693	    exit 0
694	fi
695
696	# ── Step 2: Plugins ───────────────────────────────────────────────────────────────────────────
697	echo ""
698	echo "━━━ Step 2 — Plugin Install on Mirror ━━━"
699
700	if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && [[ -n "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then
701	    FV3_PRESENT=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
702	        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
703	        "test -d /usr/local/emhttp/plugins/folder.view3 && echo yes" 2>/dev/null)
704
705	    if [[ "$FV3_PRESENT" == "yes" ]]; then
706	        log "FolderView3 already installed on $MIRROR ✅"
707	    elif [[ "$DRY_RUN" == true ]]; then
708	        warn "DRY RUN — would install FolderView3 on $MIRROR"
709	    else
710	        log "Installing FolderView3 on $MIRROR..."
711	        timeout 60 ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
712	            "plugin install '$PARTNERSHIP_FOLDERVIEW3_URL' 2>/dev/null && echo installed" \
713	            2>/dev/null | grep -q installed && \
714	            log "FolderView3 installed ✅" || {
715	            warn "FolderView3 install failed — install manually from Community Applications"
716	            STEP_PLUGINS_OK=false
717	        }
718	    fi
719	else
720	    log "FolderView3 not configured — skipping"
721	fi
722
723	# ── Step 3: Stop mirror's existing auth stack ─────────────────────────────────────────────────
724	echo ""
725	echo "━━━ Step 3 — Stop Mirror Auth Stack ━━━"
726
727	if [[ "$SKIP_AUTH_STACK" == true ]]; then
728	    warn "Skipping (--skip-auth-stack)"
729	else
730	    stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
731	fi
732
733	# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
734	echo ""
735	echo "━━━ Step 4 — Deploy Auth Stack on Mirror ━━━"
736
737	if [[ "$SKIP_AUTH_STACK" == true ]]; then
738	    warn "Skipping (--skip-auth-stack)"
739	elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
740	    warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
741	    warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
742	    STEP_AUTH_OK=false
743	else
744	    deploy_xml_stack PARTNERSHIP_AUTH_STACK
745	    AUTH_DEPLOYED=$_STACK_DEPLOYED
746	    AUTH_FAILED=$_STACK_FAILED
747	    echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
748	    [[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
749	fi
750
751	# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
752	echo ""
753	echo "━━━ Step 5 — Stop Mirror Arr Stack ━━━"
754
755	if [[ "$SKIP_ARR_STACK" == true ]]; then
756	    warn "Skipping (--skip-arr-stack)"
757	elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
758	    log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
759	    SKIP_ARR_STACK=true
760	else
761	    stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
762	fi
763
764	# ── Step 6: Deploy arr stack on mirror ───────────────────────────────────────────────────────
765	echo ""
766	echo "━━━ Step 6 — Deploy Arr Stack on Mirror ━━━"
767
768	if [[ "$SKIP_ARR_STACK" == true ]]; then
769	    warn "Skipping (--skip-arr-stack)"
770	else
771	    deploy_xml_stack PARTNERSHIP_ARR_STACK
772	    ARR_DEPLOYED=$_STACK_DEPLOYED
773	    ARR_FAILED=$_STACK_FAILED
774	    echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
775	    [[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
776	fi
777
778	# ── Step 7: Partnership onboard ───────────────────────────────────────────────────────────────
779	echo ""
780	echo "━━━ Step 7 — Partnership Onboard ━━━"
781
782	if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
783	    echo "Partnership onboard complete ✅"
784	    ONBOARD_OK=true
785	else
786	    error "Partnership onboard failed"
787	    ONBOARD_OK=false
788	fi
789
790	# ── Step 8: Arr library bootstrap ─────────────────────────────────────────────────────────────
791	echo ""
792	echo "━━━ Step 8 — Arr Library Bootstrap ━━━"
793
794	if [[ "$ONBOARD_OK" == false ]]; then
795	    warn "Skipping — onboard did not complete"
796	elif [[ "$SKIP_ARR_SYNC" == true ]]; then
797	    warn "Skipping (--skip-arr-sync)"
798	elif [[ ! -f "$SCRIPTS_ROOT/Media/arr_sync.sh" ]]; then
799	    warn "arr_sync.sh not found — run Media/arr_sync.sh manually once arrs are live"
800	elif bash "$SCRIPTS_ROOT/Media/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
801	    echo "Arr bootstrap complete ✅"
802	    ARR_SYNC_OK=true
803	else
804	    warn "Arr sync had errors — partnership still valid"
805	    warn "Re-run Media/arr_sync.sh once all arr containers are live"
806	fi
807
808	# ── Step 9: Push master.conf to all listed hosts ──────────────────────────────────────────────
809	# SSH is now established and all partners have the plugin installed.
810	# Push the authoritative master.conf so every listed host is in sync immediately.
811	echo ""
812	echo "━━━ $ICON_GEAR Step 9 — master.conf Push ━━━"
813
814	if [[ "$ONBOARD_OK" == false ]]; then
815	    warn "Skipping — onboard did not complete"
816	elif [[ "$DRY_RUN" == true ]]; then
817	    warn "DRY RUN — would push master.conf to all listed hosts"
818	    MASTER_PUSH_OK=true
819	elif ! command -v php &>/dev/null; then
820	    warn "php not available — push master.conf manually via Scheduler → master.conf → Save Conf"
821	else
822	    push_output=$(php -r "
823	require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
824	\$results = vv_push_master_conf();
825	vv_push_setup_state();
826	if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
827	\$failed = 0;
828	foreach (\$results as \$r) {
829	    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
830	    if (!\$r['ok']) \$failed++;
831	}
832	exit(\$failed > 0 ? 1 : 0);
833	" 2>/dev/null)
834	    push_rc=$?
835	    echo "$push_output"
836	    if [[ $push_rc -eq 0 ]]; then
837	        echo "master.conf sync complete ✅"
838	        MASTER_PUSH_OK=true
839	    else
840	        warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
841	    fi
842	fi
843
844	# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
845	[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
846
847	# ── Summary ───────────────────────────────────────────────────────────────────────────────────
848	END=$(date +%s)
849	echo ""
850	echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
851	echo "  Owner:    $MY_ID ($LOCAL_SERVER_NAME)"
852	echo "  Mirror:   $MIRROR ($MIRROR_IP)"
853	[[ "$PHASE2_ONLY" == true ]] && echo "  Mode:     Phase 2 (triggered by HOST2 notification)"
854	echo "  Duration: $(format_duration $(( END - START )))"
855	echo ""
856	_ok()   { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
857	_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
858
859	echo "  Step 1 — SSH keys:      $(_skip "$SKIP_SSH"        "$STEP_SSH_OK")"
860	echo "  Step 2 — Plugins:       $(_ok "$STEP_PLUGINS_OK")"
861	echo "  Step 3 — Stop auth:     $(_skip "$SKIP_AUTH_STACK"  "$STEP_STOP_AUTH_OK")"
862	echo "  Step 4 — Auth stack:    $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
863	echo "  Step 5 — Stop arr:      $(_skip "$SKIP_ARR_STACK"   "$STEP_STOP_ARR_OK")"
864	echo "  Step 6 — Arr stack:     $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
865	echo "  Step 7 — Onboard:       $(_ok "$ONBOARD_OK")"
866	echo "  Step 8 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
867	echo "  Step 9 — Conf push:     $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
868	echo ""
869
870	if [[ "$ONBOARD_OK" == true ]]; then
871	    [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
872	        echo "$ICON_DONE DONE — partnership established ✅"
873	    echo "Verify with: Partnership/partnership_manager.sh --status"
874	else
875	    error "Setup incomplete — resolve errors above and re-run"
876	fi
877	echo "━━━━━━━━━━━━━━━━━━━━━━━"
878	[[ "$ONBOARD_OK" == false ]] && exit 1
879	exit 0
880

#!/bin/bash
# ==============================================================================================
# ============================= Partnership Onboard ============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Runs once on both servers to establish a new partnership. Role is detected
# automatically via detect_hosts() — no flags needed to declare which side you are.
# Run on the mirror first (generates its SSH key), then on the owner to complete
# setup remotely.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# MIRROR PATH (1 step)
#   Step 1: SSH key setup — generate keypair, copy to owner, update conf
#           Owner completes the rest remotely. Mirror is done.
#
# OWNER PATH (8 steps)
#   Step 1: SSH key setup        — generate keypair, install on mirror, update conf
#   Step 2: Plugin install       — FolderView3 and required plugins on mirror
#   Step 3: Stop mirror auth     — stop mirror's existing auth containers before replacing
#   Step 4: Deploy auth stack    — push XMLs, pull images, create + start on mirror
#                                  Mariadb/Redis health-checked before Authelia deploys
#   Step 5: Stop mirror arr      — stop mirror's existing arr containers before replacing
#   Step 6: Deploy arr stack     — push arr XMLs, pull images, create + start on mirror
#   Step 7: Partnership onboard  — configure WebUIs → owner IP, write state, FolderView3, Emby
#   Step 8: Arr bootstrap        — bidirectional library sync (arr_sync.sh)
#   Step 9: Conf push            — push master.conf + setup state to all listed hosts
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Credentials never in SSH command strings
#   Auth stack containers hold API keys, DB passwords, etc. The deploy script is written
#   locally, SCPed to the remote, and executed there. Command-line args are never used
#   to pass credentials — they'd appear in `ps` output and shell history on both servers.
#
# XML templates are the single source of truth for deployed containers
#   The owner's templates-user/ XMLs define every container deployed on the mirror.
#   The same XMLs that Unraid's Docker Manager uses are what get SCPed — the mirror's
#   Docker Manager can manage the containers after onboard without additional config.
#
# Dependency ordering in the auth stack is owner-enforced
#   PARTNERSHIP_AUTH_STACK order matters: Mariadb and Redis must come before Authelia.
#   The array is ordered correctly in host1.conf. After each Mariadb/Redis deploy,
#   the script waits for the container to be healthy before continuing. This is a remote
#   health check — the container must be running (or report healthy) before the next
#   dependent is deployed.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root check
#   All operations run as root — SSH key management, docker operations, conf updates.
#
# SSH timeout on all remote calls
#   Every ssh/scp call uses SSH_TIMEOUT. No operation hangs indefinitely on a
#   slow or unreachable mirror.
#
# --dry-run shows exact actions without executing
#   Every step prints what it would do. SCP, deploy, plugin install, arr sync —
#   all dry-run safe.
#
# Step skip flags for partial re-runs
#   --skip-ssh, --skip-auth-stack, --skip-arr-stack, --skip-arr-sync allow
#   resuming after a partial failure without re-running completed steps.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_PARTNERSHIP_AUTH_STACK
#       XML filenames (from this server's templates-user/) to push and deploy on the
#       mirror as its auth stack. Order matters: database deps before Authelia.
#       Aliased by detect_hosts() → PARTNERSHIP_AUTH_STACK
#
#   HOST*_PARTNERSHIP_REPLACE_CONTAINERS
#       Containers to stop on the mirror before deploying the auth stack.
#       Defined in the MIRROR's own conf (host*.conf on HOST2) — never in HOST1's conf.
#       Read live from the mirror via SSH during Step 3 (sources mirror's load_config.sh at
#       the same $SCRIPTS_ROOT path — convention: both servers use the same repo location).
#       Leave empty on HOST2 if no conflicting containers exist (fresh mirror: nothing to stop).
#       Aliased by detect_hosts() → PARTNERSHIP_REPLACE_CONTAINERS (on the mirror)
#
#   HOST*_PARTNERSHIP_ARR_STACK
#       XML filenames to push and deploy on the mirror as its arr stack.
#       Leave empty to skip arr stack deploy.
#       Aliased by detect_hosts() → PARTNERSHIP_ARR_STACK
#
#   HOST*_PARTNERSHIP_ARR_REPLACE_CONTAINERS
#       Arr containers to stop on the mirror before deploying the arr stack.
#       Same rule as PARTNERSHIP_REPLACE_CONTAINERS: defined in mirror's own conf, never HOST1's.
#       Aliased by detect_hosts() → PARTNERSHIP_ARR_REPLACE_CONTAINERS (on the mirror)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# Partnership/partnership_onboard.sh
#     Full onboard — role detected automatically
#
# Partnership/partnership_onboard.sh --dry-run
#     Preview all steps without making changes
#
# Partnership/partnership_onboard.sh --log
#     Verbose per-step output
#
# Partnership/partnership_onboard.sh --skip-ssh
#     Skip SSH key setup (key already in place)
#
# Partnership/partnership_onboard.sh --skip-auth-stack
#     Skip auth stack stop + deploy (Steps 3-4)
#
# Partnership/partnership_onboard.sh --skip-arr-stack
#     Skip arr stack stop + deploy (Steps 5-6)
#
# Partnership/partnership_onboard.sh --skip-arr-sync
#     Skip arr library bootstrap (Step 8)
#
# Partnership/partnership_onboard.sh --phase1-only
#     OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
#     Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
#
# Partnership/partnership_onboard.sh --phase2-only
#     OWNER only: container deploy + arr + onboard (skips SSH). Triggered automatically
#     by HOST2 after it completes its Mirror-path onboard. Can also be run manually.
#     Writes HOST2_PHASE2_DONE=true to varaverk_setup.db.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
SSH_TIMEOUT=15

source "$SCRIPTS_ROOT/load_config.sh"

# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
SKIP_SSH=false
SKIP_AUTH_STACK=false
SKIP_ARR_STACK=false
SKIP_ARR_SYNC=false
PHASE1_ONLY=false   # OWNER: SSH + conf push only (HOST2 not yet installed)
PHASE2_ONLY=false   # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
FILTERED_ARGS=()

for arg in "$@"; do
    case "$arg" in
        --skip-ssh)        SKIP_SSH=true ;;
        --skip-auth-stack) SKIP_AUTH_STACK=true ;;
        --skip-arr-stack)  SKIP_ARR_STACK=true ;;
        --skip-arr-sync)   SKIP_ARR_SYNC=true ;;
        --phase1-only)     PHASE1_ONLY=true ;;
        --phase2-only)     PHASE2_ONLY=true; SKIP_SSH=true ;;
        *)                 FILTERED_ARGS+=("$arg") ;;
    esac
done

parse_args "${FILTERED_ARGS[@]}"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
OWNER="${!OWNER_ID}"
MIRROR="${!MIRROR_ID}"
# SSH_KEY (set by detect_hosts) is this server's own private key.
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
# HOST{N}_SSH_KEY lives in host{N}.conf — with sparse checkout, the other server's
# conf is never present here. Always use SSH_KEY (local private key) for outbound SSH.
MIRROR_SSH_KEY="$SSH_KEY"

AM_OWNER=false
AM_MIRROR=false
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true

EXTRA_FLAGS=()
[[ "$DRY_RUN"  == true ]] && EXTRA_FLAGS+=("--dry-run")
[[ "$LOG_MODE" == true ]] && EXTRA_FLAGS+=("--log")

START=$(date +%s)

# ── Helper: write phase completion flag to setup.db + push to remotes ─────────────────────────
write_onboard_phase() {
    local target_id="$1" phase="$2"
    local key="${target_id}_PHASE${phase}_DONE"
    local state_file="${VARAVERK_SETUP_FILE:-${STATE_DIR:-/boot/config}/varaverk_setup.db}"
    [[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
    if grep -q "^${key}=" "$state_file" 2>/dev/null; then
        sed -i "s|^${key}=.*|${key}=true|" "$state_file"
    else
        echo "${key}=true" >> "$state_file"
    fi
    command -v php &>/dev/null && \
        php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null || true
}

echo ""
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
echo "  Role:    $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
echo "  This:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "  Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
echo ""
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"

# ==============================================================================================
# ── HELPER: deploy a container from a local Unraid template XML to a remote host ─────────────
#
# Parses Port / Path / Variable Config entries from the XML, SCPs the template and a
# self-contained deploy script to the remote, executes it, then cleans up both sides.
# Credentials are never passed as SSH command-line args — they stay in the SCPed script.
# ==============================================================================================
deploy_container_from_xml() {
    local xml_file="$1" remote_ip="$2" ssh_key="$3"
    local xml_name
    xml_name=$(basename "$xml_file")

    # Extract top-level fields
    local name repo network extra privileged
    name=$(       awk 'match($0,/<Name>([^<]+)<\/Name>/,        a){print a[1];exit}' "$xml_file")
    repo=$(       awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
    network=$(    awk 'match($0,/<Network>([^<]+)<\/Network>/,   a){print a[1];exit}' "$xml_file")
    extra=$(      awk 'match($0,/<ExtraParams>([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file")
    privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")

    if [[ -z "$name" || -z "$repo" ]]; then
        warn "  Cannot parse Name/Repository from $xml_name — skipping"
        return 1
    fi

    log "Deploying $name..."

    # SCP the XML so Unraid Docker Manager recognises and can manage the container
    if [[ "$DRY_RUN" == false ]]; then
        timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
            "$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
            warn "  SCP failed for $xml_name — skipping $name"
            return 1
        }
    else
        warn "  DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/"
    fi

    # Build a self-contained deploy script locally.
    # Writing to a temp file keeps credentials out of SSH command strings.
    local tmp_script
    tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh)
    chmod 600 "$tmp_script"

    {
        echo "#!/bin/bash"
        echo "set -e"
        echo ""
        printf "docker pull %q 2>/dev/null || true\n" "$repo"
        printf "docker stop %q 2>/dev/null || true\n" "$name"
        printf "docker rm   %q 2>/dev/null || true\n" "$name"
        echo ""
        printf "docker create --name %q --restart=unless-stopped" "$name"
        [[ -n "$network"         ]] && printf " --network=%q"  "$network"
        [[ "$privileged" == "true" ]] && printf " --privileged"
        [[ -n "$extra"           ]] && printf " %s" "$extra"

        # Port mappings → -p host:container/proto
        awk '/Type="Port"/ {
            match($0, /Target="([^"]+)"/, t)
            match($0, /Mode="([^"]+)"/,   m)
            match($0, />([^<]+)<\/Config>/, v)
            if (t[1] != "" && v[1] != "") {
                proto = (m[1] == "udp") ? "udp" : "tcp"
                printf " -p %s:%s/%s", v[1], t[1], proto
            }
        }' "$xml_file"

        # Volume mappings → -v 'host:container:mode'
        awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ {
            match($0, /Target="([^"]+)"/, t)
            match($0, /Mode="([^"]+)"/,   m)
            match($0, />([^<]+)<\/Config>/, v)
            if (t[1] != "" && v[1] != "") {
                mode = (m[1] == "ro") ? "ro" : "rw"
                printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q
            }
        }' "$xml_file"

        # Environment variables → -e 'KEY=VALUE'  (single-quoted to protect $ and special chars)
        awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ {
            match($0, /Target="([^"]+)"/, t)
            match($0, />([^<]+)<\/Config>/, v)
            if (t[1] != "" && v[1] != "") {
                printf " -e %s%s=%s%s", q, t[1], v[1], q
            }
        }' "$xml_file"

        printf " %q\n" "$repo"
        echo ""
        printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name"
    } > "$tmp_script"

    if [[ "$DRY_RUN" == true ]]; then
        warn "  DRY RUN — would deploy $name on $MIRROR"
        rm -f "$tmp_script"
        return 0
    fi

    # SCP deploy script → remote, execute, clean up both sides
    local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"

    if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
        "$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
       timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
        "bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
        grep -q "deployed:${name}"; then
        log "  $name deployed ✅"
        rm -f "$tmp_script"
        return 0
    else
        warn "  $name deployment failed — check $MIRROR manually"
        rm -f "$tmp_script"
        return 1
    fi
}

# ==============================================================================================
# ── HELPER: wait for a container on the remote to be healthy/running ─────────────────────────
#
# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined;
# falls back to the running state for containers with no healthcheck. Non-fatal after timeout
# — Authelia may take time to fully initialize but the deploy itself succeeded.
# ==============================================================================================
wait_for_container_healthy() {
    local name="$1" remote_ip="$2" ssh_key="$3"
    local max_wait=60 interval=5 elapsed=0

    [[ "$DRY_RUN" == true ]] && return 0

    log "  Waiting for $name to be ready..."
    while (( elapsed < max_wait )); do
        local status
        status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
            "h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null)
             r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null)
             echo \${h:-\$r}" 2>/dev/null)

        case "$status" in
            healthy|true)
                log "  $name ready ✅"
                return 0
                ;;
            starting|unhealthy|false|"")
                sleep "$interval"
                (( elapsed += interval ))
                ;;
            *)
                sleep "$interval"
                (( elapsed += interval ))
                ;;
        esac
    done

    warn "  $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)"
    return 0
}

# ==============================================================================================
# ── HELPER: stop containers on the mirror by reading its own conf via SSH ────────────────────
#
# SSHes to the mirror, sources its load_config.sh at the same $SCRIPTS_ROOT path (both servers
# use the same convention), and reads the named config array from the mirror's own conf.
# HOST2's container list stays in HOST2's host2.conf — not duplicated in HOST1's conf.
# Fails gracefully if scripts aren't present yet or the array is empty (nothing to stop).
#
# deploy_container_from_xml() already stops/removes containers with the same name as what's
# being deployed. This step handles containers with DIFFERENT names that conflict.
# ==============================================================================================
stop_mirror_stack() {
    local config_var="$1" label="$2"
    local -a to_stop=()

    mapfile -t to_stop < <(
        timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
             detect_hosts 2>/dev/null
             printf '%s\n' \"\${${config_var}[@]:-}\"" 2>/dev/null | grep -v '^$'
    )

    if [[ ${#to_stop[@]} -eq 0 ]]; then
        log "No $label containers to stop on $MIRROR — skipping"
        return 0
    fi

    log "Stopping $label on $MIRROR: ${to_stop[*]}"
    for container in "${to_stop[@]}"; do
        if [[ "$DRY_RUN" == true ]]; then
            warn "  DRY RUN — would stop + rm $container on $MIRROR"
            continue
        fi
        timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
            "docker stop '$container' 2>/dev/null
             docker rm   '$container' 2>/dev/null && echo removed" 2>/dev/null | \
            grep -q removed && \
            log "  $container removed ✅" || \
            log "  $container not found on $MIRROR — skipping"
    done
}

# ==============================================================================================
# ── HELPER: deploy a stack of XMLs to the mirror, health-checking db deps between batches ────
#
# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout.
# This avoids the process-substitution capture problem: warn() writes to stdout, so any
# read -r X Y < <(func) would capture warn output as the count values.
# ==============================================================================================
_STACK_DEPLOYED=0
_STACK_FAILED=0

deploy_xml_stack() {
    local -n xml_array_ref="$1"
    _STACK_DEPLOYED=0
    _STACK_FAILED=0

    for xml_name in "${xml_array_ref[@]}"; do
        local xml_file="${TEMPLATES_DIR}/${xml_name}"
        if [[ ! -f "$xml_file" ]]; then
            warn "$xml_name not found in $TEMPLATES_DIR — skipping"
            (( _STACK_FAILED++ ))
            continue
        fi

        # Extract container name to use for health-wait matching
        local cname
        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")

        if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
            (( _STACK_DEPLOYED++ ))
            # Health-check database deps before continuing — they must be ready before
            # Authelia/app containers that depend on them can start cleanly.
            if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then
                wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY"
            fi
        else
            (( _STACK_FAILED++ ))
        fi
    done
}

# ==============================================================================================
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
if [[ "$AM_MIRROR" == true ]]; then
    echo "━━━ Step 1/2 — SSH Key Setup (Mirror) ━━━"
    echo ""
    echo "  Mirror sets up SSH keys, then notifies Owner to run Phase 2."
    echo ""

    if [[ "$SKIP_SSH" == true ]]; then
        warn "Skipping SSH setup (--skip-ssh)"
    elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
        log "SSH key ready ✅"
    else
        error "SSH key setup failed"
        exit 1
    fi

    echo ""
    echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
    echo ""

    OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
    PHASE2_TRIGGERED=false

    if [[ -n "$OWNER_IP" ]]; then
        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
            PHASE2_TRIGGERED=true
        elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            "nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
            2>/dev/null | grep -q triggered; then
            log "Phase 2 triggered on $OWNER ✅"
            log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
            PHASE2_TRIGGERED=true
        else
            warn "Could not auto-trigger Phase 2 on $OWNER"
        fi
    else
        warn "Cannot resolve $OWNER Tailscale IP"
    fi

    echo ""
    echo "━━━━━ $ICON_SUMMARY MIRROR SETUP COMPLETE ━━━━━"
    echo "  SSH key:        ready"
    echo "  Phase 2 on $OWNER: $( [[ "$PHASE2_TRIGGERED" == true ]] && echo "triggered ✅" || echo "needs manual trigger ⚠" )"
    if [[ "$PHASE2_TRIGGERED" == false ]]; then
        echo ""
        echo "  Run manually on $OWNER:"
        echo "    bash Partnership/partnership_onboard.sh --phase2-only"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP — is Tailscale running?"; exit 1; }
log "Mirror: $MIRROR ($MIRROR_IP)"
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
echo ""

STEP_SSH_OK=false
STEP_PLUGINS_OK=true
STEP_STOP_AUTH_OK=true
STEP_AUTH_OK=true
AUTH_DEPLOYED=0
AUTH_FAILED=0
STEP_STOP_ARR_OK=true
STEP_ARR_OK=true
ARR_DEPLOYED=0
ARR_FAILED=0
ONBOARD_OK=false
ARR_SYNC_OK=false
MASTER_PUSH_OK=false

# ── Step 1: SSH ───────────────────────────────────────────────────────────────────────────────
# Skipped when --phase2-only (SSH was already done in Phase 1).
echo "━━━ Step 1 — SSH Key Setup ━━━"

if [[ "$SKIP_SSH" == true ]]; then
    warn "Skipping (--skip-ssh)"
    STEP_SSH_OK=true
elif [[ "$PHASE1_ONLY" == true ]]; then
    # Phase 1 in background: test if SSH already works first — avoids ssh-copy-id
    # hanging for a password prompt with no TTY.
    if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" exit 0 2>/dev/null; then
        log "SSH to $MIRROR already works ✅ — skipping key install"
        STEP_SSH_OK=true
    else
        # Key not yet on HOST2 — try ssh_setup.sh (works interactively, may fail in background)
        if bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
            log "SSH keys ready ✅"
            STEP_SSH_OK=true
        else
            # Soft-fail: generate key locally if not present, then tell user to install manually
            warn "Could not install key on $MIRROR automatically (no terminal for password prompt)"
            if [[ -f "$SSH_KEY" ]]; then
                log "Local key exists at: $SSH_KEY"
            else
                bash "$SCRIPT_DIR/ssh_setup.sh" --key-only "${EXTRA_FLAGS[@]}" 2>/dev/null || true
            fi
            if [[ -f "${SSH_KEY}.pub" ]]; then
                echo ""
                echo "  Install this key on $MIRROR to complete SSH setup:"
                echo "  ┌─────────────────────────────────────────────────────"
                cat "${SSH_KEY}.pub" | sed 's/^/  │ /'
                echo "  └─────────────────────────────────────────────────────"
                echo "  Run on a terminal: ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
                echo "  Then click 'Push Conf' in the Partnership tab."
                # Write key-ready flag so UI can show the manual-install state
                [[ "$DRY_RUN" == false ]] && {
                    local kflag="${MIRROR_ID}_KEY_READY"
                    local _setup_f="${VARAVERK_SETUP_FILE:-${STATE_DIR:-/boot/config}/varaverk_setup.db}"
                    grep -q "^${kflag}=" "$_setup_f" 2>/dev/null \
                        && sed -i "s|^${kflag}=.*|${kflag}=true|" "$_setup_f" \
                        || echo "${kflag}=true" >> "$_setup_f"
                }
            fi
            STEP_SSH_OK=false
        fi
    fi
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
    log "SSH keys ready ✅"
    STEP_SSH_OK=true
else
    error "SSH key setup failed — aborting"
    error "Re-run or use --skip-ssh if key is already set up"
    exit 1
fi

# ── Phase 1 exit point ────────────────────────────────────────────────────────────────────────
# --phase1-only: SSH + conf push is all HOST1 needs to do before HOST2 installs Varaverk.
# HOST2's wizard will detect the pushed master.conf + state file and take the correct path.
if [[ "$PHASE1_ONLY" == true ]]; then
    if [[ "$STEP_SSH_OK" == false ]]; then
        # SSH key not yet installed on HOST2 — can't push conf, but local setup still runs.
        # UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
        echo ""
        echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
        bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
            warn "Local setup had issues — FolderView3 may need manual setup"

        END=$(date +%s)
        echo ""
        echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
        echo "  SSH keys:    key generated ✅ — NOT yet installed on $MIRROR ⚠"
        echo "  Conf push:   skipped (needs SSH access to $MIRROR)"
        echo "  HOST1 setup: done ✅"
        echo "  Duration:    $(format_duration $(( END - START )))"
        echo ""
        echo "  ACTION NEEDED: install the key on $MIRROR:"
        echo "    ssh-copy-id -i ${SSH_KEY}.pub root@${MIRROR_IP}"
        echo "  Then click 'Push Conf' in Partnership tab, or run:"
        echo "    bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
        echo "━━━━━━━━━━━━━━━━━━━━━━━"
        exit 0
    fi

    echo ""
    echo "━━━ Phase 1 — Conf Push ━━━"

    CONF_PUSH_OK=false
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would push master.conf + state file to $MIRROR"
        CONF_PUSH_OK=true
    elif ! command -v php &>/dev/null; then
        warn "php not available — push master.conf manually via Scheduler → master.conf → Save Conf"
    else
        push_output=$(php -r "
require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
\$results = vv_push_master_conf();
vv_push_setup_state();
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
\$failed = 0;
foreach (\$results as \$r) {
    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
    if (!\$r['ok']) \$failed++;
}
exit(\$failed > 0 ? 1 : 0);
" 2>/dev/null)
        push_rc=$?
        echo "$push_output"
        if [[ $push_rc -eq 0 ]]; then
            log "Conf push complete ✅"
            CONF_PUSH_OK=true
        else
            warn "Conf push had failures — retry via Scheduler → master.conf → Save Conf"
        fi
    fi

    # HOST1 local setup — runs immediately without needing HOST2
    echo ""
    echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
    bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
        warn "Local setup had issues — FolderView3 may need manual setup"

    [[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1

    END=$(date +%s)
    echo ""
    echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
    echo "  SSH keys:     $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
    echo "  Conf push:    $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
    echo "  HOST1 setup:  done ✅"
    echo "  Duration:     $(format_duration $(( END - START )))"
    echo ""
    echo "  HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
    echo "  The wizard will detect the pushed conf and take the correct path."
    echo "  When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ── Step 2: Plugins ───────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 2 — Plugin Install on Mirror ━━━"

if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && [[ -n "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then
    FV3_PRESENT=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
        "test -d /usr/local/emhttp/plugins/folder.view3 && echo yes" 2>/dev/null)

    if [[ "$FV3_PRESENT" == "yes" ]]; then
        log "FolderView3 already installed on $MIRROR ✅"
    elif [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would install FolderView3 on $MIRROR"
    else
        log "Installing FolderView3 on $MIRROR..."
        timeout 60 ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
            "plugin install '$PARTNERSHIP_FOLDERVIEW3_URL' 2>/dev/null && echo installed" \
            2>/dev/null | grep -q installed && \
            log "FolderView3 installed ✅" || {
            warn "FolderView3 install failed — install manually from Community Applications"
            STEP_PLUGINS_OK=false
        }
    fi
else
    log "FolderView3 not configured — skipping"
fi

# ── Step 3: Stop mirror's existing auth stack ─────────────────────────────────────────────────
echo ""
echo "━━━ Step 3 — Stop Mirror Auth Stack ━━━"

if [[ "$SKIP_AUTH_STACK" == true ]]; then
    warn "Skipping (--skip-auth-stack)"
else
    stop_mirror_stack "PARTNERSHIP_REPLACE_CONTAINERS" "auth stack"
fi

# ── Step 4: Deploy auth stack on mirror ───────────────────────────────────────────────────────
echo ""
echo "━━━ Step 4 — Deploy Auth Stack on Mirror ━━━"

if [[ "$SKIP_AUTH_STACK" == true ]]; then
    warn "Skipping (--skip-auth-stack)"
elif [[ ${#PARTNERSHIP_AUTH_STACK[@]} -eq 0 ]]; then
    warn "PARTNERSHIP_AUTH_STACK not set in ${MY_ID} conf — skipping auth stack deploy"
    warn "Add HOST${MY_ID: -1}_PARTNERSHIP_AUTH_STACK to host${MY_ID: -1}.conf"
    STEP_AUTH_OK=false
else
    deploy_xml_stack PARTNERSHIP_AUTH_STACK
    AUTH_DEPLOYED=$_STACK_DEPLOYED
    AUTH_FAILED=$_STACK_FAILED
    echo "Auth stack: $AUTH_DEPLOYED deployed, $AUTH_FAILED failed"
    [[ "$AUTH_FAILED" -gt 0 ]] && STEP_AUTH_OK=false
fi

# ── Step 5: Stop mirror's existing arr stack ──────────────────────────────────────────────────
echo ""
echo "━━━ Step 5 — Stop Mirror Arr Stack ━━━"

if [[ "$SKIP_ARR_STACK" == true ]]; then
    warn "Skipping (--skip-arr-stack)"
elif [[ ${#PARTNERSHIP_ARR_STACK[@]} -eq 0 ]]; then
    log "PARTNERSHIP_ARR_STACK not configured — skipping arr stack deploy"
    SKIP_ARR_STACK=true
else
    stop_mirror_stack "PARTNERSHIP_ARR_REPLACE_CONTAINERS" "arr stack"
fi

# ── Step 6: Deploy arr stack on mirror ───────────────────────────────────────────────────────
echo ""
echo "━━━ Step 6 — Deploy Arr Stack on Mirror ━━━"

if [[ "$SKIP_ARR_STACK" == true ]]; then
    warn "Skipping (--skip-arr-stack)"
else
    deploy_xml_stack PARTNERSHIP_ARR_STACK
    ARR_DEPLOYED=$_STACK_DEPLOYED
    ARR_FAILED=$_STACK_FAILED
    echo "Arr stack: $ARR_DEPLOYED deployed, $ARR_FAILED failed"
    [[ "$ARR_FAILED" -gt 0 ]] && STEP_ARR_OK=false
fi

# ── Step 7: Partnership onboard ───────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 7 — Partnership Onboard ━━━"

if bash "$SCRIPTS_ROOT/Partnership/partnership_manager.sh" --onboard "${EXTRA_FLAGS[@]}"; then
    echo "Partnership onboard complete ✅"
    ONBOARD_OK=true
else
    error "Partnership onboard failed"
    ONBOARD_OK=false
fi

# ── Step 8: Arr library bootstrap ─────────────────────────────────────────────────────────────
echo ""
echo "━━━ Step 8 — Arr Library Bootstrap ━━━"

if [[ "$ONBOARD_OK" == false ]]; then
    warn "Skipping — onboard did not complete"
elif [[ "$SKIP_ARR_SYNC" == true ]]; then
    warn "Skipping (--skip-arr-sync)"
elif [[ ! -f "$SCRIPTS_ROOT/Media/arr_sync.sh" ]]; then
    warn "arr_sync.sh not found — run Media/arr_sync.sh manually once arrs are live"
elif bash "$SCRIPTS_ROOT/Media/arr_sync.sh" "${EXTRA_FLAGS[@]}"; then
    echo "Arr bootstrap complete ✅"
    ARR_SYNC_OK=true
else
    warn "Arr sync had errors — partnership still valid"
    warn "Re-run Media/arr_sync.sh once all arr containers are live"
fi

# ── Step 9: Push master.conf to all listed hosts ──────────────────────────────────────────────
# SSH is now established and all partners have the plugin installed.
# Push the authoritative master.conf so every listed host is in sync immediately.
echo ""
echo "━━━ $ICON_GEAR Step 9 — master.conf Push ━━━"

if [[ "$ONBOARD_OK" == false ]]; then
    warn "Skipping — onboard did not complete"
elif [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would push master.conf to all listed hosts"
    MASTER_PUSH_OK=true
elif ! command -v php &>/dev/null; then
    warn "php not available — push master.conf manually via Scheduler → master.conf → Save Conf"
else
    push_output=$(php -r "
require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
\$results = vv_push_master_conf();
vv_push_setup_state();
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
\$failed = 0;
foreach (\$results as \$r) {
    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
    if (!\$r['ok']) \$failed++;
}
exit(\$failed > 0 ? 1 : 0);
" 2>/dev/null)
    push_rc=$?
    echo "$push_output"
    if [[ $push_rc -eq 0 ]]; then
        echo "master.conf sync complete ✅"
        MASTER_PUSH_OK=true
    else
        warn "master.conf push had failures — retry via Scheduler → master.conf → Save Conf"
    fi
fi

# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2

# ── Summary ───────────────────────────────────────────────────────────────────────────────────
END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
echo "  Owner:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "  Mirror:   $MIRROR ($MIRROR_IP)"
[[ "$PHASE2_ONLY" == true ]] && echo "  Mode:     Phase 2 (triggered by HOST2 notification)"
echo "  Duration: $(format_duration $(( END - START )))"
echo ""
_ok()   { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }

echo "  Step 1 — SSH keys:      $(_skip "$SKIP_SSH"        "$STEP_SSH_OK")"
echo "  Step 2 — Plugins:       $(_ok "$STEP_PLUGINS_OK")"
echo "  Step 3 — Stop auth:     $(_skip "$SKIP_AUTH_STACK"  "$STEP_STOP_AUTH_OK")"
echo "  Step 4 — Auth stack:    $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
echo "  Step 5 — Stop arr:      $(_skip "$SKIP_ARR_STACK"   "$STEP_STOP_ARR_OK")"
echo "  Step 6 — Arr stack:     $( [[ "$SKIP_ARR_STACK" == true ]] && echo "skipped" || echo "${ARR_DEPLOYED} deployed, ${ARR_FAILED} failed" )"
echo "  Step 7 — Onboard:       $(_ok "$ONBOARD_OK")"
echo "  Step 8 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
echo "  Step 9 — Conf push:     $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
echo ""

if [[ "$ONBOARD_OK" == true ]]; then
    [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
        echo "$ICON_DONE DONE — partnership established ✅"
    echo "Verify with: Partnership/partnership_manager.sh --status"
else
    error "Setup incomplete — resolve errors above and re-run"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$ONBOARD_OK" == false ]] && exit 1
exit 0


Now let me read the offboard script to see what's OS-specific there too.