1	#!/bin/bash
2	# ==============================================================================================
3	# ====================== Partnership — Unraid Container Adapter ================================
4	# ==============================================================================================
5	#
6	# Sourced by partnership_onboard.sh and partnership_offboard.sh via:
7	#   source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
8	#
9	# Provides container deploy/cleanup functions specific to the Unraid platform:
10	#   - Docker container deployment from Unraid CA XML templates
11	#
12	# Functions use variables from the calling script's scope (sourced, not exec'd):
13	#   MIRROR, MIRROR_IP, MIRROR_SSH_KEY, SSH_TIMEOUT, DRY_RUN, SCRIPTS_ROOT
14	#
15	# ==============================================================================================
16
17	TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
18
19	_STACK_DEPLOYED=0
20	_STACK_FAILED=0
21
22	# ==============================================================================================
23	# ── Wait for a container on the remote to be healthy/running ─────────────────────────────────
24	#
25	# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined;
26	# falls back to the running state. Non-fatal after timeout — some containers take time to
27	# fully initialize but the deploy itself succeeded.
28	# ==============================================================================================
29	wait_for_container_healthy() {
30	    local name="$1" remote_ip="$2" ssh_key="$3"
31	    local max_wait=60 interval=5 elapsed=0
32
33	    [[ "$DRY_RUN" == true ]] && return 0
34
35	    log "  Waiting for $name to be ready..."
36	    while (( elapsed < max_wait )); do
37	        local status
38	        status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
39	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
40	            "h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null)
41	             r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null)
42	             echo \${h:-\$r}" 2>/dev/null)
43
44	        case "$status" in
45	            healthy|true)
46	                log "  $name ready ✅"
47	                return 0
48	                ;;
49	            *)
50	                sleep "$interval"
51	                (( elapsed += interval ))
52	                ;;
53	        esac
54	    done
55
56	    warn "  $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)"
57	    return 0
58	}
59
60	# ==============================================================================================
61	# ── Deploy a container from a local Unraid CA XML template to a remote host ──────────────────
62	#
63	# Parses Port / Path / Variable Config entries from the Unraid XML, SCPs the template and a
64	# self-contained deploy script to the remote, executes it, then cleans up both sides.
65	# Credentials are never passed as SSH command-line args — they stay in the SCPed script.
66	# ==============================================================================================
67	deploy_container_from_xml() {
68	    local xml_file="$1" remote_ip="$2" ssh_key="$3"
69	    local xml_name
70	    xml_name=$(basename "$xml_file")
71
72	    local name repo network extra privileged
73	    name=$(       awk 'match($0,/<Name>([^<]+)<\/Name>/,        a){print a[1];exit}' "$xml_file")
74	    repo=$(       awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
75	    network=$(    awk 'match($0,/<Network>([^<]+)<\/Network>/,   a){print a[1];exit}' "$xml_file")
76	    extra=$(      awk 'match($0,/<ExtraParams>([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file")
77	    privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")
78
79	    if [[ -z "$name" || -z "$repo" ]]; then
80	        warn "  Cannot parse Name/Repository from $xml_name — skipping"
81	        return 1
82	    fi
83
84	    log "Deploying $name..."
85
86	    if [[ "$DRY_RUN" == false ]]; then
87	        timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
88	            "$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
89	            warn "  SCP failed for $xml_name — skipping $name"
90	            return 1
91	        }
92	    else
93	        warn "  DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/"
94	    fi
95
96	    local tmp_script
97	    tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh)
98	    chmod 600 "$tmp_script"
99
100	    {
101	        echo "#!/bin/bash"
102	        echo "set -e"
103	        echo ""
104	        printf "docker pull %q 2>/dev/null || true\n" "$repo"
105	        printf "docker stop %q 2>/dev/null || true\n" "$name"
106	        printf "docker rm   %q 2>/dev/null || true\n" "$name"
107	        echo ""
108	        printf "docker create --name %q --restart=unless-stopped" "$name"
109	        [[ -n "$network"           ]] && printf " --network=%q"  "$network"
110	        [[ "$privileged" == "true" ]] && printf " --privileged"
111	        [[ -n "$extra"             ]] && printf " %s" "$extra"
112
113	        # Port mappings → -p host:container/proto
114	        awk '/Type="Port"/ {
115	            match($0, /Target="([^"]+)"/, t)
116	            match($0, /Mode="([^"]+)"/,   m)
117	            match($0, />([^<]+)<\/Config>/, v)
118	            if (t[1] != "" && v[1] != "") {
119	                proto = (m[1] == "udp") ? "udp" : "tcp"
120	                printf " -p %s:%s/%s", v[1], t[1], proto
121	            }
122	        }' "$xml_file"
123
124	        # Volume mappings → -v 'host:container:mode'
125	        awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ {
126	            match($0, /Target="([^"]+)"/, t)
127	            match($0, /Mode="([^"]+)"/,   m)
128	            match($0, />([^<]+)<\/Config>/, v)
129	            if (t[1] != "" && v[1] != "") {
130	                mode = (m[1] == "ro") ? "ro" : "rw"
131	                printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q
132	            }
133	        }' "$xml_file"
134
135	        # Environment variables → -e 'KEY=VALUE'  (single-quoted to protect $ and special chars)
136	        awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ {
137	            match($0, /Target="([^"]+)"/, t)
138	            match($0, />([^<]+)<\/Config>/, v)
139	            if (t[1] != "" && v[1] != "") {
140	                printf " -e %s%s=%s%s", q, t[1], v[1], q
141	            }
142	        }' "$xml_file"
143
144	        printf " %q\n" "$repo"
145	        echo ""
146	        printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name"
147	    } > "$tmp_script"
148
149	    if [[ "$DRY_RUN" == true ]]; then
150	        warn "  DRY RUN — would deploy $name on $MIRROR"
151	        rm -f "$tmp_script"
152	        return 0
153	    fi
154
155	    local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
156
157	    if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
158	        "$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
159	       timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
160	        "bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
161	        grep -q "deployed:${name}"; then
162	        log "  $name deployed ✅"
163	        rm -f "$tmp_script"
164	        return 0
165	    else
166	        warn "  $name deployment failed — check $MIRROR manually"
167	        rm -f "$tmp_script"
168	        return 1
169	    fi
170	}
171
172	# ==============================================================================================
173	# ── Deploy a stack of Unraid CA XMLs to the mirror ───────────────────────────────────────────
174	#
175	# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout.
176	# Health-checks database deps (Mariadb/Redis/Postgres) between batches so dependents
177	# (e.g. Authelia) start cleanly.
178	# ==============================================================================================
179	deploy_xml_stack() {
180	    local -n xml_array_ref="$1"
181	    _STACK_DEPLOYED=0
182	    _STACK_FAILED=0
183
184	    for xml_name in "${xml_array_ref[@]}"; do
185	        local xml_file="${TEMPLATES_DIR}/${xml_name}"
186	        if [[ ! -f "$xml_file" ]]; then
187	            warn "$xml_name not found in $TEMPLATES_DIR — skipping"
188	            (( _STACK_FAILED++ ))
189	            continue
190	        fi
191
192	        local cname
193	        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
194
195	        if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
196	            (( _STACK_DEPLOYED++ ))
197	            if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then
198	                wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY"
199	            fi
200	        else
201	            (( _STACK_FAILED++ ))
202	        fi
203	    done
204	}
205
206	# ==============================================================================================
207	# ── Remove owner-deployed containers from a remote host ──────────────────────────────────────
208	#
209	# Uses PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK (owner's conf) to derive container
210	# names from local XML templates. SSHes to remote to stop, remove, and delete appdata.
211	# Appdata paths collected via docker inspect before removal. Safety gate: only
212	# /mnt/*/appdata* paths are deleted.
213	# ==============================================================================================
214	cleanup_deployed_stack_on_remote() {
215	    local remote_ip="$1" ssh_key="$2"
216	    local -a xml_names=()
217	    [[ ${#PARTNERSHIP_AUTH_STACK[@]}     -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
218	    [[ ${#PARTNERSHIP_ARR_STACK[@]}      -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
219	    [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")
220
221	    if [[ ${#xml_names[@]} -eq 0 ]]; then
222	        log "No auth/arr/services stack arrays configured — skipping deployed stack cleanup"
223	        return 0
224	    fi
225
226	    log "Removing owner-deployed containers (auth/arr/services stacks) from $MIRROR..."
227	    for xml_name in "${xml_names[@]}"; do
228	        [[ -z "$xml_name" ]] && continue
229	        local xml_file="${TEMPLATES_DIR}/${xml_name}"
230	        if [[ ! -f "$xml_file" ]]; then
231	            warn "  $xml_name not found in local $TEMPLATES_DIR — skipping"
232	            continue
233	        fi
234
235	        local cname
236	        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
237	        [[ -z "$cname" ]] && continue
238
239	        if [[ "$DRY_RUN" == true ]]; then
240	            warn "  DRY RUN — would stop + rm $cname on $MIRROR"
241	            warn "  DRY RUN — would delete appdata for $cname on $MIRROR"
242	            continue
243	        fi
244
245	        local appdata_paths
246	        appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
247	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
248	            "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$cname' 2>/dev/null \
249	             | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
250
251	        timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
252	            -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
253	            "docker stop '$cname' >/dev/null 2>&1
254	             docker rm   '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
255	            grep -q removed && \
256	            log "  $cname removed from $MIRROR ✅" || \
257	            log "  $cname not found on $MIRROR — skipping"
258
259	        while IFS= read -r path; do
260	            [[ -z "$path" ]] && continue
261	            timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
262	                -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
263	                "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
264	                log "  Appdata removed on $MIRROR: $path ✅" || \
265	                warn "  Failed to remove appdata on $MIRROR: $path"
266	        done <<< "$appdata_paths"
267	    done
268	}
269
270	# ==============================================================================================
271	# ── Remove owner-deployed containers locally (mirror-initiated offboard) ─────────────────────
272	#
273	# SSHes to owner to read PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK, then uses the
274	# local templates-user/ copies (SCPed there during onboard) to get container names and
275	# appdata paths. Appdata collected before removal. Skips gracefully if owner unreachable.
276	# ==============================================================================================
277	cleanup_deployed_stack_locally() {
278	    local owner_ip="$1" ssh_key="$2"
279	    local -a xml_names=()
280
281	    if [[ -n "$owner_ip" ]]; then
282	        local -a auth_arr arr_arr
283	        mapfile -t auth_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
284	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
285	            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
286	             detect_hosts 2>/dev/null
287	             printf '%s\n' \"\${PARTNERSHIP_AUTH_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
288	        mapfile -t arr_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
289	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
290	            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
291	             detect_hosts 2>/dev/null
292	             printf '%s\n' \"\${PARTNERSHIP_ARR_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
293	        local -a svc_arr
294	        mapfile -t svc_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
295	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
296	            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
297	             detect_hosts 2>/dev/null
298	             printf '%s\n' \"\${PARTNERSHIP_SERVICES_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
299	        xml_names=("${auth_arr[@]}" "${arr_arr[@]}" "${svc_arr[@]}")
300	    fi
301
302	    if [[ ${#xml_names[@]} -eq 0 ]]; then
303	        log "Could not read deployed stack from owner — skipping auth/arr/services cleanup"
304	        return 0
305	    fi
306
307	    log "Removing owner-deployed containers (auth/arr/services stacks) locally..."
308	    for xml_name in "${xml_names[@]}"; do
309	        [[ -z "$xml_name" ]] && continue
310	        local xml_file="${TEMPLATES_DIR}/${xml_name}"
311	        if [[ ! -f "$xml_file" ]]; then
312	            warn "  $xml_name not found locally — skipping"
313	            continue
314	        fi
315
316	        local cname
317	        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
318	        [[ -z "$cname" ]] && continue
319
320	        if [[ "$DRY_RUN" == true ]]; then
321	            warn "  DRY RUN — would stop + rm $cname"
322	            warn "  DRY RUN — would delete appdata for $cname"
323	            continue
324	        fi
325
326	        local appdata_paths=""
327	        if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$cname" >/dev/null 2>&1; then
328	            appdata_paths=$(docker inspect \
329	                --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
330	                "$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
331	            timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
332	            _PM_TRAP_STOPPED+=("$cname")
333	            timeout "${DOCKER_TIMEOUT:-30}" docker rm   "$cname" >/dev/null 2>&1 && \
334	                log "  $cname removed ✅" || warn "  $cname rm failed"
335	        else
336	            log "  $cname not found locally — skipping"
337	        fi
338
339	        while IFS= read -r path; do
340	            [[ -z "$path" ]] && continue
341	            rm -rf "$path" && log "  Appdata removed: $path ✅" || warn "  Failed to remove: $path"
342	        done <<< "$appdata_paths"
343	    done
344	}
345
346	# ==============================================================================================
347	# ── Reconfigure a container's WebUI on the remote server ─────────────────────────────────────
348	# ==============================================================================================
349	reconfigure_webui() {
350	    local container="$1" port="$2" target_ip="$3"
351	    local ssh_key="$4" remote_ip="$5" label="${6:-remote}"
352
353	    log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..."
354
355	    if [[ "$DRY_RUN" == true ]]; then
356	        warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/"
357	        return 0
358	    fi
359
360	    local template
361	    template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
362	        -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
363	        "grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
364	         xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
365
366	    if [[ -z "$template" ]]; then
367	        warn "$container template not found on $label — WebUI needs manual reconfiguration"
368	        return 1
369	    fi
370
371	    timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
372	        -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
373	        "sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
374	        2>/dev/null && \
375	        log "$container → http://${target_ip}:${port}/ ✅" || {
376	        error "Failed to reconfigure $container WebUI on $label"
377	        return 1
378	    }
379	}
380
381	# ==============================================================================================
382	# ── Reconfigure local auth WebUIs to target IP ───────────────────────────────────────────────
383	# ==============================================================================================
384	reconfigure_local_webuis() {
385	    local target_ip="$1"
386	    log "Reconfiguring local auth WebUIs → ${target_ip}..."
387
388	    local failures=0
389	    for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
390	        [[ -z "$entry" ]] && continue
391	        local container="${entry%%|*}"
392	        local port="${entry##*|}"
393
394	        local template
395	        template=$(grep -rl '<WebUI>' "$TEMPLATES_DIR/" 2>/dev/null | \
396	            xargs grep -l "\"$container\"" 2>/dev/null | head -1)
397
398	        if [[ -z "$template" ]]; then
399	            warn "$container template not found locally"
400	            (( failures++ ))
401	            continue
402	        fi
403
404	        if [[ "$DRY_RUN" == true ]]; then
405	            warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/"
406	            continue
407	        fi
408
409	        sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
410	            "$template" 2>/dev/null && \
411	            log "$container → http://${target_ip}:${port}/ ✅" || \
412	            { error "Failed to reconfigure $container"; (( failures++ )); }
413	    done
414	    return $failures
415	}
416
417

#!/bin/bash
# ==============================================================================================
# ====================== Partnership — Unraid Container Adapter ================================
# ==============================================================================================
#
# Sourced by partnership_onboard.sh and partnership_offboard.sh via:
#   source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
#
# Provides container deploy/cleanup functions specific to the Unraid platform:
#   - Docker container deployment from Unraid CA XML templates
#
# Functions use variables from the calling script's scope (sourced, not exec'd):
#   MIRROR, MIRROR_IP, MIRROR_SSH_KEY, SSH_TIMEOUT, DRY_RUN, SCRIPTS_ROOT
#
# ==============================================================================================

TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"

_STACK_DEPLOYED=0
_STACK_FAILED=0

# ==============================================================================================
# ── Wait for a container on the remote to be healthy/running ─────────────────────────────────
#
# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined;
# falls back to the running state. Non-fatal after timeout — some containers take time to
# fully initialize but the deploy itself succeeded.
# ==============================================================================================
wait_for_container_healthy() {
    local name="$1" remote_ip="$2" ssh_key="$3"
    local max_wait=60 interval=5 elapsed=0

    [[ "$DRY_RUN" == true ]] && return 0

    log "  Waiting for $name to be ready..."
    while (( elapsed < max_wait )); do
        local status
        status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
            "h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null)
             r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null)
             echo \${h:-\$r}" 2>/dev/null)

        case "$status" in
            healthy|true)
                log "  $name ready ✅"
                return 0
                ;;
            *)
                sleep "$interval"
                (( elapsed += interval ))
                ;;
        esac
    done

    warn "  $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)"
    return 0
}

# ==============================================================================================
# ── Deploy a container from a local Unraid CA XML template to a remote host ──────────────────
#
# Parses Port / Path / Variable Config entries from the Unraid XML, SCPs the template and a
# self-contained deploy script to the remote, executes it, then cleans up both sides.
# Credentials are never passed as SSH command-line args — they stay in the SCPed script.
# ==============================================================================================
deploy_container_from_xml() {
    local xml_file="$1" remote_ip="$2" ssh_key="$3"
    local xml_name
    xml_name=$(basename "$xml_file")

    local name repo network extra privileged
    name=$(       awk 'match($0,/<Name>([^<]+)<\/Name>/,        a){print a[1];exit}' "$xml_file")
    repo=$(       awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
    network=$(    awk 'match($0,/<Network>([^<]+)<\/Network>/,   a){print a[1];exit}' "$xml_file")
    extra=$(      awk 'match($0,/<ExtraParams>([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file")
    privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")

    if [[ -z "$name" || -z "$repo" ]]; then
        warn "  Cannot parse Name/Repository from $xml_name — skipping"
        return 1
    fi

    log "Deploying $name..."

    if [[ "$DRY_RUN" == false ]]; then
        timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
            "$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
            warn "  SCP failed for $xml_name — skipping $name"
            return 1
        }
    else
        warn "  DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/"
    fi

    local tmp_script
    tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh)
    chmod 600 "$tmp_script"

    {
        echo "#!/bin/bash"
        echo "set -e"
        echo ""
        printf "docker pull %q 2>/dev/null || true\n" "$repo"
        printf "docker stop %q 2>/dev/null || true\n" "$name"
        printf "docker rm   %q 2>/dev/null || true\n" "$name"
        echo ""
        printf "docker create --name %q --restart=unless-stopped" "$name"
        [[ -n "$network"           ]] && printf " --network=%q"  "$network"
        [[ "$privileged" == "true" ]] && printf " --privileged"
        [[ -n "$extra"             ]] && printf " %s" "$extra"

        # Port mappings → -p host:container/proto
        awk '/Type="Port"/ {
            match($0, /Target="([^"]+)"/, t)
            match($0, /Mode="([^"]+)"/,   m)
            match($0, />([^<]+)<\/Config>/, v)
            if (t[1] != "" && v[1] != "") {
                proto = (m[1] == "udp") ? "udp" : "tcp"
                printf " -p %s:%s/%s", v[1], t[1], proto
            }
        }' "$xml_file"

        # Volume mappings → -v 'host:container:mode'
        awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ {
            match($0, /Target="([^"]+)"/, t)
            match($0, /Mode="([^"]+)"/,   m)
            match($0, />([^<]+)<\/Config>/, v)
            if (t[1] != "" && v[1] != "") {
                mode = (m[1] == "ro") ? "ro" : "rw"
                printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q
            }
        }' "$xml_file"

        # Environment variables → -e 'KEY=VALUE'  (single-quoted to protect $ and special chars)
        awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ {
            match($0, /Target="([^"]+)"/, t)
            match($0, />([^<]+)<\/Config>/, v)
            if (t[1] != "" && v[1] != "") {
                printf " -e %s%s=%s%s", q, t[1], v[1], q
            }
        }' "$xml_file"

        printf " %q\n" "$repo"
        echo ""
        printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name"
    } > "$tmp_script"

    if [[ "$DRY_RUN" == true ]]; then
        warn "  DRY RUN — would deploy $name on $MIRROR"
        rm -f "$tmp_script"
        return 0
    fi

    local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"

    if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
        "$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
       timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
        "bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
        grep -q "deployed:${name}"; then
        log "  $name deployed ✅"
        rm -f "$tmp_script"
        return 0
    else
        warn "  $name deployment failed — check $MIRROR manually"
        rm -f "$tmp_script"
        return 1
    fi
}

# ==============================================================================================
# ── Deploy a stack of Unraid CA XMLs to the mirror ───────────────────────────────────────────
#
# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout.
# Health-checks database deps (Mariadb/Redis/Postgres) between batches so dependents
# (e.g. Authelia) start cleanly.
# ==============================================================================================
deploy_xml_stack() {
    local -n xml_array_ref="$1"
    _STACK_DEPLOYED=0
    _STACK_FAILED=0

    for xml_name in "${xml_array_ref[@]}"; do
        local xml_file="${TEMPLATES_DIR}/${xml_name}"
        if [[ ! -f "$xml_file" ]]; then
            warn "$xml_name not found in $TEMPLATES_DIR — skipping"
            (( _STACK_FAILED++ ))
            continue
        fi

        local cname
        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")

        if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
            (( _STACK_DEPLOYED++ ))
            if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then
                wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY"
            fi
        else
            (( _STACK_FAILED++ ))
        fi
    done
}

# ==============================================================================================
# ── Remove owner-deployed containers from a remote host ──────────────────────────────────────
#
# Uses PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK (owner's conf) to derive container
# names from local XML templates. SSHes to remote to stop, remove, and delete appdata.
# Appdata paths collected via docker inspect before removal. Safety gate: only
# /mnt/*/appdata* paths are deleted.
# ==============================================================================================
cleanup_deployed_stack_on_remote() {
    local remote_ip="$1" ssh_key="$2"
    local -a xml_names=()
    [[ ${#PARTNERSHIP_AUTH_STACK[@]}     -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
    [[ ${#PARTNERSHIP_ARR_STACK[@]}      -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
    [[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")

    if [[ ${#xml_names[@]} -eq 0 ]]; then
        log "No auth/arr/services stack arrays configured — skipping deployed stack cleanup"
        return 0
    fi

    log "Removing owner-deployed containers (auth/arr/services stacks) from $MIRROR..."
    for xml_name in "${xml_names[@]}"; do
        [[ -z "$xml_name" ]] && continue
        local xml_file="${TEMPLATES_DIR}/${xml_name}"
        if [[ ! -f "$xml_file" ]]; then
            warn "  $xml_name not found in local $TEMPLATES_DIR — skipping"
            continue
        fi

        local cname
        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
        [[ -z "$cname" ]] && continue

        if [[ "$DRY_RUN" == true ]]; then
            warn "  DRY RUN — would stop + rm $cname on $MIRROR"
            warn "  DRY RUN — would delete appdata for $cname on $MIRROR"
            continue
        fi

        local appdata_paths
        appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
            "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$cname' 2>/dev/null \
             | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)

        timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
            "docker stop '$cname' >/dev/null 2>&1
             docker rm   '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
            grep -q removed && \
            log "  $cname removed from $MIRROR ✅" || \
            log "  $cname not found on $MIRROR — skipping"

        while IFS= read -r path; do
            [[ -z "$path" ]] && continue
            timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
                -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
                "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
                log "  Appdata removed on $MIRROR: $path ✅" || \
                warn "  Failed to remove appdata on $MIRROR: $path"
        done <<< "$appdata_paths"
    done
}

# ==============================================================================================
# ── Remove owner-deployed containers locally (mirror-initiated offboard) ─────────────────────
#
# SSHes to owner to read PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK, then uses the
# local templates-user/ copies (SCPed there during onboard) to get container names and
# appdata paths. Appdata collected before removal. Skips gracefully if owner unreachable.
# ==============================================================================================
cleanup_deployed_stack_locally() {
    local owner_ip="$1" ssh_key="$2"
    local -a xml_names=()

    if [[ -n "$owner_ip" ]]; then
        local -a auth_arr arr_arr
        mapfile -t auth_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
             detect_hosts 2>/dev/null
             printf '%s\n' \"\${PARTNERSHIP_AUTH_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
        mapfile -t arr_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
             detect_hosts 2>/dev/null
             printf '%s\n' \"\${PARTNERSHIP_ARR_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
        local -a svc_arr
        mapfile -t svc_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
             detect_hosts 2>/dev/null
             printf '%s\n' \"\${PARTNERSHIP_SERVICES_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
        xml_names=("${auth_arr[@]}" "${arr_arr[@]}" "${svc_arr[@]}")
    fi

    if [[ ${#xml_names[@]} -eq 0 ]]; then
        log "Could not read deployed stack from owner — skipping auth/arr/services cleanup"
        return 0
    fi

    log "Removing owner-deployed containers (auth/arr/services stacks) locally..."
    for xml_name in "${xml_names[@]}"; do
        [[ -z "$xml_name" ]] && continue
        local xml_file="${TEMPLATES_DIR}/${xml_name}"
        if [[ ! -f "$xml_file" ]]; then
            warn "  $xml_name not found locally — skipping"
            continue
        fi

        local cname
        cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
        [[ -z "$cname" ]] && continue

        if [[ "$DRY_RUN" == true ]]; then
            warn "  DRY RUN — would stop + rm $cname"
            warn "  DRY RUN — would delete appdata for $cname"
            continue
        fi

        local appdata_paths=""
        if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$cname" >/dev/null 2>&1; then
            appdata_paths=$(docker inspect \
                --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
                "$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
            timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
            _PM_TRAP_STOPPED+=("$cname")
            timeout "${DOCKER_TIMEOUT:-30}" docker rm   "$cname" >/dev/null 2>&1 && \
                log "  $cname removed ✅" || warn "  $cname rm failed"
        else
            log "  $cname not found locally — skipping"
        fi

        while IFS= read -r path; do
            [[ -z "$path" ]] && continue
            rm -rf "$path" && log "  Appdata removed: $path ✅" || warn "  Failed to remove: $path"
        done <<< "$appdata_paths"
    done
}

# ==============================================================================================
# ── Reconfigure a container's WebUI on the remote server ─────────────────────────────────────
# ==============================================================================================
reconfigure_webui() {
    local container="$1" port="$2" target_ip="$3"
    local ssh_key="$4" remote_ip="$5" label="${6:-remote}"

    log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..."

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/"
        return 0
    fi

    local template
    template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
        "grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
         xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)

    if [[ -z "$template" ]]; then
        warn "$container template not found on $label — WebUI needs manual reconfiguration"
        return 1
    fi

    timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
        "sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
        2>/dev/null && \
        log "$container → http://${target_ip}:${port}/ ✅" || {
        error "Failed to reconfigure $container WebUI on $label"
        return 1
    }
}

# ==============================================================================================
# ── Reconfigure local auth WebUIs to target IP ───────────────────────────────────────────────
# ==============================================================================================
reconfigure_local_webuis() {
    local target_ip="$1"
    log "Reconfiguring local auth WebUIs → ${target_ip}..."

    local failures=0
    for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
        [[ -z "$entry" ]] && continue
        local container="${entry%%|*}"
        local port="${entry##*|}"

        local template
        template=$(grep -rl '<WebUI>' "$TEMPLATES_DIR/" 2>/dev/null | \
            xargs grep -l "\"$container\"" 2>/dev/null | head -1)

        if [[ -z "$template" ]]; then
            warn "$container template not found locally"
            (( failures++ ))
            continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/"
            continue
        fi

        sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
            "$template" 2>/dev/null && \
            log "$container → http://${target_ip}:${port}/ ✅" || \
            { error "Failed to reconfigure $container"; (( failures++ )); }
    done
    return $failures
}