Gmer4Lfe
c377ddfcca
Finishes the pass: every script now documents its safeguards, and the deliberate absences in the sourced libraries are recorded so they are not "corrected" later.
638 lines
29 KiB
Bash
Executable File
638 lines
29 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ====================== Partnership — Unraid Container Adapter ================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Platform adapter providing container deploy/cleanup functions for the Unraid
|
|
# partnership system. Sourced (not executed) by partnership_onboard.sh and
|
|
# partnership_offboard.sh via:
|
|
# source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Functions operate on variables from the calling script's scope:
|
|
# MIRROR, MIRROR_IP, MIRROR_SSH_KEY, SSH_TIMEOUT, DRY_RUN, SCRIPTS_ROOT
|
|
#
|
|
# Capabilities:
|
|
# - Docker container deployment from Unraid CA XML templates
|
|
# - Remote GPU type detection (cached per session — one SSH call per onboard)
|
|
# - Deployed stack tracking via _STACK_DEPLOYED / _STACK_FAILED counters
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Platform Adapter, Same Contract as adapter.sh
|
|
# Container deployment is Unraid-specific — CA XML templates, dockerMan paths, the
|
|
# templates-user directory. Confining it here means the partnership scripts contain no
|
|
# Unraid knowledge and a second platform is a new Partnership/containers.sh, not edits
|
|
# scattered through onboard and offboard.
|
|
#
|
|
# Deploy From the Template, Not a Copy
|
|
# Containers are created from the CA XML the operator already maintains, so a partnership
|
|
# deployment produces the same container the Unraid UI would. Hand-built docker run lines
|
|
# would drift from the template the moment anyone edited it in the UI.
|
|
#
|
|
# GPU Detection Cached Per Session
|
|
# The remote GPU type is probed once and reused. Onboarding deploys several containers and
|
|
# each would otherwise repeat the same SSH round-trip to learn an answer that cannot change
|
|
# mid-run.
|
|
#
|
|
# Count Outcomes, Do Not Abort
|
|
# Failures increment _STACK_FAILED rather than exiting. A stack deployment that fails on
|
|
# one container should report which one and continue — the caller owns whether a partial
|
|
# stack is acceptable, and it is the only side with the context to decide.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# No Root, No Lock, No detect_hosts — Deliberate
|
|
# Sourced by partnership_onboard.sh and partnership_offboard.sh, both of which already
|
|
# enforce root and hold their own strict locks. Re-checking here would be redundant, and
|
|
# taking a lock would deadlock against the caller's. Do not add them.
|
|
#
|
|
# Caller Scope Is the Contract
|
|
# Functions read MIRROR, MIRROR_IP, MIRROR_SSH_KEY, SSH_TIMEOUT, DRY_RUN and SCRIPTS_ROOT
|
|
# from the caller. That coupling is deliberate — it keeps one definition of who the mirror
|
|
# is — but it means these functions are only valid inside the partnership scripts and
|
|
# cannot be sourced standalone.
|
|
#
|
|
# DRY_RUN Honoured Throughout
|
|
# Every deploy and cleanup path checks the caller's DRY_RUN, so a dry-run onboard makes no
|
|
# remote container changes.
|
|
#
|
|
# SSH Timeouts on Every Remote Call
|
|
# All remote operations use the caller's SSH_TIMEOUT — an unreachable mirror cannot hang
|
|
# an onboard partway through a stack deployment.
|
|
#
|
|
# Template Existence Checked
|
|
# A missing CA XML is counted as a failure for that container rather than producing a
|
|
# container built from nothing.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# No config vars of its own. Inputs come from the calling script's scope (see above).
|
|
#
|
|
# Platform paths it owns:
|
|
#
|
|
# /boot/config/plugins/dockerMan/templates-user
|
|
# Unraid CA template directory. Source of every container definition deployed here.
|
|
#
|
|
# The container lists themselves live in host*.conf as HOST*_PARTNERSHIP_AUTH_STACK and
|
|
# HOST*_PARTNERSHIP_ARR_STACK — read by the partnership scripts, passed in as arguments.
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# None — sourced, never executed:
|
|
#
|
|
# source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
|
#
|
|
# No argument parsing and no flags. Dry-run behaviour comes from the caller's DRY_RUN.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
|
|
|
|
_STACK_DEPLOYED=0
|
|
_STACK_FAILED=0
|
|
_REMOTE_GPU_TYPE="" # cached after first detection
|
|
|
|
# ==============================================================================================
|
|
# ── Detect GPU type on a remote host ─────────────────────────────────────────────────────────
|
|
#
|
|
# Returns one of: nvidia | intel | amd | dri | none
|
|
# nvidia — /dev/nvidia0 present (NVIDIA driver loaded)
|
|
# intel — /dev/dri present, vendor 0x8086
|
|
# amd — /dev/dri present, vendor 0x1002 (also exposes /dev/kfd)
|
|
# dri — /dev/dri present but vendor unreadable
|
|
# none — no GPU device found
|
|
#
|
|
# Result is cached in _REMOTE_GPU_TYPE for the session — SSH'd once per onboard run.
|
|
# ==============================================================================================
|
|
detect_remote_gpu() {
|
|
local remote_ip="$1" ssh_key="$2"
|
|
|
|
if [[ -n "$_REMOTE_GPU_TYPE" ]]; then
|
|
echo "$_REMOTE_GPU_TYPE"
|
|
return 0
|
|
fi
|
|
|
|
local result
|
|
result=$(timeout 10 ssh -i "$ssh_key" \
|
|
-o ConnectTimeout=10 -o BatchMode=yes root@"$remote_ip" '
|
|
if [ -c /dev/nvidia0 ]; then
|
|
echo nvidia
|
|
elif [ -d /dev/dri ]; then
|
|
vendor=""
|
|
for f in /sys/class/drm/card*/device/vendor; do
|
|
[ -f "$f" ] && { vendor=$(cat "$f" 2>/dev/null); break; }
|
|
done
|
|
case "$vendor" in
|
|
0x8086) echo intel ;;
|
|
0x1002) echo amd ;;
|
|
*) echo dri ;;
|
|
esac
|
|
else
|
|
echo none
|
|
fi
|
|
' 2>/dev/null)
|
|
|
|
_REMOTE_GPU_TYPE="${result:-none}"
|
|
echo "$_REMOTE_GPU_TYPE"
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Rewrite GPU config in an XML for a target GPU type ───────────────────────────────────────
|
|
#
|
|
# Called when deploying to a remote whose GPU differs from the owner's. Takes the owner's
|
|
# XML (NVIDIA-configured) and rewrites it for the remote's hardware without modifying the
|
|
# original on disk.
|
|
#
|
|
# Returns the path to a temp file — caller must clean it up.
|
|
# Returns the original path unchanged if the XML has no NVIDIA markers (not GPU-aware).
|
|
#
|
|
# Handles both GPU config styles:
|
|
# New: --gpus "device=GPU-UUID" in ExtraParams (current approach)
|
|
# Old: --runtime=nvidia in ExtraParams + NVIDIA_VISIBLE_DEVICES Variable Config
|
|
#
|
|
# Transforms applied:
|
|
# nvidia → nvidia: normalise device UUID to "all" (both styles)
|
|
# nvidia → intel/dri: strip NVIDIA params, inject /dev/dri Device Config
|
|
# nvidia → amd: strip NVIDIA params, inject /dev/dri + /dev/kfd Device Configs
|
|
# nvidia → none: strip NVIDIA params, no device added
|
|
# ==============================================================================================
|
|
transform_xml_for_gpu() {
|
|
local src_xml="$1" gpu_type="$2"
|
|
|
|
# Detect GPU-aware XMLs — new style (--gpus "device=) or old style (--runtime=nvidia)
|
|
if ! grep -qE '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
|
echo "$src_xml"
|
|
return 0
|
|
fi
|
|
|
|
local tmp_xml
|
|
tmp_xml=$(mktemp /tmp/vv_xml_gpu_XXXXXX.xml)
|
|
|
|
case "$gpu_type" in
|
|
nvidia)
|
|
# Normalise to --gpus all (new style) or NVIDIA_VISIBLE_DEVICES=all (old style)
|
|
sed \
|
|
-e 's/--gpus[[:space:]]*"device=[^"]*"/--gpus all/g' \
|
|
-e 's/\(Target="NVIDIA_VISIBLE_DEVICES"[^>]*>\)[^<]*/\1all/' \
|
|
"$src_xml" > "$tmp_xml"
|
|
;;
|
|
intel|dri)
|
|
# Strip NVIDIA params, add /dev/dri device
|
|
sed \
|
|
-e 's/--gpus[[:space:]]*"device=[^"]*"[[:space:]]*//' \
|
|
-e 's/--runtime=nvidia[[:space:]]*//' \
|
|
-e '/Target="NVIDIA_VISIBLE_DEVICES"/d' \
|
|
"$src_xml" > "$tmp_xml"
|
|
sed -i 's|</Container>| <Config Name="GPU" Target="/dev/dri" Default="/dev/dri" Mode="rwm" Description="" Type="Device" Display="always" Required="false" Mask="false">/dev/dri</Config>\n</Container>|' "$tmp_xml"
|
|
;;
|
|
amd)
|
|
# AMD needs /dev/dri for VA-API and /dev/kfd for ROCm/OpenCL
|
|
sed \
|
|
-e 's/--gpus[[:space:]]*"device=[^"]*"[[:space:]]*//' \
|
|
-e 's/--runtime=nvidia[[:space:]]*//' \
|
|
-e '/Target="NVIDIA_VISIBLE_DEVICES"/d' \
|
|
"$src_xml" > "$tmp_xml"
|
|
sed -i 's|</Container>| <Config Name="GPU" Target="/dev/dri" Default="/dev/dri" Mode="rwm" Description="" Type="Device" Display="always" Required="false" Mask="false">/dev/dri</Config>\n <Config Name="GPU KFD" Target="/dev/kfd" Default="/dev/kfd" Mode="rwm" Description="" Type="Device" Display="always" Required="false" Mask="false">/dev/kfd</Config>\n</Container>|' "$tmp_xml"
|
|
;;
|
|
none)
|
|
# No GPU — strip all GPU params, no device added
|
|
sed \
|
|
-e 's/--gpus[[:space:]]*"device=[^"]*"[[:space:]]*//' \
|
|
-e 's/--runtime=nvidia[[:space:]]*//' \
|
|
-e '/Target="NVIDIA_VISIBLE_DEVICES"/d' \
|
|
"$src_xml" > "$tmp_xml"
|
|
;;
|
|
esac
|
|
|
|
echo "$tmp_xml"
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Wait for a container on the remote to be healthy/running ─────────────────────────────────
|
|
#
|
|
# Polls docker inspect on the remote. Prefers the health status if a healthcheck is defined;
|
|
# falls back to the running state. Non-fatal after timeout — some containers take time to
|
|
# fully initialize but the deploy itself succeeded.
|
|
# ==============================================================================================
|
|
wait_for_container_healthy() {
|
|
local name="$1" remote_ip="$2" ssh_key="$3"
|
|
local max_wait=60 interval=5 elapsed=0
|
|
|
|
[[ "$DRY_RUN" == true ]] && return 0
|
|
|
|
log " Waiting for $name to be ready..."
|
|
while (( elapsed < max_wait )); do
|
|
local status
|
|
status=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"h=\$(docker inspect --format '{{.State.Health.Status}}' '$name' 2>/dev/null)
|
|
r=\$(docker inspect --format '{{.State.Running}}' '$name' 2>/dev/null)
|
|
echo \${h:-\$r}" 2>/dev/null)
|
|
|
|
case "$status" in
|
|
healthy|true)
|
|
echo " $name ready ✅"
|
|
return 0
|
|
;;
|
|
*)
|
|
sleep "$interval"
|
|
(( elapsed += interval ))
|
|
;;
|
|
esac
|
|
done
|
|
|
|
warn " $name not confirmed healthy after ${max_wait}s — continuing (may affect dependents)"
|
|
return 0
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Deploy a container from a local Unraid CA XML template to a remote host ──────────────────
|
|
#
|
|
# Parses Port / Path / Variable Config entries from the Unraid XML, SCPs the template and a
|
|
# self-contained deploy script to the remote, executes it, then cleans up both sides.
|
|
# Credentials are never passed as SSH command-line args — they stay in the SCPed script.
|
|
# ==============================================================================================
|
|
deploy_container_from_xml() {
|
|
local xml_file="$1" remote_ip="$2" ssh_key="$3"
|
|
local xml_name
|
|
xml_name=$(basename "$xml_file")
|
|
|
|
# GPU transform — rewrite GPU params for the remote's hardware before parsing or SCP.
|
|
# detect_remote_gpu is cached after the first SSH call.
|
|
local _gpu_type _transformed_xml _gpu_tmp=""
|
|
_gpu_type=$(detect_remote_gpu "$remote_ip" "$ssh_key")
|
|
_transformed_xml=$(transform_xml_for_gpu "$xml_file" "$_gpu_type")
|
|
[[ "$_transformed_xml" != "$xml_file" ]] && _gpu_tmp="$_transformed_xml"
|
|
xml_file="$_transformed_xml"
|
|
|
|
local name repo network extra privileged
|
|
name=$( awk 'match($0,/<Name>([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file")
|
|
repo=$( awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
|
|
network=$( awk 'match($0,/<Network>([^<]+)<\/Network>/, a){print a[1];exit}' "$xml_file")
|
|
extra=$( awk 'match($0,/<ExtraParams>([^<]*)<\/ExtraParams>/,a){print a[1];exit}' "$xml_file")
|
|
privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")
|
|
|
|
if [[ -z "$name" || -z "$repo" ]]; then
|
|
rm -f "$_gpu_tmp"
|
|
warn " Cannot parse Name/Repository from $xml_name — skipping"
|
|
return 1
|
|
fi
|
|
|
|
[[ -n "$_gpu_tmp" ]] && log " GPU: ${_gpu_type} (rewritten from owner NVIDIA config)"
|
|
log "Deploying $name..."
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
|
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
|
|
rm -f "$_gpu_tmp"
|
|
warn " SCP failed for $xml_name — skipping $name"
|
|
return 1
|
|
}
|
|
else
|
|
warn " DRY RUN — would SCP $xml_name → $MIRROR:${TEMPLATES_DIR}/"
|
|
fi
|
|
|
|
local tmp_script
|
|
tmp_script=$(mktemp /tmp/deploy_XXXXXX.sh)
|
|
chmod 600 "$tmp_script"
|
|
|
|
{
|
|
echo "#!/bin/bash"
|
|
echo "set -e"
|
|
echo ""
|
|
printf "docker pull %q 2>/dev/null || true\n" "$repo"
|
|
printf "docker stop %q 2>/dev/null || true\n" "$name"
|
|
printf "docker rm %q 2>/dev/null || true\n" "$name"
|
|
echo ""
|
|
printf "docker create --name %q --restart=unless-stopped" "$name"
|
|
[[ -n "$network" ]] && printf " --network=%q" "$network"
|
|
[[ "$privileged" == "true" ]] && printf " --privileged"
|
|
[[ -n "$extra" ]] && printf " %s" "$extra"
|
|
|
|
# Port mappings → -p host:container/proto
|
|
awk '/Type="Port"/ {
|
|
match($0, /Target="([^"]+)"/, t)
|
|
match($0, /Mode="([^"]+)"/, m)
|
|
match($0, />([^<]+)<\/Config>/, v)
|
|
if (t[1] != "" && v[1] != "") {
|
|
proto = (m[1] == "udp") ? "udp" : "tcp"
|
|
printf " -p %s:%s/%s", v[1], t[1], proto
|
|
}
|
|
}' "$xml_file"
|
|
|
|
# Volume mappings → -v 'host:container:mode'
|
|
awk 'BEGIN{q=sprintf("%c",39)} /Type="Path"/ {
|
|
match($0, /Target="([^"]+)"/, t)
|
|
match($0, /Mode="([^"]+)"/, m)
|
|
match($0, />([^<]+)<\/Config>/, v)
|
|
if (t[1] != "" && v[1] != "") {
|
|
mode = (m[1] == "ro") ? "ro" : "rw"
|
|
printf " -v %s%s:%s:%s%s", q, v[1], t[1], mode, q
|
|
}
|
|
}' "$xml_file"
|
|
|
|
# Environment variables → -e 'KEY=VALUE' (single-quoted to protect $ and special chars)
|
|
awk 'BEGIN{q=sprintf("%c",39)} /Type="Variable"/ {
|
|
match($0, /Target="([^"]+)"/, t)
|
|
match($0, />([^<]+)<\/Config>/, v)
|
|
if (t[1] != "" && v[1] != "") {
|
|
printf " -e %s%s=%s%s", q, t[1], v[1], q
|
|
}
|
|
}' "$xml_file"
|
|
|
|
printf " %q\n" "$repo"
|
|
echo ""
|
|
printf "docker start %q && echo 'deployed:%s'\n" "$name" "$name"
|
|
} > "$tmp_script"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would deploy $name on $MIRROR"
|
|
rm -f "$tmp_script" "$_gpu_tmp"
|
|
return 0
|
|
fi
|
|
|
|
local remote_script="/tmp/deploy_${name//[^a-zA-Z0-9_]/_}.sh"
|
|
|
|
if timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
|
"$tmp_script" "root@${remote_ip}:${remote_script}" 2>/dev/null && \
|
|
timeout 120 ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
|
|
grep -q "deployed:${name}"; then
|
|
echo " $name deployed ✅"
|
|
rm -f "$tmp_script" "$_gpu_tmp"
|
|
return 0
|
|
else
|
|
warn " $name deployment failed — check $MIRROR manually"
|
|
rm -f "$tmp_script" "$_gpu_tmp"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Deploy a stack of Unraid CA XMLs to the mirror ───────────────────────────────────────────
|
|
#
|
|
# Sets globals _STACK_DEPLOYED and _STACK_FAILED rather than printing to stdout.
|
|
# Health-checks database deps (Mariadb/Redis/Postgres) between batches so dependents
|
|
# (e.g. Authelia) start cleanly.
|
|
# ==============================================================================================
|
|
deploy_xml_stack() {
|
|
local -n xml_array_ref="$1"
|
|
_STACK_DEPLOYED=0
|
|
_STACK_FAILED=0
|
|
|
|
for xml_name in "${xml_array_ref[@]}"; do
|
|
local xml_file="${TEMPLATES_DIR}/${xml_name}"
|
|
if [[ ! -f "$xml_file" ]]; then
|
|
warn "$xml_name not found in $TEMPLATES_DIR — skipping"
|
|
(( _STACK_FAILED++ ))
|
|
continue
|
|
fi
|
|
|
|
local cname
|
|
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
|
|
|
if deploy_container_from_xml "$xml_file" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
(( _STACK_DEPLOYED++ ))
|
|
if [[ -n "$cname" ]] && echo "$cname" | grep -qiE 'mariadb|redis|postgres|mysql'; then
|
|
wait_for_container_healthy "$cname" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
|
fi
|
|
else
|
|
(( _STACK_FAILED++ ))
|
|
fi
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Remove owner-deployed containers from a remote host ──────────────────────────────────────
|
|
#
|
|
# Uses PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK (owner's conf) to derive container
|
|
# names from local XML templates. SSHes to remote to stop, remove, and delete appdata.
|
|
# Appdata paths collected via docker inspect before removal. Safety gate: only
|
|
# /mnt/*/appdata* paths are deleted.
|
|
# ==============================================================================================
|
|
cleanup_deployed_stack_on_remote() {
|
|
local remote_ip="$1" ssh_key="$2"
|
|
local -a xml_names=()
|
|
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
|
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
|
[[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")
|
|
|
|
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
|
log "No auth/arr/services stack arrays configured — skipping deployed stack cleanup"
|
|
return 0
|
|
fi
|
|
|
|
log "Removing owner-deployed containers (auth/arr/services stacks) from $MIRROR..."
|
|
for xml_name in "${xml_names[@]}"; do
|
|
[[ -z "$xml_name" ]] && continue
|
|
local xml_file="${TEMPLATES_DIR}/${xml_name}"
|
|
if [[ ! -f "$xml_file" ]]; then
|
|
warn " $xml_name not found in local $TEMPLATES_DIR — skipping"
|
|
continue
|
|
fi
|
|
|
|
local cname
|
|
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
|
[[ -z "$cname" ]] && continue
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would stop + rm $cname on $MIRROR"
|
|
warn " DRY RUN — would delete appdata for $cname on $MIRROR"
|
|
continue
|
|
fi
|
|
|
|
local appdata_paths
|
|
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$cname' 2>/dev/null \
|
|
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
|
|
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"docker stop '$cname' >/dev/null 2>&1
|
|
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
|
grep -q removed && \
|
|
echo " $cname removed from $MIRROR ✅" || \
|
|
log " $cname not found on $MIRROR — skipping"
|
|
|
|
while IFS= read -r path; do
|
|
[[ -z "$path" ]] && continue
|
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
|
echo " Appdata removed on $MIRROR: $path ✅" || \
|
|
warn " Failed to remove appdata on $MIRROR: $path"
|
|
done <<< "$appdata_paths"
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Remove owner-deployed containers locally (mirror-initiated offboard) ─────────────────────
|
|
#
|
|
# SSHes to owner to read PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK, then uses the
|
|
# local templates-user/ copies (SCPed there during onboard) to get container names and
|
|
# appdata paths. Appdata collected before removal. Skips gracefully if owner unreachable.
|
|
# ==============================================================================================
|
|
cleanup_deployed_stack_locally() {
|
|
local owner_ip="$1" ssh_key="$2"
|
|
local -a xml_names=()
|
|
|
|
if [[ -n "$owner_ip" ]]; then
|
|
local -a auth_arr arr_arr
|
|
mapfile -t auth_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
|
|
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
|
detect_hosts 2>/dev/null
|
|
printf '%s\n' \"\${PARTNERSHIP_AUTH_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
|
|
mapfile -t arr_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
|
|
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
|
detect_hosts 2>/dev/null
|
|
printf '%s\n' \"\${PARTNERSHIP_ARR_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
|
|
local -a svc_arr
|
|
mapfile -t svc_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
|
|
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
|
detect_hosts 2>/dev/null
|
|
printf '%s\n' \"\${PARTNERSHIP_SERVICES_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
|
|
xml_names=("${auth_arr[@]}" "${arr_arr[@]}" "${svc_arr[@]}")
|
|
fi
|
|
|
|
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
|
log "Could not read deployed stack from owner — skipping auth/arr/services cleanup"
|
|
return 0
|
|
fi
|
|
|
|
local _local_short
|
|
_local_short=$(derive_short_name "${LOCAL_SERVER_NAME:-}")
|
|
|
|
log "Removing owner-deployed containers (auth/arr/services stacks) locally..."
|
|
for xml_name in "${xml_names[@]}"; do
|
|
[[ -z "$xml_name" ]] && continue
|
|
local xml_file="${TEMPLATES_DIR}/${xml_name}"
|
|
if [[ ! -f "$xml_file" ]]; then
|
|
warn " $xml_name not found locally — skipping"
|
|
continue
|
|
fi
|
|
|
|
local cname
|
|
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
|
[[ -z "$cname" ]] && continue
|
|
|
|
if [[ -n "$_local_short" && "$cname" == *"$_local_short"* ]]; then
|
|
warn " $cname contains local server identity ($_local_short) — skipping to protect local containers"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn " DRY RUN — would stop + rm $cname"
|
|
warn " DRY RUN — would delete appdata for $cname"
|
|
continue
|
|
fi
|
|
|
|
local appdata_paths=""
|
|
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$cname" >/dev/null 2>&1; then
|
|
appdata_paths=$(docker inspect \
|
|
--format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
|
|
"$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
|
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
|
|
_PM_TRAP_STOPPED+=("$cname")
|
|
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1 && \
|
|
echo " $cname removed ✅" || warn " $cname rm failed"
|
|
else
|
|
log " $cname not found locally — skipping"
|
|
fi
|
|
|
|
while IFS= read -r path; do
|
|
[[ -z "$path" ]] && continue
|
|
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
|
done <<< "$appdata_paths"
|
|
done
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Reconfigure a container's WebUI on the remote server ─────────────────────────────────────
|
|
# ==============================================================================================
|
|
reconfigure_webui() {
|
|
local container="$1" port="$2" target_ip="$3"
|
|
local ssh_key="$4" remote_ip="$5" label="${6:-remote}"
|
|
|
|
log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..."
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/"
|
|
return 0
|
|
fi
|
|
|
|
local template
|
|
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
|
|
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
|
|
|
if [[ -z "$template" ]]; then
|
|
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
|
return 1
|
|
fi
|
|
|
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
|
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
|
|
2>/dev/null && \
|
|
echo "$container → http://${target_ip}:${port}/ ✅" || {
|
|
error "Failed to reconfigure $container WebUI on $label"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ── Reconfigure local auth WebUIs to target IP ───────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
reconfigure_local_webuis() {
|
|
local target_ip="$1"
|
|
log "Reconfiguring local auth WebUIs → ${target_ip}..."
|
|
|
|
local failures=0
|
|
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
local container="${entry%%|*}"
|
|
local port="${entry##*|}"
|
|
|
|
local template
|
|
template=$(grep -rl '<WebUI>' "$TEMPLATES_DIR/" 2>/dev/null | \
|
|
xargs grep -l "\"$container\"" 2>/dev/null | head -1)
|
|
|
|
if [[ -z "$template" ]]; then
|
|
warn "$container template not found locally"
|
|
(( failures++ ))
|
|
continue
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/"
|
|
continue
|
|
fi
|
|
|
|
sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
|
|
"$template" 2>/dev/null && \
|
|
echo "$container → http://${target_ip}:${port}/ ✅" || \
|
|
{ error "Failed to reconfigure $container"; (( failures++ )); }
|
|
done
|
|
return $failures
|
|
}
|
|
|