Auto-detect remote GPU and rewrite XML config on deploy — NVIDIA/Intel/AMD/none
This commit is contained in:
Gmer4Lfe
@@ -18,6 +18,112 @@ TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
|
||||
|
||||
_STACK_DEPLOYED=0
|
||||
_STACK_FAILED=0
|
||||
_REMOTE_GPU_TYPE="" # cached after first detection
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Detect GPU type on a remote host ─────────────────────────────────────────────────────────
|
||||
#
|
||||
# Returns one of: nvidia | intel | amd | dri | none
|
||||
# nvidia — /dev/nvidia0 present (NVIDIA driver loaded)
|
||||
# intel — /dev/dri present, vendor 0x8086
|
||||
# amd — /dev/dri present, vendor 0x1002 (also exposes /dev/kfd)
|
||||
# dri — /dev/dri present but vendor unreadable
|
||||
# none — no GPU device found
|
||||
#
|
||||
# Result is cached in _REMOTE_GPU_TYPE for the session — SSH'd once per onboard run.
|
||||
# ==============================================================================================
|
||||
detect_remote_gpu() {
|
||||
local remote_ip="$1" ssh_key="$2"
|
||||
|
||||
if [[ -n "$_REMOTE_GPU_TYPE" ]]; then
|
||||
echo "$_REMOTE_GPU_TYPE"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local result
|
||||
result=$(timeout 10 ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout=10 -o BatchMode=yes root@"$remote_ip" '
|
||||
if [ -c /dev/nvidia0 ]; then
|
||||
echo nvidia
|
||||
elif [ -d /dev/dri ]; then
|
||||
vendor=""
|
||||
for f in /sys/class/drm/card*/device/vendor; do
|
||||
[ -f "$f" ] && { vendor=$(cat "$f" 2>/dev/null); break; }
|
||||
done
|
||||
case "$vendor" in
|
||||
0x8086) echo intel ;;
|
||||
0x1002) echo amd ;;
|
||||
*) echo dri ;;
|
||||
esac
|
||||
else
|
||||
echo none
|
||||
fi
|
||||
' 2>/dev/null)
|
||||
|
||||
_REMOTE_GPU_TYPE="${result:-none}"
|
||||
echo "$_REMOTE_GPU_TYPE"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Rewrite GPU config in an XML for a target GPU type ───────────────────────────────────────
|
||||
#
|
||||
# Called when deploying to a remote whose GPU differs from the owner's. Takes the owner's
|
||||
# XML (NVIDIA-configured) and rewrites it for the remote's hardware without modifying the
|
||||
# original on disk.
|
||||
#
|
||||
# Returns the path to a temp file — caller must clean it up.
|
||||
# Returns the original path unchanged if the XML has no NVIDIA markers (not GPU-aware).
|
||||
#
|
||||
# Transforms applied:
|
||||
# nvidia → nvidia: replace UUID with "all" so any NVIDIA GPU is accepted
|
||||
# nvidia → intel/amd/dri: strip --runtime=nvidia + NVIDIA_VISIBLE_DEVICES,
|
||||
# inject /dev/dri Device Config (+ /dev/kfd for AMD)
|
||||
# nvidia → none: strip --runtime=nvidia + NVIDIA_VISIBLE_DEVICES, no device added
|
||||
# ==============================================================================================
|
||||
transform_xml_for_gpu() {
|
||||
local src_xml="$1" gpu_type="$2"
|
||||
|
||||
# Only transform GPU-aware XMLs (containers with NVIDIA config)
|
||||
if ! grep -qE 'runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
||||
echo "$src_xml"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local tmp_xml
|
||||
tmp_xml=$(mktemp /tmp/vv_xml_gpu_XXXXXX.xml)
|
||||
|
||||
case "$gpu_type" in
|
||||
nvidia)
|
||||
# Same vendor — normalise UUID to "all" so any NVIDIA card is accepted
|
||||
sed 's/\(Target="NVIDIA_VISIBLE_DEVICES"[^>]*>\)[^<]*/\1all/' "$src_xml" > "$tmp_xml"
|
||||
;;
|
||||
intel|dri)
|
||||
# Strip NVIDIA params, add /dev/dri device
|
||||
sed \
|
||||
-e 's/--runtime=nvidia[[:space:]]*//' \
|
||||
-e '/Target="NVIDIA_VISIBLE_DEVICES"/d' \
|
||||
"$src_xml" > "$tmp_xml"
|
||||
sed -i 's|</Container>| <Config Name="GPU" Target="/dev/dri" Default="/dev/dri" Mode="rwm" Description="" Type="Device" Display="always" Required="false" Mask="false">/dev/dri</Config>\n</Container>|' "$tmp_xml"
|
||||
;;
|
||||
amd)
|
||||
# AMD needs /dev/dri for VA-API and /dev/kfd for ROCm/OpenCL
|
||||
sed \
|
||||
-e 's/--runtime=nvidia[[:space:]]*//' \
|
||||
-e '/Target="NVIDIA_VISIBLE_DEVICES"/d' \
|
||||
"$src_xml" > "$tmp_xml"
|
||||
sed -i 's|</Container>| <Config Name="GPU" Target="/dev/dri" Default="/dev/dri" Mode="rwm" Description="" Type="Device" Display="always" Required="false" Mask="false">/dev/dri</Config>\n <Config Name="GPU KFD" Target="/dev/kfd" Default="/dev/kfd" Mode="rwm" Description="" Type="Device" Display="always" Required="false" Mask="false">/dev/kfd</Config>\n</Container>|' "$tmp_xml"
|
||||
;;
|
||||
none)
|
||||
# No GPU — strip all GPU params, no device added
|
||||
sed \
|
||||
-e 's/--runtime=nvidia[[:space:]]*//' \
|
||||
-e '/Target="NVIDIA_VISIBLE_DEVICES"/d' \
|
||||
"$src_xml" > "$tmp_xml"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "$tmp_xml"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Wait for a container on the remote to be healthy/running ─────────────────────────────────
|
||||
@@ -69,6 +175,14 @@ deploy_container_from_xml() {
|
||||
local xml_name
|
||||
xml_name=$(basename "$xml_file")
|
||||
|
||||
# GPU transform — rewrite GPU params for the remote's hardware before parsing or SCP.
|
||||
# detect_remote_gpu is cached after the first SSH call.
|
||||
local _gpu_type _transformed_xml _gpu_tmp=""
|
||||
_gpu_type=$(detect_remote_gpu "$remote_ip" "$ssh_key")
|
||||
_transformed_xml=$(transform_xml_for_gpu "$xml_file" "$_gpu_type")
|
||||
[[ "$_transformed_xml" != "$xml_file" ]] && _gpu_tmp="$_transformed_xml"
|
||||
xml_file="$_transformed_xml"
|
||||
|
||||
local name repo network extra privileged
|
||||
name=$( awk 'match($0,/<Name>([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file")
|
||||
repo=$( awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
|
||||
@@ -77,15 +191,18 @@ deploy_container_from_xml() {
|
||||
privileged=$( awk 'match($0,/<Privileged>([^<]+)<\/Privileged>/,a){print a[1];exit}' "$xml_file")
|
||||
|
||||
if [[ -z "$name" || -z "$repo" ]]; then
|
||||
rm -f "$_gpu_tmp"
|
||||
warn " Cannot parse Name/Repository from $xml_name — skipping"
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ -n "$_gpu_tmp" ]] && log " GPU: ${_gpu_type} (rewritten from owner NVIDIA config)"
|
||||
log "Deploying $name..."
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
"$xml_file" "root@${remote_ip}:${TEMPLATES_DIR}/${xml_name}" 2>/dev/null || {
|
||||
rm -f "$_gpu_tmp"
|
||||
warn " SCP failed for $xml_name — skipping $name"
|
||||
return 1
|
||||
}
|
||||
@@ -148,7 +265,7 @@ deploy_container_from_xml() {
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would deploy $name on $MIRROR"
|
||||
rm -f "$tmp_script"
|
||||
rm -f "$tmp_script" "$_gpu_tmp"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -160,11 +277,11 @@ deploy_container_from_xml() {
|
||||
"bash '$remote_script' 2>&1; rc=\$?; rm -f '$remote_script'; exit \$rc" 2>/dev/null | \
|
||||
grep -q "deployed:${name}"; then
|
||||
echo " $name deployed ✅"
|
||||
rm -f "$tmp_script"
|
||||
rm -f "$tmp_script" "$_gpu_tmp"
|
||||
return 0
|
||||
else
|
||||
warn " $name deployment failed — check $MIRROR manually"
|
||||
rm -f "$tmp_script"
|
||||
rm -f "$tmp_script" "$_gpu_tmp"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user