305 lines
13 KiB
Bash
Executable File
305 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================== PCIe AER Quiet ================================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Removes listed PCIe devices from the bus at array start so dead hardware stops
|
|
# flooding syslog with correctable AER errors. Run once via ARRAY_START_SCRIPTS.
|
|
# Idempotent — silent when the devices are already absent.
|
|
#
|
|
# Some onboard controllers throw endless correctable PCIe errors. Correctable
|
|
# means the link recovered and nothing was lost, but the kernel logs every one,
|
|
# and on HOST1 that was ~180 lines/day drowning the syslog. Masking the error in
|
|
# the device's AER registers hides it; `pci=noaer` silences the whole machine
|
|
# and costs uncorrectable reporting on every device. Removing the device ends it
|
|
# at the source and leaves AER fully intact everywhere else.
|
|
#
|
|
# This only makes sense for hardware nothing is using. The guards below refuse
|
|
# anything else — see OPERATIONAL SAFEGUARDS.
|
|
#
|
|
# A reboot re-enumerates the devices, which is why this reapplies every array
|
|
# start rather than being a one-time step. It is also the undo: clear the conf
|
|
# list and reboot.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Refuse Anything In Use
|
|
# A PCI address is one typo away from the HBA holding the array. The script
|
|
# removes a device only when the kernel shows it bound to vfio-pci or to no
|
|
# driver at all. A device with a real driver is skipped loudly, never removed.
|
|
#
|
|
# Address Shape Validated Before Any Write
|
|
# The address goes into a sysfs path. It is matched against the full
|
|
# DDDD:BB:DD.F form before being used to build one.
|
|
#
|
|
# Idempotent Presence Check
|
|
# A device already gone is not an error — it is the desired state. Silent skip.
|
|
#
|
|
# Apply Sequence (per device)
|
|
# 1. Validate the address shape
|
|
# 2. Absent from sysfs → already done, skip silently
|
|
# 3. Bound to a driver other than vfio-pci → refuse, warn, continue
|
|
# 4. Claimed by a running VM → refuse, warn, continue
|
|
# 5. Write 1 to the device's sysfs remove node
|
|
# 6. Verify it left the bus
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Required
|
|
# Writing to /sys/bus/pci/devices requires root.
|
|
#
|
|
# Driver Guard
|
|
# Only vfio-pci-bound or unbound devices are eligible. This is what stops a
|
|
# mistyped address from pulling the SAS controller, a NIC or an NVMe drive out
|
|
# from under a running system.
|
|
#
|
|
# Running VM Guard
|
|
# A vfio-pci device may be passed through to a live VM. Removing it would rip
|
|
# the device away mid-flight. Running domains are checked for the address.
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent runs at array start.
|
|
#
|
|
# Disabled By Default
|
|
# PCIE_QUIET_ENABLED is false and the device list is empty until an operator
|
|
# fills it in per host. An empty list exits silently, so a host that has no
|
|
# such hardware runs a no-op.
|
|
#
|
|
# Silent on Success
|
|
# Runs every boot — no noise when the devices are already gone.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# PCIE_QUIET_ENABLED
|
|
# Master gate. Nothing is removed while false. (default: false)
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_PCIE_QUIET_DEVICES
|
|
# PCI addresses to remove, full DDDD:BB:DD.F form. (default: empty)
|
|
# Aliased to PCIE_QUIET_DEVICES by detect_hosts.
|
|
#
|
|
# HOST1 example — both are onboard, stubbed to vfio, used by nothing:
|
|
# "0000:03:00.0" ASMedia ASM1143 USB 3.1 controller
|
|
# "0000:04:00.0" Intel Wireless 8265
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# pcie_aer_quiet.sh
|
|
# Remove every eligible listed device. Silent if all are already absent.
|
|
#
|
|
# pcie_aer_quiet.sh --dry-run
|
|
# Show what would be removed, refused or skipped. No sysfs write.
|
|
#
|
|
# pcie_aer_quiet.sh --status
|
|
# Show each listed device, its driver, and whether it is present.
|
|
#
|
|
# pcie_aer_quiet.sh --log
|
|
# Verbose output showing each guard decision.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../../../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root — removing PCI devices requires root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no devices will be removed"
|
|
|
|
PCI_DEVICE_ROOT="/sys/bus/pci/devices"
|
|
PCI_ADDR_PATTERN='^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$'
|
|
|
|
# ── Which driver currently owns the device, or "none" ────────────────────────────────────────
|
|
pcie_driver_of() {
|
|
local addr="$1" link
|
|
link=$(readlink "$PCI_DEVICE_ROOT/$addr/driver" 2>/dev/null) || { echo "none"; return 0; }
|
|
[[ -n "$link" ]] && basename "$link" || echo "none"
|
|
}
|
|
|
|
# ── Is the address passed through to a VM that is running right now? ─────────────────────────
|
|
# Returns the domain name on stdout when claimed, empty when free.
|
|
pcie_claimed_by_vm() {
|
|
local addr="$1"
|
|
command -v virsh >/dev/null 2>&1 || return 0
|
|
|
|
local domain bus slot func xml_addr dom
|
|
domain="${addr%%:*}"
|
|
bus="${addr#*:}"; bus="${bus%%:*}"
|
|
slot="${addr##*:}"; slot="${slot%%.*}"
|
|
func="${addr##*.}"
|
|
|
|
xml_addr="domain='0x${domain}' bus='0x${bus}' slot='0x${slot}' function='0x${func}'"
|
|
|
|
while read -r dom; do
|
|
[[ -z "$dom" ]] && continue
|
|
if virsh dumpxml "$dom" 2>/dev/null | awk '/<hostdev/,/<\/hostdev>/' | \
|
|
grep -qF "$xml_addr"; then
|
|
echo "$dom"
|
|
return 0
|
|
fi
|
|
done < <(virsh list --name --state-running 2>/dev/null)
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Enabled: $PCIE_QUIET_ENABLED"
|
|
echo "$ICON_PLUGIN Devices: ${#PCIE_QUIET_DEVICES[@]} listed"
|
|
echo ""
|
|
|
|
if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
|
|
echo " $ICON_SKIP No devices configured — nothing to do on this host"
|
|
else
|
|
for addr in "${PCIE_QUIET_DEVICES[@]}"; do
|
|
if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
|
|
echo " $ICON_ERROR $addr — malformed address"
|
|
elif [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
|
echo " $ICON_SUCCESS $addr — already absent from the bus ✅"
|
|
else
|
|
echo " $ICON_WARN $addr — present, driver: $(pcie_driver_of "$addr")"
|
|
fi
|
|
done
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Gates ━━━
|
|
# ==============================================================================================
|
|
if [[ "$PCIE_QUIET_ENABLED" != true ]]; then
|
|
log "$ICON_SKIP PCIE_QUIET_ENABLED is false — nothing removed"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
|
|
log "$ICON_SKIP No PCIe devices configured for $MY_ID — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Remove ━━━
|
|
# ==============================================================================================
|
|
START=$(date +%s)
|
|
|
|
REMOVED=0
|
|
ALREADY=0
|
|
REFUSED=0
|
|
|
|
for addr in "${PCIE_QUIET_DEVICES[@]}"; do
|
|
|
|
# ── Shape ────────────────────────────────────────────────────────────────────────────────
|
|
if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
|
|
error "Malformed PCI address, refusing: '$addr' (expected DDDD:BB:DD.F)"
|
|
REFUSED=$(( REFUSED + 1 ))
|
|
continue
|
|
fi
|
|
|
|
# ── Present? ─────────────────────────────────────────────────────────────────────────────
|
|
if [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
|
log "$ICON_SUCCESS $addr already absent"
|
|
ALREADY=$(( ALREADY + 1 ))
|
|
continue
|
|
fi
|
|
|
|
# ── Driver guard ─────────────────────────────────────────────────────────────────────────
|
|
DRIVER=$(pcie_driver_of "$addr")
|
|
if [[ "$DRIVER" != "vfio-pci" && "$DRIVER" != "none" ]]; then
|
|
error "$addr is bound to '$DRIVER' — refusing to remove a device in use"
|
|
error "Only vfio-pci-bound or unbound devices are eligible. Stub it first, or remove it from PCIE_QUIET_DEVICES."
|
|
notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — bound to $DRIVER" \
|
|
"PCIe AER Quiet" "warning"
|
|
REFUSED=$(( REFUSED + 1 ))
|
|
continue
|
|
fi
|
|
|
|
# ── Running VM guard ─────────────────────────────────────────────────────────────────────
|
|
CLAIMED_BY=$(pcie_claimed_by_vm "$addr")
|
|
if [[ -n "$CLAIMED_BY" ]]; then
|
|
error "$addr is passed through to running VM '$CLAIMED_BY' — refusing to remove"
|
|
notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — in use by VM $CLAIMED_BY" \
|
|
"PCIe AER Quiet" "warning"
|
|
REFUSED=$(( REFUSED + 1 ))
|
|
continue
|
|
fi
|
|
|
|
# ── Apply ────────────────────────────────────────────────────────────────────────────────
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would remove $addr (driver: $DRIVER)"
|
|
continue
|
|
fi
|
|
|
|
log "Removing $addr (driver: $DRIVER)..."
|
|
if ! echo 1 > "$PCI_DEVICE_ROOT/$addr/remove" 2>/dev/null; then
|
|
error "Write to $PCI_DEVICE_ROOT/$addr/remove failed"
|
|
REFUSED=$(( REFUSED + 1 ))
|
|
continue
|
|
fi
|
|
|
|
# ── Verify ───────────────────────────────────────────────────────────────────────────────
|
|
if [[ -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
|
error "$addr still present after remove — kernel refused"
|
|
REFUSED=$(( REFUSED + 1 ))
|
|
continue
|
|
fi
|
|
|
|
echo "Removed $addr ✅"
|
|
REMOVED=$(( REMOVED + 1 ))
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
if [[ $REMOVED -eq 0 && $REFUSED -eq 0 ]]; then
|
|
log "$ICON_SUCCESS All ${ALREADY} configured device(s) already absent"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_SUCCESS Removed: $REMOVED"
|
|
echo "$ICON_SKIP Already: $ALREADY"
|
|
[[ $REFUSED -gt 0 ]] && echo "$ICON_ERROR Refused: $REFUSED"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
if [[ $REFUSED -gt 0 ]]; then
|
|
echo "$ICON_WARN Status: completed with refusals ⚠️"
|
|
else
|
|
echo "$ICON_DONE Status: done ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
exit 0
|