Remove dead PCIe hardware at array start so correctable AER errors stop burying the syslog
This commit is contained in:
Gmer4Lfe
@@ -67,6 +67,7 @@
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
# PCIe AER QUIET dead PCIe devices removed at array start to stop AER log spam
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
@@ -510,6 +511,19 @@
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ PCIe AER Quiet ━━━
|
||||
# PCI addresses removed from the bus at array start so dead hardware stops spamming
|
||||
# correctable AER errors into syslog. Full DDDD:BB:DD.F form — find them with:
|
||||
# grep -o "from [0-9a-f:.]*" /var/log/syslog | sort | uniq -c | sort -rn
|
||||
#
|
||||
# Only devices bound to vfio-pci or to no driver at all are eligible. Anything with a
|
||||
# live driver is refused, so a mistyped address cannot pull an HBA or NIC out from
|
||||
# under a running system. Devices claimed by a running VM are refused too.
|
||||
# Gated by PCIE_QUIET_ENABLED in master.conf. Empty list = no-op.
|
||||
HOSTN_PCIE_QUIET_DEVICES=(
|
||||
# "0000:03:00.0"
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
HOSTN_ZFS_REPORT_IGNORE_POOLS=(
|
||||
# "disk5"
|
||||
|
||||
@@ -358,6 +358,7 @@
|
||||
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
|
||||
# watchdog_orchestrator.sh — NOT launched here.
|
||||
ARRAY_START_SCRIPTS=(
|
||||
"Plugin/unraid/System_Essentials/pcie_aer_quiet.sh" # drop AER-spamming dead hardware — runs first so later logs stay readable
|
||||
"Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register Varaverk API key — registry is ephemeral
|
||||
"System_Essentials/conf_sync.sh" # pull partner confs + push own conf into /tmp/.vv/ RAM cache
|
||||
"System_Essentials/conf_cache_restore.sh" # load partner confs from persistent backup if conf_sync couldn't reach partner
|
||||
@@ -993,6 +994,19 @@
|
||||
# log lines per hour that have no diagnostic value. Filter removes them at source.
|
||||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||||
|
||||
# ━━━ PCIe AER Quiet ━━━
|
||||
# Master gate for pcie_aer_quiet.sh — removes dead PCIe hardware from the bus at
|
||||
# array start so it stops flooding syslog with correctable AER errors.
|
||||
#
|
||||
# Correctable means the link recovered, so the errors are harmless — but the kernel
|
||||
# logs every one. Removing the device ends it at the source. Unlike pci=noaer this
|
||||
# keeps uncorrectable AER reporting alive on every other device, which the AI repair
|
||||
# triage relies on to tell a real fault from this noise.
|
||||
#
|
||||
# Devices are listed per host in host*.conf as HOST*_PCIE_QUIET_DEVICES.
|
||||
# Off by default — turn on only after filling in that list for this host.
|
||||
PCIE_QUIET_ENABLED=false
|
||||
|
||||
# ━━━ PHP-FPM ━━━
|
||||
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
|
||||
# Default is very low — increasing it prevents WebGUI slowdowns under load.
|
||||
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================== PCIe AER Quiet ================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Removes listed PCIe devices from the bus at array start so dead hardware stops
|
||||
# flooding syslog with correctable AER errors. Run once via ARRAY_START_SCRIPTS.
|
||||
# Idempotent — silent when the devices are already absent.
|
||||
#
|
||||
# Some onboard controllers throw endless correctable PCIe errors. Correctable
|
||||
# means the link recovered and nothing was lost, but the kernel logs every one,
|
||||
# and on HOST1 that was ~180 lines/day drowning the syslog. Masking the error in
|
||||
# the device's AER registers hides it; `pci=noaer` silences the whole machine
|
||||
# and costs uncorrectable reporting on every device. Removing the device ends it
|
||||
# at the source and leaves AER fully intact everywhere else.
|
||||
#
|
||||
# This only makes sense for hardware nothing is using. The guards below refuse
|
||||
# anything else — see OPERATIONAL SAFEGUARDS.
|
||||
#
|
||||
# A reboot re-enumerates the devices, which is why this reapplies every array
|
||||
# start rather than being a one-time step. It is also the undo: clear the conf
|
||||
# list and reboot.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Refuse Anything In Use
|
||||
# A PCI address is one typo away from the HBA holding the array. The script
|
||||
# removes a device only when the kernel shows it bound to vfio-pci or to no
|
||||
# driver at all. A device with a real driver is skipped loudly, never removed.
|
||||
#
|
||||
# Address Shape Validated Before Any Write
|
||||
# The address goes into a sysfs path. It is matched against the full
|
||||
# DDDD:BB:DD.F form before being used to build one.
|
||||
#
|
||||
# Idempotent Presence Check
|
||||
# A device already gone is not an error — it is the desired state. Silent skip.
|
||||
#
|
||||
# Apply Sequence (per device)
|
||||
# 1. Validate the address shape
|
||||
# 2. Absent from sysfs → already done, skip silently
|
||||
# 3. Bound to a driver other than vfio-pci → refuse, warn, continue
|
||||
# 4. Claimed by a running VM → refuse, warn, continue
|
||||
# 5. Write 1 to the device's sysfs remove node
|
||||
# 6. Verify it left the bus
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# Writing to /sys/bus/pci/devices requires root.
|
||||
#
|
||||
# Driver Guard
|
||||
# Only vfio-pci-bound or unbound devices are eligible. This is what stops a
|
||||
# mistyped address from pulling the SAS controller, a NIC or an NVMe drive out
|
||||
# from under a running system.
|
||||
#
|
||||
# Running VM Guard
|
||||
# A vfio-pci device may be passed through to a live VM. Removing it would rip
|
||||
# the device away mid-flight. Running domains are checked for the address.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs at array start.
|
||||
#
|
||||
# Disabled By Default
|
||||
# PCIE_QUIET_ENABLED is false and the device list is empty until an operator
|
||||
# fills it in per host. An empty list exits silently, so a host that has no
|
||||
# such hardware runs a no-op.
|
||||
#
|
||||
# Silent on Success
|
||||
# Runs every boot — no noise when the devices are already gone.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PCIE_QUIET_ENABLED
|
||||
# Master gate. Nothing is removed while false. (default: false)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_PCIE_QUIET_DEVICES
|
||||
# PCI addresses to remove, full DDDD:BB:DD.F form. (default: empty)
|
||||
# Aliased to PCIE_QUIET_DEVICES by detect_hosts.
|
||||
#
|
||||
# HOST1 example — both are onboard, stubbed to vfio, used by nothing:
|
||||
# "0000:03:00.0" ASMedia ASM1143 USB 3.1 controller
|
||||
# "0000:04:00.0" Intel Wireless 8265
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# pcie_aer_quiet.sh
|
||||
# Remove every eligible listed device. Silent if all are already absent.
|
||||
#
|
||||
# pcie_aer_quiet.sh --dry-run
|
||||
# Show what would be removed, refused or skipped. No sysfs write.
|
||||
#
|
||||
# pcie_aer_quiet.sh --status
|
||||
# Show each listed device, its driver, and whether it is present.
|
||||
#
|
||||
# pcie_aer_quiet.sh --log
|
||||
# Verbose output showing each guard decision.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — removing PCI devices requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no devices will be removed"
|
||||
|
||||
PCI_DEVICE_ROOT="/sys/bus/pci/devices"
|
||||
PCI_ADDR_PATTERN='^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$'
|
||||
|
||||
# ── Which driver currently owns the device, or "none" ────────────────────────────────────────
|
||||
pcie_driver_of() {
|
||||
local addr="$1" link
|
||||
link=$(readlink "$PCI_DEVICE_ROOT/$addr/driver" 2>/dev/null) || { echo "none"; return 0; }
|
||||
[[ -n "$link" ]] && basename "$link" || echo "none"
|
||||
}
|
||||
|
||||
# ── Is the address passed through to a VM that is running right now? ─────────────────────────
|
||||
# Returns the domain name on stdout when claimed, empty when free.
|
||||
pcie_claimed_by_vm() {
|
||||
local addr="$1"
|
||||
command -v virsh >/dev/null 2>&1 || return 0
|
||||
|
||||
local domain bus slot func xml_addr dom
|
||||
domain="${addr%%:*}"
|
||||
bus="${addr#*:}"; bus="${bus%%:*}"
|
||||
slot="${addr##*:}"; slot="${slot%%.*}"
|
||||
func="${addr##*.}"
|
||||
|
||||
xml_addr="domain='0x${domain}' bus='0x${bus}' slot='0x${slot}' function='0x${func}'"
|
||||
|
||||
while read -r dom; do
|
||||
[[ -z "$dom" ]] && continue
|
||||
if virsh dumpxml "$dom" 2>/dev/null | awk '/<hostdev/,/<\/hostdev>/' | \
|
||||
grep -qF "$xml_addr"; then
|
||||
echo "$dom"
|
||||
return 0
|
||||
fi
|
||||
done < <(virsh list --name --state-running 2>/dev/null)
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Enabled: $PCIE_QUIET_ENABLED"
|
||||
echo "$ICON_PLUGIN Devices: ${#PCIE_QUIET_DEVICES[@]} listed"
|
||||
echo ""
|
||||
|
||||
if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
|
||||
echo " $ICON_SKIP No devices configured — nothing to do on this host"
|
||||
else
|
||||
for addr in "${PCIE_QUIET_DEVICES[@]}"; do
|
||||
if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
|
||||
echo " $ICON_ERROR $addr — malformed address"
|
||||
elif [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
||||
echo " $ICON_SUCCESS $addr — already absent from the bus ✅"
|
||||
else
|
||||
echo " $ICON_WARN $addr — present, driver: $(pcie_driver_of "$addr")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Gates ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$PCIE_QUIET_ENABLED" != true ]]; then
|
||||
log "$ICON_SKIP PCIE_QUIET_ENABLED is false — nothing removed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
|
||||
log "$ICON_SKIP No PCIe devices configured for $MY_ID — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remove ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
REMOVED=0
|
||||
ALREADY=0
|
||||
REFUSED=0
|
||||
|
||||
for addr in "${PCIE_QUIET_DEVICES[@]}"; do
|
||||
|
||||
# ── Shape ────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
|
||||
error "Malformed PCI address, refusing: '$addr' (expected DDDD:BB:DD.F)"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Present? ─────────────────────────────────────────────────────────────────────────────
|
||||
if [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
||||
log "$ICON_SUCCESS $addr already absent"
|
||||
ALREADY=$(( ALREADY + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Driver guard ─────────────────────────────────────────────────────────────────────────
|
||||
DRIVER=$(pcie_driver_of "$addr")
|
||||
if [[ "$DRIVER" != "vfio-pci" && "$DRIVER" != "none" ]]; then
|
||||
error "$addr is bound to '$DRIVER' — refusing to remove a device in use"
|
||||
error "Only vfio-pci-bound or unbound devices are eligible. Stub it first, or remove it from PCIE_QUIET_DEVICES."
|
||||
notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — bound to $DRIVER" \
|
||||
"PCIe AER Quiet" "warning"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Running VM guard ─────────────────────────────────────────────────────────────────────
|
||||
CLAIMED_BY=$(pcie_claimed_by_vm "$addr")
|
||||
if [[ -n "$CLAIMED_BY" ]]; then
|
||||
error "$addr is passed through to running VM '$CLAIMED_BY' — refusing to remove"
|
||||
notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — in use by VM $CLAIMED_BY" \
|
||||
"PCIe AER Quiet" "warning"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Apply ────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove $addr (driver: $DRIVER)"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Removing $addr (driver: $DRIVER)..."
|
||||
if ! echo 1 > "$PCI_DEVICE_ROOT/$addr/remove" 2>/dev/null; then
|
||||
error "Write to $PCI_DEVICE_ROOT/$addr/remove failed"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Verify ───────────────────────────────────────────────────────────────────────────────
|
||||
if [[ -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
||||
error "$addr still present after remove — kernel refused"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Removed $addr ✅"
|
||||
REMOVED=$(( REMOVED + 1 ))
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ $REMOVED -eq 0 && $REFUSED -eq 0 ]]; then
|
||||
log "$ICON_SUCCESS All ${ALREADY} configured device(s) already absent"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SUCCESS Removed: $REMOVED"
|
||||
echo "$ICON_SKIP Already: $ALREADY"
|
||||
[[ $REFUSED -gt 0 ]] && echo "$ICON_ERROR Refused: $REFUSED"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
if [[ $REFUSED -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: completed with refusals ⚠️"
|
||||
else
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
@@ -755,6 +755,7 @@ detect_hosts() {
|
||||
_alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS"
|
||||
_alias_array "RW_PAUSE_CONTAINERS"
|
||||
_alias_array "RW_STOP_CONTAINERS"
|
||||
_alias_array "PCIE_QUIET_DEVICES"
|
||||
|
||||
# ── Set array aliases — associative arrays ────────────────────────────────
|
||||
# Associative arrays cannot be copied with eval — must be rebuilt key by key
|
||||
|
||||
Reference in New Issue
Block a user