diff --git a/Deployment/host.conf.template b/Deployment/host.conf.template index 576b11b..d9fa539 100644 --- a/Deployment/host.conf.template +++ b/Deployment/host.conf.template @@ -67,6 +67,7 @@ # CERTIFICATE MONITOR domains checked for SSL expiry # SMART HEALTH drives to skip in SMART monitoring # ZFS REPORT pools to exclude from ZFS health report +# PCIe AER QUIET dead PCIe devices removed at array start to stop AER log spam # # ── RESOURCE MANAGER ─────────────────────────────────────────────────────────────────────── # RESOURCE MANAGER containers paused/stopped under memory pressure @@ -510,6 +511,19 @@ "sda" # boot USB — SMART not meaningful on flash drives ) +# ━━━ PCIe AER Quiet ━━━ +# PCI addresses removed from the bus at array start so dead hardware stops spamming +# correctable AER errors into syslog. Full DDDD:BB:DD.F form — find them with: +# grep -o "from [0-9a-f:.]*" /var/log/syslog | sort | uniq -c | sort -rn +# +# Only devices bound to vfio-pci or to no driver at all are eligible. Anything with a +# live driver is refused, so a mistyped address cannot pull an HBA or NIC out from +# under a running system. Devices claimed by a running VM are refused too. +# Gated by PCIE_QUIET_ENABLED in master.conf. Empty list = no-op. + HOSTN_PCIE_QUIET_DEVICES=( + # "0000:03:00.0" + ) + # ━━━ ZFS Report ━━━ HOSTN_ZFS_REPORT_IGNORE_POOLS=( # "disk5" diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 62ccea5..2b8eb41 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -358,6 +358,7 @@ # Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via # watchdog_orchestrator.sh — NOT launched here. ARRAY_START_SCRIPTS=( + "Plugin/unraid/System_Essentials/pcie_aer_quiet.sh" # drop AER-spamming dead hardware — runs first so later logs stay readable "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register Varaverk API key — registry is ephemeral "System_Essentials/conf_sync.sh" # pull partner confs + push own conf into /tmp/.vv/ RAM cache "System_Essentials/conf_cache_restore.sh" # load partner confs from persistent backup if conf_sync couldn't reach partner @@ -993,6 +994,19 @@ # log lines per hour that have no diagnostic value. Filter removes them at source. FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" +# ━━━ PCIe AER Quiet ━━━ +# Master gate for pcie_aer_quiet.sh — removes dead PCIe hardware from the bus at +# array start so it stops flooding syslog with correctable AER errors. +# +# Correctable means the link recovered, so the errors are harmless — but the kernel +# logs every one. Removing the device ends it at the source. Unlike pci=noaer this +# keeps uncorrectable AER reporting alive on every other device, which the AI repair +# triage relies on to tell a real fault from this noise. +# +# Devices are listed per host in host*.conf as HOST*_PCIE_QUIET_DEVICES. +# Off by default — turn on only after filling in that list for this host. + PCIE_QUIET_ENABLED=false + # ━━━ PHP-FPM ━━━ # Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. # Default is very low — increasing it prevents WebGUI slowdowns under load. diff --git a/Plugin/unraid/System_Essentials/pcie_aer_quiet.sh b/Plugin/unraid/System_Essentials/pcie_aer_quiet.sh new file mode 100755 index 0000000..21a203d --- /dev/null +++ b/Plugin/unraid/System_Essentials/pcie_aer_quiet.sh @@ -0,0 +1,304 @@ +#!/bin/bash +# ============================================================================================== +# ============================== PCIe AER Quiet ================================================ +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Removes listed PCIe devices from the bus at array start so dead hardware stops +# flooding syslog with correctable AER errors. Run once via ARRAY_START_SCRIPTS. +# Idempotent — silent when the devices are already absent. +# +# Some onboard controllers throw endless correctable PCIe errors. Correctable +# means the link recovered and nothing was lost, but the kernel logs every one, +# and on HOST1 that was ~180 lines/day drowning the syslog. Masking the error in +# the device's AER registers hides it; `pci=noaer` silences the whole machine +# and costs uncorrectable reporting on every device. Removing the device ends it +# at the source and leaves AER fully intact everywhere else. +# +# This only makes sense for hardware nothing is using. The guards below refuse +# anything else — see OPERATIONAL SAFEGUARDS. +# +# A reboot re-enumerates the devices, which is why this reapplies every array +# start rather than being a one-time step. It is also the undo: clear the conf +# list and reboot. +# +# ============================================================================================== +# DESIGN PRINCIPLES +# ============================================================================================== +# +# Refuse Anything In Use +# A PCI address is one typo away from the HBA holding the array. The script +# removes a device only when the kernel shows it bound to vfio-pci or to no +# driver at all. A device with a real driver is skipped loudly, never removed. +# +# Address Shape Validated Before Any Write +# The address goes into a sysfs path. It is matched against the full +# DDDD:BB:DD.F form before being used to build one. +# +# Idempotent Presence Check +# A device already gone is not an error — it is the desired state. Silent skip. +# +# Apply Sequence (per device) +# 1. Validate the address shape +# 2. Absent from sysfs → already done, skip silently +# 3. Bound to a driver other than vfio-pci → refuse, warn, continue +# 4. Claimed by a running VM → refuse, warn, continue +# 5. Write 1 to the device's sysfs remove node +# 6. Verify it left the bus +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Root Required +# Writing to /sys/bus/pci/devices requires root. +# +# Driver Guard +# Only vfio-pci-bound or unbound devices are eligible. This is what stops a +# mistyped address from pulling the SAS controller, a NIC or an NVMe drive out +# from under a running system. +# +# Running VM Guard +# A vfio-pci device may be passed through to a live VM. Removing it would rip +# the device away mid-flight. Running domains are checked for the address. +# +# Single Instance Lock +# acquire_lock prevents concurrent runs at array start. +# +# Disabled By Default +# PCIE_QUIET_ENABLED is false and the device list is empty until an operator +# fills it in per host. An empty list exits silently, so a host that has no +# such hardware runs a no-op. +# +# Silent on Success +# Runs every boot — no noise when the devices are already gone. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# master.conf +# +# PCIE_QUIET_ENABLED +# Master gate. Nothing is removed while false. (default: false) +# +# host*.conf +# +# HOST*_PCIE_QUIET_DEVICES +# PCI addresses to remove, full DDDD:BB:DD.F form. (default: empty) +# Aliased to PCIE_QUIET_DEVICES by detect_hosts. +# +# HOST1 example — both are onboard, stubbed to vfio, used by nothing: +# "0000:03:00.0" ASMedia ASM1143 USB 3.1 controller +# "0000:04:00.0" Intel Wireless 8265 +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# pcie_aer_quiet.sh +# Remove every eligible listed device. Silent if all are already absent. +# +# pcie_aer_quiet.sh --dry-run +# Show what would be removed, refused or skipped. No sysfs write. +# +# pcie_aer_quiet.sh --status +# Show each listed device, its driver, and whether it is present. +# +# pcie_aer_quiet.sh --log +# Verbose output showing each guard decision. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../../../load_config.sh" + +parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root — removing PCI devices requires root" + exit 1 +fi + +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no devices will be removed" + +PCI_DEVICE_ROOT="/sys/bus/pci/devices" +PCI_ADDR_PATTERN='^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$' + +# ── Which driver currently owns the device, or "none" ──────────────────────────────────────── +pcie_driver_of() { + local addr="$1" link + link=$(readlink "$PCI_DEVICE_ROOT/$addr/driver" 2>/dev/null) || { echo "none"; return 0; } + [[ -n "$link" ]] && basename "$link" || echo "none" +} + +# ── Is the address passed through to a VM that is running right now? ───────────────────────── +# Returns the domain name on stdout when claimed, empty when free. +pcie_claimed_by_vm() { + local addr="$1" + command -v virsh >/dev/null 2>&1 || return 0 + + local domain bus slot func xml_addr dom + domain="${addr%%:*}" + bus="${addr#*:}"; bus="${bus%%:*}" + slot="${addr##*:}"; slot="${slot%%.*}" + func="${addr##*.}" + + xml_addr="domain='0x${domain}' bus='0x${bus}' slot='0x${slot}' function='0x${func}'" + + while read -r dom; do + [[ -z "$dom" ]] && continue + if virsh dumpxml "$dom" 2>/dev/null | awk '//' | \ + grep -qF "$xml_addr"; then + echo "$dom" + return 0 + fi + done < <(virsh list --name --state-running 2>/dev/null) +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_GEAR Enabled: $PCIE_QUIET_ENABLED" + echo "$ICON_PLUGIN Devices: ${#PCIE_QUIET_DEVICES[@]} listed" + echo "" + + if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then + echo " $ICON_SKIP No devices configured — nothing to do on this host" + else + for addr in "${PCIE_QUIET_DEVICES[@]}"; do + if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then + echo " $ICON_ERROR $addr — malformed address" + elif [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then + echo " $ICON_SUCCESS $addr — already absent from the bus ✅" + else + echo " $ICON_WARN $addr — present, driver: $(pcie_driver_of "$addr")" + fi + done + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Gates ━━━ +# ============================================================================================== +if [[ "$PCIE_QUIET_ENABLED" != true ]]; then + log "$ICON_SKIP PCIE_QUIET_ENABLED is false — nothing removed" + exit 0 +fi + +if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then + log "$ICON_SKIP No PCIe devices configured for $MY_ID — nothing to do" + exit 0 +fi + +# ============================================================================================== +# ━━━ Remove ━━━ +# ============================================================================================== +START=$(date +%s) + +REMOVED=0 +ALREADY=0 +REFUSED=0 + +for addr in "${PCIE_QUIET_DEVICES[@]}"; do + + # ── Shape ──────────────────────────────────────────────────────────────────────────────── + if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then + error "Malformed PCI address, refusing: '$addr' (expected DDDD:BB:DD.F)" + REFUSED=$(( REFUSED + 1 )) + continue + fi + + # ── Present? ───────────────────────────────────────────────────────────────────────────── + if [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then + log "$ICON_SUCCESS $addr already absent" + ALREADY=$(( ALREADY + 1 )) + continue + fi + + # ── Driver guard ───────────────────────────────────────────────────────────────────────── + DRIVER=$(pcie_driver_of "$addr") + if [[ "$DRIVER" != "vfio-pci" && "$DRIVER" != "none" ]]; then + error "$addr is bound to '$DRIVER' — refusing to remove a device in use" + error "Only vfio-pci-bound or unbound devices are eligible. Stub it first, or remove it from PCIE_QUIET_DEVICES." + notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — bound to $DRIVER" \ + "PCIe AER Quiet" "warning" + REFUSED=$(( REFUSED + 1 )) + continue + fi + + # ── Running VM guard ───────────────────────────────────────────────────────────────────── + CLAIMED_BY=$(pcie_claimed_by_vm "$addr") + if [[ -n "$CLAIMED_BY" ]]; then + error "$addr is passed through to running VM '$CLAIMED_BY' — refusing to remove" + notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — in use by VM $CLAIMED_BY" \ + "PCIe AER Quiet" "warning" + REFUSED=$(( REFUSED + 1 )) + continue + fi + + # ── Apply ──────────────────────────────────────────────────────────────────────────────── + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would remove $addr (driver: $DRIVER)" + continue + fi + + log "Removing $addr (driver: $DRIVER)..." + if ! echo 1 > "$PCI_DEVICE_ROOT/$addr/remove" 2>/dev/null; then + error "Write to $PCI_DEVICE_ROOT/$addr/remove failed" + REFUSED=$(( REFUSED + 1 )) + continue + fi + + # ── Verify ─────────────────────────────────────────────────────────────────────────────── + if [[ -e "$PCI_DEVICE_ROOT/$addr" ]]; then + error "$addr still present after remove — kernel refused" + REFUSED=$(( REFUSED + 1 )) + continue + fi + + echo "Removed $addr ✅" + REMOVED=$(( REMOVED + 1 )) +done + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +if [[ $REMOVED -eq 0 && $REFUSED -eq 0 ]]; then + log "$ICON_SUCCESS All ${ALREADY} configured device(s) already absent" + exit 0 +fi + +echo "" +echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_SUCCESS Removed: $REMOVED" +echo "$ICON_SKIP Already: $ALREADY" +[[ $REFUSED -gt 0 ]] && echo "$ICON_ERROR Refused: $REFUSED" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +echo "" +if [[ $REFUSED -gt 0 ]]; then + echo "$ICON_WARN Status: completed with refusals ⚠️" +else + echo "$ICON_DONE Status: done ✅" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +exit 0 diff --git a/common.sh b/common.sh index a0179ba..6e2ca2f 100755 --- a/common.sh +++ b/common.sh @@ -755,6 +755,7 @@ detect_hosts() { _alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS" _alias_array "RW_PAUSE_CONTAINERS" _alias_array "RW_STOP_CONTAINERS" + _alias_array "PCIE_QUIET_DEVICES" # ── Set array aliases — associative arrays ──────────────────────────────── # Associative arrays cannot be copied with eval — must be rebuilt key by key