Remove dead PCIe hardware at array start so correctable AER errors stop burying the syslog

Deployment/host.conf.template

@@ -67,6 +67,7 @@
 #   CERTIFICATE MONITOR     domains checked for SSL expiry
 #   SMART HEALTH            drives to skip in SMART monitoring
 #   ZFS REPORT              pools to exclude from ZFS health report
+#   PCIe AER QUIET          dead PCIe devices removed at array start to stop AER log spam
 #
 #   ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
 #   RESOURCE MANAGER        containers paused/stopped under memory pressure
@@ -510,6 +511,19 @@
         "sda"   # boot USB — SMART not meaningful on flash drives
     )
 
+# ━━━ PCIe AER Quiet ━━━
+# PCI addresses removed from the bus at array start so dead hardware stops spamming
+# correctable AER errors into syslog. Full DDDD:BB:DD.F form — find them with:
+#   grep -o "from [0-9a-f:.]*" /var/log/syslog | sort | uniq -c | sort -rn
+#
+# Only devices bound to vfio-pci or to no driver at all are eligible. Anything with a
+# live driver is refused, so a mistyped address cannot pull an HBA or NIC out from
+# under a running system. Devices claimed by a running VM are refused too.
+# Gated by PCIE_QUIET_ENABLED in master.conf. Empty list = no-op.
+    HOSTN_PCIE_QUIET_DEVICES=(
+        # "0000:03:00.0"
+    )
+
 # ━━━ ZFS Report ━━━
     HOSTN_ZFS_REPORT_IGNORE_POOLS=(
         # "disk5"

Deployment/master.conf.template

@@ -358,6 +358,7 @@
 # Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
 # watchdog_orchestrator.sh — NOT launched here.
     ARRAY_START_SCRIPTS=(
+        "Plugin/unraid/System_Essentials/pcie_aer_quiet.sh"         # drop AER-spamming dead hardware — runs first so later logs stay readable
         "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
         "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
         "System_Essentials/conf_cache_restore.sh"     # load partner confs from persistent backup if conf_sync couldn't reach partner
@@ -993,6 +994,19 @@
 # log lines per hour that have no diagnostic value. Filter removes them at source.
     FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
 
+# ━━━ PCIe AER Quiet ━━━
+# Master gate for pcie_aer_quiet.sh — removes dead PCIe hardware from the bus at
+# array start so it stops flooding syslog with correctable AER errors.
+#
+# Correctable means the link recovered, so the errors are harmless — but the kernel
+# logs every one. Removing the device ends it at the source. Unlike pci=noaer this
+# keeps uncorrectable AER reporting alive on every other device, which the AI repair
+# triage relies on to tell a real fault from this noise.
+#
+# Devices are listed per host in host*.conf as HOST*_PCIE_QUIET_DEVICES.
+# Off by default — turn on only after filling in that list for this host.
+    PCIE_QUIET_ENABLED=false
+
 # ━━━ PHP-FPM ━━━
 # Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
 # Default is very low — increasing it prevents WebGUI slowdowns under load.

Plugin/unraid/System_Essentials/pcie_aer_quiet.sh

@@ -0,0 +1,304 @@
+#!/bin/bash
+# ==============================================================================================
+# ============================== PCIe AER Quiet ================================================
+# ==============================================================================================
+#
+# PURPOSE
+# ─────────────────────────────────────────────────────────────────────────────
+# Removes listed PCIe devices from the bus at array start so dead hardware stops
+# flooding syslog with correctable AER errors. Run once via ARRAY_START_SCRIPTS.
+# Idempotent — silent when the devices are already absent.
+#
+# Some onboard controllers throw endless correctable PCIe errors. Correctable
+# means the link recovered and nothing was lost, but the kernel logs every one,
+# and on HOST1 that was ~180 lines/day drowning the syslog. Masking the error in
+# the device's AER registers hides it; `pci=noaer` silences the whole machine
+# and costs uncorrectable reporting on every device. Removing the device ends it
+# at the source and leaves AER fully intact everywhere else.
+#
+# This only makes sense for hardware nothing is using. The guards below refuse
+# anything else — see OPERATIONAL SAFEGUARDS.
+#
+# A reboot re-enumerates the devices, which is why this reapplies every array
+# start rather than being a one-time step. It is also the undo: clear the conf
+# list and reboot.
+#
+# ==============================================================================================
+# DESIGN PRINCIPLES
+# ==============================================================================================
+#
+# Refuse Anything In Use
+#   A PCI address is one typo away from the HBA holding the array. The script
+#   removes a device only when the kernel shows it bound to vfio-pci or to no
+#   driver at all. A device with a real driver is skipped loudly, never removed.
+#
+# Address Shape Validated Before Any Write
+#   The address goes into a sysfs path. It is matched against the full
+#   DDDD:BB:DD.F form before being used to build one.
+#
+# Idempotent Presence Check
+#   A device already gone is not an error — it is the desired state. Silent skip.
+#
+# Apply Sequence (per device)
+#   1. Validate the address shape
+#   2. Absent from sysfs → already done, skip silently
+#   3. Bound to a driver other than vfio-pci → refuse, warn, continue
+#   4. Claimed by a running VM → refuse, warn, continue
+#   5. Write 1 to the device's sysfs remove node
+#   6. Verify it left the bus
+#
+# ==============================================================================================
+# OPERATIONAL SAFEGUARDS
+# ==============================================================================================
+#
+# Root Required
+#   Writing to /sys/bus/pci/devices requires root.
+#
+# Driver Guard
+#   Only vfio-pci-bound or unbound devices are eligible. This is what stops a
+#   mistyped address from pulling the SAS controller, a NIC or an NVMe drive out
+#   from under a running system.
+#
+# Running VM Guard
+#   A vfio-pci device may be passed through to a live VM. Removing it would rip
+#   the device away mid-flight. Running domains are checked for the address.
+#
+# Single Instance Lock
+#   acquire_lock prevents concurrent runs at array start.
+#
+# Disabled By Default
+#   PCIE_QUIET_ENABLED is false and the device list is empty until an operator
+#   fills it in per host. An empty list exits silently, so a host that has no
+#   such hardware runs a no-op.
+#
+# Silent on Success
+#   Runs every boot — no noise when the devices are already gone.
+#
+# ==============================================================================================
+# CONFIGURATION
+# ==============================================================================================
+#
+# master.conf
+#
+#   PCIE_QUIET_ENABLED
+#       Master gate. Nothing is removed while false. (default: false)
+#
+# host*.conf
+#
+#   HOST*_PCIE_QUIET_DEVICES
+#       PCI addresses to remove, full DDDD:BB:DD.F form. (default: empty)
+#       Aliased to PCIE_QUIET_DEVICES by detect_hosts.
+#
+#       HOST1 example — both are onboard, stubbed to vfio, used by nothing:
+#           "0000:03:00.0"   ASMedia ASM1143 USB 3.1 controller
+#           "0000:04:00.0"   Intel Wireless 8265
+#
+# ==============================================================================================
+# RUNTIME MODES
+# ==============================================================================================
+#
+# pcie_aer_quiet.sh
+#     Remove every eligible listed device. Silent if all are already absent.
+#
+# pcie_aer_quiet.sh --dry-run
+#     Show what would be removed, refused or skipped. No sysfs write.
+#
+# pcie_aer_quiet.sh --status
+#     Show each listed device, its driver, and whether it is present.
+#
+# pcie_aer_quiet.sh --log
+#     Verbose output showing each guard decision.
+#
+# ==============================================================================================
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "$SCRIPT_DIR/../../../load_config.sh"
+
+parse_args "$@"
+
+# ==============================================================================================
+# ━━━ Setup ━━━
+# ==============================================================================================
+if [[ "$EUID" -ne 0 ]]; then
+    error "Must be run as root — removing PCI devices requires root"
+    exit 1
+fi
+
+acquire_lock
+
+detect_hosts
+
+[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no devices will be removed"
+
+PCI_DEVICE_ROOT="/sys/bus/pci/devices"
+PCI_ADDR_PATTERN='^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$'
+
+# ── Which driver currently owns the device, or "none" ────────────────────────────────────────
+pcie_driver_of() {
+    local addr="$1" link
+    link=$(readlink "$PCI_DEVICE_ROOT/$addr/driver" 2>/dev/null) || { echo "none"; return 0; }
+    [[ -n "$link" ]] && basename "$link" || echo "none"
+}
+
+# ── Is the address passed through to a VM that is running right now? ─────────────────────────
+# Returns the domain name on stdout when claimed, empty when free.
+pcie_claimed_by_vm() {
+    local addr="$1"
+    command -v virsh >/dev/null 2>&1 || return 0
+
+    local domain bus slot func xml_addr dom
+    domain="${addr%%:*}"
+    bus="${addr#*:}"; bus="${bus%%:*}"
+    slot="${addr##*:}"; slot="${slot%%.*}"
+    func="${addr##*.}"
+
+    xml_addr="domain='0x${domain}' bus='0x${bus}' slot='0x${slot}' function='0x${func}'"
+
+    while read -r dom; do
+        [[ -z "$dom" ]] && continue
+        if virsh dumpxml "$dom" 2>/dev/null | awk '/<hostdev/,/<\/hostdev>/' | \
+           grep -qF "$xml_addr"; then
+            echo "$dom"
+            return 0
+        fi
+    done < <(virsh list --name --state-running 2>/dev/null)
+}
+
+# ==============================================================================================
+# ━━━ Status ━━━
+# ==============================================================================================
+if [[ "$SHOW_STATUS" == true ]]; then
+    echo ""
+    echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET STATUS ━━━━━"
+    echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
+    echo "$ICON_GEAR   Enabled:   $PCIE_QUIET_ENABLED"
+    echo "$ICON_PLUGIN Devices:   ${#PCIE_QUIET_DEVICES[@]} listed"
+    echo ""
+
+    if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
+        echo "  $ICON_SKIP No devices configured — nothing to do on this host"
+    else
+        for addr in "${PCIE_QUIET_DEVICES[@]}"; do
+            if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
+                echo "  $ICON_ERROR $addr — malformed address"
+            elif [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
+                echo "  $ICON_SUCCESS $addr — already absent from the bus ✅"
+            else
+                echo "  $ICON_WARN   $addr — present, driver: $(pcie_driver_of "$addr")"
+            fi
+        done
+    fi
+    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    exit 0
+fi
+
+# ==============================================================================================
+# ━━━ Gates ━━━
+# ==============================================================================================
+if [[ "$PCIE_QUIET_ENABLED" != true ]]; then
+    log "$ICON_SKIP PCIE_QUIET_ENABLED is false — nothing removed"
+    exit 0
+fi
+
+if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
+    log "$ICON_SKIP No PCIe devices configured for $MY_ID — nothing to do"
+    exit 0
+fi
+
+# ==============================================================================================
+# ━━━ Remove ━━━
+# ==============================================================================================
+START=$(date +%s)
+
+REMOVED=0
+ALREADY=0
+REFUSED=0
+
+for addr in "${PCIE_QUIET_DEVICES[@]}"; do
+
+    # ── Shape ────────────────────────────────────────────────────────────────────────────────
+    if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
+        error "Malformed PCI address, refusing: '$addr' (expected DDDD:BB:DD.F)"
+        REFUSED=$(( REFUSED + 1 ))
+        continue
+    fi
+
+    # ── Present? ─────────────────────────────────────────────────────────────────────────────
+    if [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
+        log "$ICON_SUCCESS $addr already absent"
+        ALREADY=$(( ALREADY + 1 ))
+        continue
+    fi
+
+    # ── Driver guard ─────────────────────────────────────────────────────────────────────────
+    DRIVER=$(pcie_driver_of "$addr")
+    if [[ "$DRIVER" != "vfio-pci" && "$DRIVER" != "none" ]]; then
+        error "$addr is bound to '$DRIVER' — refusing to remove a device in use"
+        error "Only vfio-pci-bound or unbound devices are eligible. Stub it first, or remove it from PCIE_QUIET_DEVICES."
+        notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — bound to $DRIVER" \
+            "PCIe AER Quiet" "warning"
+        REFUSED=$(( REFUSED + 1 ))
+        continue
+    fi
+
+    # ── Running VM guard ─────────────────────────────────────────────────────────────────────
+    CLAIMED_BY=$(pcie_claimed_by_vm "$addr")
+    if [[ -n "$CLAIMED_BY" ]]; then
+        error "$addr is passed through to running VM '$CLAIMED_BY' — refusing to remove"
+        notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — in use by VM $CLAIMED_BY" \
+            "PCIe AER Quiet" "warning"
+        REFUSED=$(( REFUSED + 1 ))
+        continue
+    fi
+
+    # ── Apply ────────────────────────────────────────────────────────────────────────────────
+    if [[ "$DRY_RUN" == true ]]; then
+        warn "DRY RUN — would remove $addr (driver: $DRIVER)"
+        continue
+    fi
+
+    log "Removing $addr (driver: $DRIVER)..."
+    if ! echo 1 > "$PCI_DEVICE_ROOT/$addr/remove" 2>/dev/null; then
+        error "Write to $PCI_DEVICE_ROOT/$addr/remove failed"
+        REFUSED=$(( REFUSED + 1 ))
+        continue
+    fi
+
+    # ── Verify ───────────────────────────────────────────────────────────────────────────────
+    if [[ -e "$PCI_DEVICE_ROOT/$addr" ]]; then
+        error "$addr still present after remove — kernel refused"
+        REFUSED=$(( REFUSED + 1 ))
+        continue
+    fi
+
+    echo "Removed $addr ✅"
+    REMOVED=$(( REMOVED + 1 ))
+done
+
+END=$(date +%s)
+
+# ==============================================================================================
+# ━━━ Summary ━━━
+# ==============================================================================================
+if [[ $REMOVED -eq 0 && $REFUSED -eq 0 ]]; then
+    log "$ICON_SUCCESS All ${ALREADY} configured device(s) already absent"
+    exit 0
+fi
+
+echo ""
+echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET SUMMARY ━━━━━"
+echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
+echo "$ICON_SUCCESS Removed:   $REMOVED"
+echo "$ICON_SKIP  Already:   $ALREADY"
+[[ $REFUSED -gt 0 ]] && echo "$ICON_ERROR Refused:   $REFUSED"
+echo "$ICON_TIME   Duration:  $(format_duration $(( END - START )))"
+echo ""
+if [[ $REFUSED -gt 0 ]]; then
+    echo "$ICON_WARN Status: completed with refusals ⚠️"
+else
+    echo "$ICON_DONE Status: done ✅"
+fi
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+exit 0

common.sh

@@ -755,6 +755,7 @@ detect_hosts() {
     _alias_array "PARTNERSHIP_SERVICES_REPLACE_CONTAINERS"
     _alias_array "RW_PAUSE_CONTAINERS"
     _alias_array "RW_STOP_CONTAINERS"
+    _alias_array "PCIE_QUIET_DEVICES"
 
     # ── Set array aliases — associative arrays ────────────────────────────────
     # Associative arrays cannot be copied with eval — must be rebuilt key by key