#!/bin/bash
# ==============================================================================================
# ============================== PCIe AER Quiet ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Removes listed PCIe devices from the bus at array start so dead hardware stops
# flooding syslog with correctable AER errors. Run once via ARRAY_START_SCRIPTS.
# Idempotent — silent when the devices are already absent.
#
# Some onboard controllers throw endless correctable PCIe errors. Correctable
# means the link recovered and nothing was lost, but the kernel logs every one,
# and on HOST1 that was ~180 lines/day drowning the syslog. Masking the error in
# the device's AER registers hides it; `pci=noaer` silences the whole machine
# and costs uncorrectable reporting on every device. Removing the device ends it
# at the source and leaves AER fully intact everywhere else.
#
# This only makes sense for hardware nothing is using. The guards below refuse
# anything else — see OPERATIONAL SAFEGUARDS.
#
# A reboot re-enumerates the devices, which is why this reapplies every array
# start rather than being a one-time step. It is also the undo: clear the conf
# list and reboot.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# One pass over PCIE_QUIET_DEVICES at array start. Each address is looked up under
# /sys/bus/pci/devices, checked against the guards below, and removed through the kernel's own
# per-device remove attribute. Nothing is masked and no kernel parameter is set.
#
# Idempotent, and silent when there is nothing to do. An address that is already gone is not an
# error — that is the normal state on every array start after the first within one boot.
#
# Reapplied every array start rather than once, because a reboot re-enumerates the bus and the
# devices come back. That is also the undo: clear PCIE_QUIET_DEVICES and reboot.
#
# PCIE_QUIET_ENABLED gates the whole run before any device is touched.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Refuse Anything In Use
#   A PCI address is one typo away from the HBA holding the array. The script
#   removes a device only when the kernel shows it bound to vfio-pci or to no
#   driver at all. A device with a real driver is skipped loudly, never removed.
#
# Address Shape Validated Before Any Write
#   The address goes into a sysfs path. It is matched against the full
#   DDDD:BB:DD.F form before being used to build one.
#
# Idempotent Presence Check
#   A device already gone is not an error — it is the desired state. Silent skip.
#
# Apply Sequence (per device)
#   1. Validate the address shape
#   2. Absent from sysfs → already done, skip silently
#   3. Bound to a driver other than vfio-pci → refuse, warn, continue
#   4. Claimed by a running VM → refuse, warn, continue
#   5. Write 1 to the device's sysfs remove node
#   6. Verify it left the bus
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Required
#   Writing to /sys/bus/pci/devices requires root.
#
# Driver Guard
#   Only vfio-pci-bound or unbound devices are eligible. This is what stops a
#   mistyped address from pulling the SAS controller, a NIC or an NVMe drive out
#   from under a running system.
#
# Running VM Guard
#   A vfio-pci device may be passed through to a live VM. Removing it would rip
#   the device away mid-flight. Running domains are checked for the address.
#
# Single Instance Lock
#   acquire_lock prevents concurrent runs at array start.
#
# Disabled By Default
#   PCIE_QUIET_ENABLED is false and the device list is empty until an operator
#   fills it in per host. An empty list exits silently, so a host that has no
#   such hardware runs a no-op.
#
# Silent on Success
#   Runs every boot — no noise when the devices are already gone.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   PCIE_QUIET_ENABLED
#       Master gate. Nothing is removed while false. (default: false)
#
# host*.conf
#
#   HOST*_PCIE_QUIET_DEVICES
#       PCI addresses to remove, full DDDD:BB:DD.F form. (default: empty)
#       Aliased to PCIE_QUIET_DEVICES by detect_hosts.
#
#       HOST1 example — both are onboard, stubbed to vfio, used by nothing:
#           "0000:03:00.0"   ASMedia ASM1143 USB 3.1 controller
#           "0000:04:00.0"   Intel Wireless 8265
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# pcie_aer_quiet.sh
#     Remove every eligible listed device. Silent if all are already absent.
#
# pcie_aer_quiet.sh --dry-run
#     Show what would be removed, refused or skipped. No sysfs write.
#
# pcie_aer_quiet.sh --status
#     Show each listed device, its driver, and whether it is present.
#
# pcie_aer_quiet.sh --log
#     Verbose output showing each guard decision.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root — removing PCI devices requires root"
    exit 1
fi

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no devices will be removed"

PCI_DEVICE_ROOT="/sys/bus/pci/devices"
PCI_ADDR_PATTERN='^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$'

# ── Which driver currently owns the device, or "none" ────────────────────────────────────────
pcie_driver_of() {
    local addr="$1" link
    link=$(readlink "$PCI_DEVICE_ROOT/$addr/driver" 2>/dev/null) || { echo "none"; return 0; }
    [[ -n "$link" ]] && basename "$link" || echo "none"
}

# ── Is the address passed through to a VM that is running right now? ─────────────────────────
# Returns the domain name on stdout when claimed, empty when free.
pcie_claimed_by_vm() {
    local addr="$1"
    command -v virsh >/dev/null 2>&1 || return 0

    local domain bus slot func xml_addr dom
    domain="${addr%%:*}"
    bus="${addr#*:}"; bus="${bus%%:*}"
    slot="${addr##*:}"; slot="${slot%%.*}"
    func="${addr##*.}"

    xml_addr="domain='0x${domain}' bus='0x${bus}' slot='0x${slot}' function='0x${func}'"

    while read -r dom; do
        [[ -z "$dom" ]] && continue
        if virsh dumpxml "$dom" 2>/dev/null | awk '/<hostdev/,/<\/hostdev>/' | \
           grep -qF "$xml_addr"; then
            echo "$dom"
            return 0
        fi
    done < <(virsh list --name --state-running 2>/dev/null)
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET STATUS ━━━━━"
    echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR   Enabled:   $PCIE_QUIET_ENABLED"
    echo "$ICON_PLUGIN Devices:   ${#PCIE_QUIET_DEVICES[@]} listed"
    echo ""

    if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
        echo "  $ICON_SKIP No devices configured — nothing to do on this host"
    else
        for addr in "${PCIE_QUIET_DEVICES[@]}"; do
            if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
                echo "  $ICON_ERROR $addr — malformed address"
            elif [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
                echo "  $ICON_SUCCESS $addr — already absent from the bus ✅"
            else
                echo "  $ICON_WARN   $addr — present, driver: $(pcie_driver_of "$addr")"
            fi
        done
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Gates ━━━
# ==============================================================================================
if [[ "$PCIE_QUIET_ENABLED" != true ]]; then
    log "$ICON_SKIP PCIE_QUIET_ENABLED is false — nothing removed"
    exit 0
fi

if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
    log "$ICON_SKIP No PCIe devices configured for $MY_ID — nothing to do"
    exit 0
fi

# ==============================================================================================
# ━━━ Remove ━━━
# ==============================================================================================
START=$(date +%s)

REMOVED=0
ALREADY=0
REFUSED=0

for addr in "${PCIE_QUIET_DEVICES[@]}"; do

    # ── Shape ────────────────────────────────────────────────────────────────────────────────
    if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
        error "Malformed PCI address, refusing: '$addr' (expected DDDD:BB:DD.F)"
        REFUSED=$(( REFUSED + 1 ))
        continue
    fi

    # ── Present? ─────────────────────────────────────────────────────────────────────────────
    if [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
        log "$ICON_SUCCESS $addr already absent"
        ALREADY=$(( ALREADY + 1 ))
        continue
    fi

    # ── Driver guard ─────────────────────────────────────────────────────────────────────────
    DRIVER=$(pcie_driver_of "$addr")
    if [[ "$DRIVER" != "vfio-pci" && "$DRIVER" != "none" ]]; then
        error "$addr is bound to '$DRIVER' — refusing to remove a device in use"
        error "Only vfio-pci-bound or unbound devices are eligible. Stub it first, or remove it from PCIE_QUIET_DEVICES."
        notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — bound to $DRIVER" \
            "PCIe AER Quiet" "warning"
        REFUSED=$(( REFUSED + 1 ))
        continue
    fi

    # ── Running VM guard ─────────────────────────────────────────────────────────────────────
    CLAIMED_BY=$(pcie_claimed_by_vm "$addr")
    if [[ -n "$CLAIMED_BY" ]]; then
        error "$addr is passed through to running VM '$CLAIMED_BY' — refusing to remove"
        notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — in use by VM $CLAIMED_BY" \
            "PCIe AER Quiet" "warning"
        REFUSED=$(( REFUSED + 1 ))
        continue
    fi

    # ── Apply ────────────────────────────────────────────────────────────────────────────────
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would remove $addr (driver: $DRIVER)"
        continue
    fi

    log "Removing $addr (driver: $DRIVER)..."
    if ! echo 1 > "$PCI_DEVICE_ROOT/$addr/remove" 2>/dev/null; then
        error "Write to $PCI_DEVICE_ROOT/$addr/remove failed"
        REFUSED=$(( REFUSED + 1 ))
        continue
    fi

    # ── Verify ───────────────────────────────────────────────────────────────────────────────
    if [[ -e "$PCI_DEVICE_ROOT/$addr" ]]; then
        error "$addr still present after remove — kernel refused"
        REFUSED=$(( REFUSED + 1 ))
        continue
    fi

    echo "Removed $addr ✅"
    REMOVED=$(( REMOVED + 1 ))
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ $REMOVED -eq 0 && $REFUSED -eq 0 ]]; then
    log "$ICON_SUCCESS All ${ALREADY} configured device(s) already absent"
    exit 0
fi

echo ""
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET SUMMARY ━━━━━"
echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SUCCESS Removed:   $REMOVED"
echo "$ICON_SKIP  Already:   $ALREADY"
[[ $REFUSED -gt 0 ]] && echo "$ICON_ERROR Refused:   $REFUSED"
echo "$ICON_TIME   Duration:  $(format_duration $(( END - START )))"
echo ""
if [[ $REFUSED -gt 0 ]]; then
    echo "$ICON_WARN Status: completed with refusals ⚠️"
else
    echo "$ICON_DONE Status: done ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

exit 0