Varaverk/Monitors/smart_health.sh

#!/bin/bash
# ==============================================================================================
# ================================= SMART Health Monitor =======================================
# ==============================================================================================
# Checks SMART health attributes for all drives on the system.
# Reads data live from each drive via smartctl — no persistent writes.
# Designed to run weekly as a scheduled report.
#
# ── MONITORED ATTRIBUTES ──────────────────────────────────────────────────────────────────────
#   Overall SMART status      — PASSED/FAILED — immediate fail = drive is dying
#   Reallocated_Sector_Ct     — bad sectors remapped — any > 0 is concerning
#   Current_Pending_Sector    — sectors waiting for reallocation — any > 0 is concerning
#   Offline_Uncorrectable     — sectors that could not be corrected — any > 0 is critical
#   Temperature_Celsius       — vs thresholds from dynamix.cfg (or master.conf fallback)
#   Power_On_Hours            — informational — drive age in days
#
# ── DRIVE DISCOVERY ───────────────────────────────────────────────────────────────────────────
# Discovers drives automatically via /dev/sd* and /dev/nvme* — no config needed.
# NVMe drives use different attribute names — detected and handled automatically.
# HOST*_SMART_IGNORE_DRIVES skips specific drives (e.g. boot USB flash drive).
#
# ── TEMPERATURE THRESHOLDS ────────────────────────────────────────────────────────────────────
# Reads hot/max/hotssd/maxssd from /boot/config/plugins/dynamix/dynamix.cfg at runtime.
# Uses unRAID's own configured thresholds — no need to duplicate them here.
# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if dynamix.cfg not found.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES.
# Each server monitors its own drives with its own ignore list.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   acquire_lock         — smartctl calls are slow, prevent duplicate runs
#   detect_hosts()       — correct ignore list per host via MY_ID aliases
#   validate_unraid_cmd  — smartctl and notify validated before use
#   Silent healthy drives — only problems produce output
#   Silent healthy run   — no notify when all drives pass
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
#   HOST*_SMART_IGNORE_DRIVES — drives skipped in SMART monitoring
#   Aliased by detect_hosts() — script uses SMART_IGNORE_DRIVES
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   SMART_TEMP_WARN  — fallback warn threshold in °C (if dynamix.cfg not found)
#   SMART_TEMP_CRIT  — fallback crit threshold in °C (if dynamix.cfg not found)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   smart_health.sh              — normal run
#   smart_health.sh --dry-run    — show which drives would be checked
#   smart_health.sh --log        — verbose output
#   smart_health.sh --status     — show config and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Monitor script — output is the point
SILENT_MODE=false

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# Validate smartctl — required for all drive checks
validate_unraid_cmd \
    "$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
    "--version" "smartmontools" \
    "smartctl" || {
        error "smartctl not found — install smartmontools"
        notify "SMART health check failed on $(hostname) — smartmontools not installed" \
            "SMART Health" "warning"
        exit 1
    }

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES
detect_hosts

# Load temperature thresholds from dynamix.cfg — unRAID's own settings
get_unraid_temp_thresholds

log "HDD warn: ${UNRAID_DISK_HOT}°C  crit: ${UNRAID_DISK_MAX}°C"
log "SSD warn: ${UNRAID_SSD_HOT}°C   crit: ${UNRAID_SSD_MAX}°C"
log "Ignore:   ${SMART_IGNORE_DRIVES[*]:-none}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST    Identity:      $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_SMART   HDD warn:      ${UNRAID_DISK_HOT}°C"
    echo "$ICON_SMART   HDD crit:      ${UNRAID_DISK_MAX}°C"
    echo "$ICON_SMART   SSD warn:      ${UNRAID_SSD_HOT}°C"
    echo "$ICON_SMART   SSD crit:      ${UNRAID_SSD_MAX}°C"
    echo "$ICON_SMART   Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
    echo "$ICON_GEAR    Dry Run:       $DRY_RUN"
    echo ""
    echo "━━━ Discovered Drives ━━━"
    for drive in /dev/sd? /dev/nvme?; do
        [[ ! -e "$drive" ]] && continue
        drive_name=$(basename "$drive")
        ignored=false
        for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
            [[ "$drive_name" == "$ignore" ]] && ignored=true && break
        done
        if [[ "$ignored" == true ]]; then
            echo "  $ICON_WARN $drive — ignored"
        else
            echo "  $ICON_SMART $drive — would check"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Extract a named SMART attribute value (column 10 — raw value)
get_smart_attr() {
    local drive="$1" attr="$2"
    smartctl -A "$drive" 2>/dev/null | \
        awk -v attr="$attr" '$2 == attr {print $10}'
}

# Get drive temperature — handles HDD (attribute) and NVMe (different output format)
get_drive_temp() {
    local drive="$1"
    local temp

    # Standard HDD SMART attribute
    temp=$(get_smart_attr "$drive" "Temperature_Celsius")
    [[ -n "$temp" ]] && echo "$temp" && return

    # NVMe — temperature in different section
    temp=$(smartctl -A "$drive" 2>/dev/null | \
        awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}')
    [[ -n "$temp" ]] && echo "$temp" && return

    # Fallback — any temperature line
    temp=$(smartctl -A "$drive" 2>/dev/null | \
        awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}')
    echo "${temp:-}"
}

# Detect if a drive is SSD/NVMe (rotational=0)
is_ssd() {
    local drive="$1"
    local dev_name
    dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
    local rotational="/sys/block/$(basename "$drive")/queue/rotational"
    [[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
    # NVMe is always SSD
    [[ "$drive" == *nvme* ]] && return 0
    return 1
}

# ==============================================================================================
# ━━━ SMART Health Check ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""

START=$(date +%s)
DRIVES_OK=()
DRIVES_WARN=()
DRIVES_CRIT=()
DRIVES_SKIP=()

for drive in /dev/sd? /dev/nvme?; do
    [[ ! -e "$drive" ]] && continue
    drive_name=$(basename "$drive")

    # Check ignore list
    ignored=false
    for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
        [[ "$drive_name" == "$ignore" ]] && ignored=true && break
    done

    if [[ "$ignored" == true ]]; then
        log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
        DRIVES_SKIP+=("$drive_name")
        continue
    fi

    echo "━━━ $ICON_SMART $drive_name ━━━"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would check $drive_name"
        echo ""
        continue
    fi

    # Check SMART support
    if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
        warn "$drive_name — SMART not enabled or not supported — skipping"
        DRIVES_SKIP+=("$drive_name")
        echo ""
        continue
    fi

    DRIVE_WARN=false
    DRIVE_CRIT=false

    # Overall SMART status
    SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \
        grep "overall-health" | awk '{print $NF}')
    case "${SMART_STATUS:-}" in
        PASSED)
            log "$drive_name overall status: PASSED" ;;
        FAILED*)
            error "$drive_name overall status: FAILED — drive may be failing"
            DRIVE_CRIT=true ;;
        "")
            warn "$drive_name overall status: unknown — could not read SMART data" ;;
        *)
            warn "$drive_name overall status: $SMART_STATUS" ;;
    esac

    # Reallocated sectors
    REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
    if [[ -n "$REALLOC" ]]; then
        if [[ "$REALLOC" -gt 0 ]]; then
            warn "$ICON_SMART $drive_name — Reallocated sectors: $REALLOC (drive showing wear)"
            DRIVE_WARN=true
        else
            log "$drive_name reallocated sectors: 0 ✅"
        fi
    fi

    # Pending sectors
    PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
    if [[ -n "$PENDING" ]]; then
        if [[ "$PENDING" -gt 0 ]]; then
            warn "$ICON_SMART $drive_name — Pending sectors: $PENDING (awaiting reallocation)"
            DRIVE_WARN=true
        else
            log "$drive_name pending sectors: 0 ✅"
        fi
    fi

    # Uncorrectable sectors — critical threshold
    UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
    if [[ -n "$UNCORR" ]]; then
        if [[ "$UNCORR" -gt 0 ]]; then
            error "$ICON_SMART $drive_name — Uncorrectable sectors: $UNCORR — CRITICAL"
            DRIVE_CRIT=true
        else
            log "$drive_name uncorrectable sectors: 0 ✅"
        fi
    fi

    # Temperature — use SSD/HDD thresholds from dynamix.cfg
    TEMP=$(get_drive_temp "$drive")
    if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then
        if is_ssd "$drive"; then
            WARN_THRESH="$UNRAID_SSD_HOT"
            CRIT_THRESH="$UNRAID_SSD_MAX"
            DRIVE_TYPE="SSD"
        else
            WARN_THRESH="$UNRAID_DISK_HOT"
            CRIT_THRESH="$UNRAID_DISK_MAX"
            DRIVE_TYPE="HDD"
        fi

        if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then
            error "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)"
            DRIVE_CRIT=true
        elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then
            warn "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)"
            DRIVE_WARN=true
        else
            log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE} ✅"
        fi
    fi

    # Power on hours — informational only
    POH=$(get_smart_attr "$drive" "Power_On_Hours")
    if [[ -n "$POH" ]]; then
        POH_DAYS=$(( POH / 24 ))
        log "$drive_name power on hours: $POH (${POH_DAYS} days)"
    fi

    # Classify drive
    if [[ "$DRIVE_CRIT" == true ]]; then
        DRIVES_CRIT+=("$drive_name")
    elif [[ "$DRIVE_WARN" == true ]]; then
        DRIVES_WARN+=("$drive_name")
    else
        DRIVES_OK+=("$drive_name")
    fi

    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
echo "$ICON_HOST    Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME    Duration:  $(format_duration $(( END - START )))"
echo ""
echo "  $ICON_SUCCESS Healthy:  ${#DRIVES_OK[@]}"
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning:  ${#DRIVES_WARN[@]} — ${DRIVES_WARN[*]}"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]} — ${DRIVES_CRIT[*]}"
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped:  ${#DRIVES_SKIP[@]} — ${DRIVES_SKIP[*]}"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no SMART data read"
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
    echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
    notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \
        "SMART Health" "warning"
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
    warn "Status: WARNING — ${DRIVES_WARN[*]}"
    notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \
        "SMART Health" "warning"
else
    log "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1
exit 0