#!/bin/bash
# ==============================================================================================
# ============================= SMART Health Monitor ===========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Drive SMART health monitoring for all drives on the system. Scheduled weekly
# (Sunday 7am). Queries live SMART attributes via smartctl — no persistent writes.
#
# Monitored per drive: overall SMART status (PASSED/FAILED), Reallocated_Sector_Ct
# (any > 0 is concerning), Current_Pending_Sector (any > 0 is concerning),
# Offline_Uncorrectable (any > 0 is critical), Temperature_Celsius vs thresholds,
# Power_On_Hours (informational). NVMe drives use different attribute names —
# detected and handled automatically. Silent when all drives pass.
#
# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg —
# unRAID's own configured values. Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT
# from master.conf if dynamix.cfg is not found.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Single Instance Lock
#   acquire_lock prevents concurrent runs — smartctl calls are slow.
#
# Per-Host Ignore List
#   detect_hosts() aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES.
#   Typically used to skip the boot USB flash drive (no meaningful SMART data).
#
# Automatic Drive Discovery
#   Scans /dev/sd* and /dev/nvme* on every run — no drive list to maintain.
#
# Dynamix Temperature Thresholds
#   Reads hot/max/hotssd/maxssd from dynamix.cfg so smart_health.sh and unRAID's
#   dashboard use the same thresholds. Falls back to master.conf values if not found.
#
# Notifications Validated
#   validate_unraid_cmd confirms smartctl and notify script are present before use.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master_host*.conf
#
#   HOST*_SMART_IGNORE_DRIVES
#       Drives skipped in SMART monitoring. Aliased by detect_hosts() →
#       SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive (sda).
#
# master.conf
#
#   SMART_TEMP_WARN
#       Fallback warn threshold in °C if dynamix.cfg not found. (default: 45)
#
#   SMART_TEMP_CRIT
#       Fallback critical threshold in °C if dynamix.cfg not found. (default: 55)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# smart_health.sh
#     Query SMART attributes for all drives. Notify on any concerning results.
#     Silent when all drives pass.
#
# smart_health.sh --dry-run
#     Show which drives would be checked. No smartctl queries, no notifications.
#
# smart_health.sh --status
#     Show configured ignore list and temperature thresholds. Then exit.
#
# smart_health.sh --log
#     Verbose per-drive attribute output during the run.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Monitor script — output is the point
SILENT_MODE=false

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# Validate smartctl — required for all drive checks
validate_unraid_cmd \
    "$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
    "--version" "smartmontools" \
    "smartctl" || {
        error "smartctl not found — install smartmontools"
        notify "SMART health check failed on $(hostname) — smartmontools not installed" \
            "SMART Health" "warning"
        exit 1
    }

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES
detect_hosts

# Load temperature thresholds from dynamix.cfg — unRAID's own settings
get_unraid_temp_thresholds

log "HDD warn: ${UNRAID_DISK_HOT}°C  crit: ${UNRAID_DISK_MAX}°C"
log "SSD warn: ${UNRAID_SSD_HOT}°C   crit: ${UNRAID_SSD_MAX}°C"
log "Ignore:   ${SMART_IGNORE_DRIVES[*]:-none}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST    Identity:      $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_SMART   HDD warn:      ${UNRAID_DISK_HOT}°C"
    echo "$ICON_SMART   HDD crit:      ${UNRAID_DISK_MAX}°C"
    echo "$ICON_SMART   SSD warn:      ${UNRAID_SSD_HOT}°C"
    echo "$ICON_SMART   SSD crit:      ${UNRAID_SSD_MAX}°C"
    echo "$ICON_SMART   Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
    echo "$ICON_GEAR    Dry Run:       $DRY_RUN"
    echo ""
    echo "━━━ Discovered Drives ━━━"
    for drive in /dev/sd? /dev/nvme?; do
        [[ ! -e "$drive" ]] && continue
        drive_name=$(basename "$drive")
        ignored=false
        for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
            [[ "$drive_name" == "$ignore" ]] && ignored=true && break
        done
        if [[ "$ignored" == true ]]; then
            echo "  $ICON_WARN $drive — ignored"
        else
            echo "  $ICON_SMART $drive — would check"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Extract a named SMART attribute value (column 10 — raw value)
get_smart_attr() {
    local drive="$1" attr="$2"
    smartctl -A "$drive" 2>/dev/null | \
        awk -v attr="$attr" '$2 == attr {print $10}'
}

# Get drive temperature — handles HDD (attribute) and NVMe (different output format)
get_drive_temp() {
    local drive="$1"
    local temp

    # Standard HDD SMART attribute
    temp=$(get_smart_attr "$drive" "Temperature_Celsius")
    [[ -n "$temp" ]] && echo "$temp" && return

    # NVMe — temperature in different section
    temp=$(smartctl -A "$drive" 2>/dev/null | \
        awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}')
    [[ -n "$temp" ]] && echo "$temp" && return

    # Fallback — any temperature line
    temp=$(smartctl -A "$drive" 2>/dev/null | \
        awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}')
    echo "${temp:-}"
}

# Detect if a drive is SSD/NVMe (rotational=0)
is_ssd() {
    local drive="$1"
    local dev_name
    dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
    local rotational="/sys/block/$(basename "$drive")/queue/rotational"
    [[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
    # NVMe is always SSD
    [[ "$drive" == *nvme* ]] && return 0
    return 1
}

# ==============================================================================================
# ━━━ SMART Health Check ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""

START=$(date +%s)
DRIVES_OK=()
DRIVES_WARN=()
DRIVES_CRIT=()
DRIVES_SKIP=()

for drive in /dev/sd? /dev/nvme?; do
    [[ ! -e "$drive" ]] && continue
    drive_name=$(basename "$drive")

    # Check ignore list
    ignored=false
    for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
        [[ "$drive_name" == "$ignore" ]] && ignored=true && break
    done

    if [[ "$ignored" == true ]]; then
        log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
        DRIVES_SKIP+=("$drive_name")
        continue
    fi

    echo "━━━ $ICON_SMART $drive_name ━━━"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would check $drive_name"
        echo ""
        continue
    fi

    # Check SMART support
    if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
        warn "$drive_name — SMART not enabled or not supported — skipping"
        DRIVES_SKIP+=("$drive_name")
        echo ""
        continue
    fi

    DRIVE_WARN=false
    DRIVE_CRIT=false

    # Overall SMART status
    SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \
        grep "overall-health" | awk '{print $NF}')
    case "${SMART_STATUS:-}" in
        PASSED)
            log "$drive_name overall status: PASSED" ;;
        FAILED*)
            error "$drive_name overall status: FAILED — drive may be failing"
            DRIVE_CRIT=true ;;
        "")
            warn "$drive_name overall status: unknown — could not read SMART data" ;;
        *)
            warn "$drive_name overall status: $SMART_STATUS" ;;
    esac

    # Reallocated sectors
    REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
    if [[ -n "$REALLOC" ]]; then
        if [[ "$REALLOC" -gt 0 ]]; then
            warn "$ICON_SMART $drive_name — Reallocated sectors: $REALLOC (drive showing wear)"
            DRIVE_WARN=true
        else
            log "$drive_name reallocated sectors: 0 ✅"
        fi
    fi

    # Pending sectors
    PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
    if [[ -n "$PENDING" ]]; then
        if [[ "$PENDING" -gt 0 ]]; then
            warn "$ICON_SMART $drive_name — Pending sectors: $PENDING (awaiting reallocation)"
            DRIVE_WARN=true
        else
            log "$drive_name pending sectors: 0 ✅"
        fi
    fi

    # Uncorrectable sectors — critical threshold
    UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
    if [[ -n "$UNCORR" ]]; then
        if [[ "$UNCORR" -gt 0 ]]; then
            error "$ICON_SMART $drive_name — Uncorrectable sectors: $UNCORR — CRITICAL"
            DRIVE_CRIT=true
        else
            log "$drive_name uncorrectable sectors: 0 ✅"
        fi
    fi

    # Temperature — use SSD/HDD thresholds from dynamix.cfg
    TEMP=$(get_drive_temp "$drive")
    if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then
        if is_ssd "$drive"; then
            WARN_THRESH="$UNRAID_SSD_HOT"
            CRIT_THRESH="$UNRAID_SSD_MAX"
            DRIVE_TYPE="SSD"
        else
            WARN_THRESH="$UNRAID_DISK_HOT"
            CRIT_THRESH="$UNRAID_DISK_MAX"
            DRIVE_TYPE="HDD"
        fi

        if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then
            error "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)"
            DRIVE_CRIT=true
        elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then
            warn "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)"
            DRIVE_WARN=true
        else
            log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE} ✅"
        fi
    fi

    # Power on hours — informational only
    POH=$(get_smart_attr "$drive" "Power_On_Hours")
    if [[ -n "$POH" ]]; then
        POH_DAYS=$(( POH / 24 ))
        log "$drive_name power on hours: $POH (${POH_DAYS} days)"
    fi

    # Classify drive
    if [[ "$DRIVE_CRIT" == true ]]; then
        DRIVES_CRIT+=("$drive_name")
    elif [[ "$DRIVE_WARN" == true ]]; then
        DRIVES_WARN+=("$drive_name")
    else
        DRIVES_OK+=("$drive_name")
    fi

    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
echo "$ICON_HOST    Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME    Duration:  $(format_duration $(( END - START )))"
echo ""
echo "  $ICON_SUCCESS Healthy:  ${#DRIVES_OK[@]}"
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning:  ${#DRIVES_WARN[@]} — ${DRIVES_WARN[*]}"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]} — ${DRIVES_CRIT[*]}"
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped:  ${#DRIVES_SKIP[@]} — ${DRIVES_SKIP[*]}"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no SMART data read"
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
    echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
    notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \
        "SMART Health" "warning"
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
    warn "Status: WARNING — ${DRIVES_WARN[*]}"
    notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \
        "SMART Health" "warning"
else
    log "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1
exit 0