massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2

This commit is contained in:
2026-05-03 17:16:49 -04:00
parent 2691a35e80
commit ec7de648dc
72 changed files with 25640 additions and 14629 deletions
+195 -95
View File
@@ -1,35 +1,67 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- SMART Health Monitor ---------------------------------------
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= SMART Health Monitor =======================================
# ==============================================================================================
# Checks SMART health attributes for all drives on the system.
# Reads data live from each drive via smartctl — no persistent writes.
# Designed to run weekly as a scheduled report.
#
# Monitored attributes:
# Reallocated_Sector_Ctbad sectors remapped — any > 0 is concerning
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
# Temperature_Celsius — drive temperature vs SMART_TEMP_WARN / SMART_TEMP_CRIT
# Power_On_Hours — informational — drive age estimation
# SMART overall status — pass/fail per drive
# ── MONITORED ATTRIBUTES ──────────────────────────────────────────────────────────────────────
# Overall SMART status PASSED/FAILED — immediate fail = drive is dying
# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
# Temperature_Celsius — vs thresholds from dynamix.cfg (or master.conf fallback)
# Power_On_Hours — informational — drive age in days
#
# Discovers drives automatically — no configuration needed for drive list.
# SMART_IGNORE_DRIVES allows skipping specific drives (e.g. USB flash drives).
# ── DRIVE DISCOVERY ───────────────────────────────────────────────────────────────────────────
# Discovers drives automatically via /dev/sd* and /dev/nvme* — no config needed.
# NVMe drives use different attribute names — detected and handled automatically.
# HOST*_SMART_IGNORE_DRIVES skips specific drives (e.g. boot USB flash drive).
#
# All configuration in Master.conf under SMART Health section.
# Supports --dry-run to show which drives would be checked without running smartctl.
# -----------------------------------------------------------------------------------------------
# ── TEMPERATURE THRESHOLDS ────────────────────────────────────────────────────────────────────
# Reads hot/max/hotssd/maxssd from /boot/config/plugins/dynamix/dynamix.cfg at runtime.
# Uses unRAID's own configured thresholds — no need to duplicate them here.
# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if dynamix.cfg not found.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES.
# Each server monitors its own drives with its own ignore list.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
# acquire_lock — smartctl calls are slow, prevent duplicate runs
# detect_hosts() — correct ignore list per host via MY_ID aliases
# validate_unraid_cmd — smartctl and notify validated before use
# Silent healthy drives — only problems produce output
# Silent healthy run — no notify when all drives pass
#
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
# HOST*_SMART_IGNORE_DRIVES — drives skipped in SMART monitoring
# Aliased by detect_hosts() — script uses SMART_IGNORE_DRIVES
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# SMART_TEMP_WARN — fallback warn threshold in °C (if dynamix.cfg not found)
# SMART_TEMP_CRIT — fallback crit threshold in °C (if dynamix.cfg not found)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# smart_health.sh — normal run
# smart_health.sh --dry-run — show which drives would be checked
# smart_health.sh --log — verbose output
# smart_health.sh --status — show config and exit
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
source "$SCRIPT_DIR/../load_config.sh"
# Monitor script — output is the point
SILENT_MODE=false
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
@@ -38,35 +70,56 @@ if [[ "$EUID" -ne 0 ]]; then
exit 1
fi
success "Running as root"
# Validate smartctl — required for all drive checks
validate_unraid_cmd \
"$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
"--version" "smartmontools" \
"smartctl" || {
error "smartctl not found — install smartmontools"
notify "SMART health check failed on $(hostname) — smartmontools not installed" \
"SMART Health" "warning"
exit 1
}
if ! command -v smartctl >/dev/null 2>&1; then
error "smartctl not found — install smartmontools"
notify "SMART health check failed on $(hostname) — smartmontools not installed" "SMART Health" "warning"
exit 1
fi
validate_unraid_cmd \
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
"" "" \
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
success "smartctl available"
acquire_lock
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES
detect_hosts
# Load temperature thresholds from dynamix.cfg — unRAID's own settings
get_unraid_temp_thresholds
log "HDD warn: ${UNRAID_DISK_HOT}°C crit: ${UNRAID_DISK_MAX}°C"
log "SSD warn: ${UNRAID_SSD_HOT}°C crit: ${UNRAID_SSD_MAX}°C"
log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_SMART Temp warn: ${SMART_TEMP_WARN}°C"
echo "$ICON_SMART Temp crit: ${SMART_TEMP_CRIT}°C"
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SMART HDD warn: ${UNRAID_DISK_HOT}°C"
echo "$ICON_SMART HDD crit: ${UNRAID_DISK_MAX}°C"
echo "$ICON_SMART SSD warn: ${UNRAID_SSD_HOT}°C"
echo "$ICON_SMART SSD crit: ${UNRAID_SSD_MAX}°C"
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo ""
# Show drives that would be checked
echo "━━━ Discovered Drives ━━━"
for drive in /dev/sd? /dev/nvme?; do
[[ ! -e "$drive" ]] && continue
drive_name=$(basename "$drive")
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
@@ -79,23 +132,55 @@ if [[ "$SHOW_STATUS" == true ]]; then
exit 0
fi
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# -----------------------------------------------------------------------------------------------
# HELPER — extract SMART attribute value
# Usage: get_smart_attr "/dev/sda" "Reallocated_Sector_Ct"
# -----------------------------------------------------------------------------------------------
# Extract a named SMART attribute value (column 10 — raw value)
get_smart_attr() {
local drive="$1" attr="$2"
smartctl -A "$drive" 2>/dev/null | \
awk -v attr="$attr" '$2 == attr {print $10}'
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SMART SMART Health Check ━━━
# -----------------------------------------------------------------------------------------------
# Get drive temperature — handles HDD (attribute) and NVMe (different output format)
get_drive_temp() {
local drive="$1"
local temp
# Standard HDD SMART attribute
temp=$(get_smart_attr "$drive" "Temperature_Celsius")
[[ -n "$temp" ]] && echo "$temp" && return
# NVMe — temperature in different section
temp=$(smartctl -A "$drive" 2>/dev/null | \
awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}')
[[ -n "$temp" ]] && echo "$temp" && return
# Fallback — any temperature line
temp=$(smartctl -A "$drive" 2>/dev/null | \
awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}')
echo "${temp:-}"
}
# Detect if a drive is SSD/NVMe (rotational=0)
is_ssd() {
local drive="$1"
local dev_name
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
# NVMe is always SSD
[[ "$drive" == *nvme* ]] && return 0
return 1
}
# ==============================================================================================
# ━━━ SMART Health Check ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""
START=$(date +%s)
@@ -110,12 +195,12 @@ for drive in /dev/sd? /dev/nvme?; do
# Check ignore list
ignored=false
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
done
if [[ "$ignored" == true ]]; then
info "$drive_name — ignored (in SMART_IGNORE_DRIVES)"
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
DRIVES_SKIP+=("$drive_name")
continue
fi
@@ -128,34 +213,40 @@ for drive in /dev/sd? /dev/nvme?; do
continue
fi
# Check if drive supports SMART
# Check SMART support
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
warn "$drive_name — SMART not enabled or not supported"
warn "$drive_name — SMART not enabled or not supported — skipping"
DRIVES_SKIP+=("$drive_name")
echo ""
continue
fi
# Overall SMART status
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | grep "overall-health" | awk '{print $NF}')
if [[ "$SMART_STATUS" == "PASSED" ]]; then
success "Overall status: PASSED"
else
error "Overall status: $SMART_STATUS"
fi
# Key attributes
DRIVE_WARN=false
DRIVE_CRIT=false
# Overall SMART status
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \
grep "overall-health" | awk '{print $NF}')
case "${SMART_STATUS:-}" in
PASSED)
log "$drive_name overall status: PASSED" ;;
FAILED*)
error "$drive_name overall status: FAILED — drive may be failing"
DRIVE_CRIT=true ;;
"")
warn "$drive_name overall status: unknown — could not read SMART data" ;;
*)
warn "$drive_name overall status: $SMART_STATUS" ;;
esac
# Reallocated sectors
REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
if [[ -n "$REALLOC" ]]; then
if [[ "$REALLOC" -gt 0 ]]; then
warn "$ICON_SMART Reallocated sectors: $REALLOC drive showing wear"
warn "$ICON_SMART $drive_name Reallocated sectors: $REALLOC (drive showing wear)"
DRIVE_WARN=true
else
success "$ICON_SMART Reallocated sectors: $REALLOC"
log "$drive_name reallocated sectors: 0 ✅"
fi
fi
@@ -163,47 +254,53 @@ for drive in /dev/sd? /dev/nvme?; do
PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
if [[ -n "$PENDING" ]]; then
if [[ "$PENDING" -gt 0 ]]; then
warn "$ICON_SMART Pending sectors: $PENDING — sectors awaiting reallocation"
warn "$ICON_SMART $drive_name Pending sectors: $PENDING (awaiting reallocation)"
DRIVE_WARN=true
else
success "$ICON_SMART Pending sectors: $PENDING"
log "$drive_name pending sectors: 0 ✅"
fi
fi
# Uncorrectable sectors
# Uncorrectable sectors — critical threshold
UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
if [[ -n "$UNCORR" ]]; then
if [[ "$UNCORR" -gt 0 ]]; then
error "$ICON_SMART Uncorrectable sectors: $UNCORR — CRITICAL"
error "$ICON_SMART $drive_name Uncorrectable sectors: $UNCORR — CRITICAL"
DRIVE_CRIT=true
else
success "$ICON_SMART Uncorrectable sectors: $UNCORR"
log "$drive_name uncorrectable sectors: 0 ✅"
fi
fi
# Temperature
TEMP=$(get_smart_attr "$drive" "Temperature_Celsius")
# NVMe uses different attribute name
[[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$drive" 2>/dev/null | \
awk '/Temperature/{print $2}' | head -1)
# Temperature — use SSD/HDD thresholds from dynamix.cfg
TEMP=$(get_drive_temp "$drive")
if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then
if is_ssd "$drive"; then
WARN_THRESH="$UNRAID_SSD_HOT"
CRIT_THRESH="$UNRAID_SSD_MAX"
DRIVE_TYPE="SSD"
else
WARN_THRESH="$UNRAID_DISK_HOT"
CRIT_THRESH="$UNRAID_DISK_MAX"
DRIVE_TYPE="HDD"
fi
if [[ -n "$TEMP" ]]; then
if [[ "$TEMP" -ge "$SMART_TEMP_CRIT" ]]; then
error "$ICON_SMART Temperature: ${TEMP}°C — CRITICAL (threshold: ${SMART_TEMP_CRIT}°C)"
if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then
error "$ICON_SMART $drive_name${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)"
DRIVE_CRIT=true
elif [[ "$TEMP" -ge "$SMART_TEMP_WARN" ]]; then
warn "$ICON_SMART Temperature: ${TEMP}°C — warning (threshold: ${SMART_TEMP_WARN}°C)"
elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then
warn "$ICON_SMART $drive_name${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)"
DRIVE_WARN=true
else
success "$ICON_SMART Temperature: ${TEMP}°C"
log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE}"
fi
fi
# Power on hours — informational
# Power on hours — informational only
POH=$(get_smart_attr "$drive" "Power_On_Hours")
if [[ -n "$POH" ]]; then
POH_DAYS=$(( POH / 24 ))
info "$ICON_SMART Power on hours: $POH (${POH_DAYS} days)"
log "$drive_name power on hours: $POH (${POH_DAYS} days)"
fi
# Classify drive
@@ -220,30 +317,33 @@ done
END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo ""
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]} $ICON_WARN Warning: ${#DRIVES_WARN[@]} $ICON_ERROR Critical: ${#DRIVES_CRIT[@]} skipped: ${#DRIVES_SKIP[@]}"
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]}"
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning: ${#DRIVES_WARN[@]}${DRIVES_WARN[*]}"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]}${DRIVES_CRIT[*]}"
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped: ${#DRIVES_SKIP[@]}${DRIVES_SKIP[*]}"
echo ""
[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo " $ICON_SUCCESS ${DRIVES_OK[*]}"
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && echo " $ICON_WARN ${DRIVES_WARN[*]}"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo " $ICON_ERROR ${DRIVES_CRIT[*]}"
echo ""
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN"
warn "DRY RUN — no SMART data read"
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
notify "SMART CRITICAL on $(hostname) drives need immediate attention: ${DRIVES_CRIT[*]}" "SMART Health" "warning"
notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \
"SMART Health" "warning"
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
echo "$ICON_WARN Status: WARNING — ${DRIVES_WARN[*]}"
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" "SMART Health" "warning"
warn "Status: WARNING — ${DRIVES_WARN[*]}"
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \
"SMART Health" "warning"
else
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DRIVES HEALTHY"
notify "SMART health check passed on $(hostname)${#DRIVES_OK[@]} drives healthy" "SMART Health" "normal"
log "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1
exit 0