Gmer4Lfe
95151c2278
Move all watchdog scripts to a dedicated Watchdogs/ folder: Docker_Essentials/docker_watchdog.sh → Watchdogs/ unRAID_Essentials/system_watchdog.sh → Watchdogs/ unRAID_Essentials/resource_watchdog.sh → Watchdogs/ Orchestrators/watchdog_orchestrator.sh → Watchdogs/ Tools/watchdog_skip_list_manager.sh → Watchdogs/ Rename host config files: master_host1.conf → host1.conf master_host2.conf → host2.conf Update all references across the ecosystem: master.conf: WATCHDOG_ORCHESTRATOR_SCRIPTS paths → Watchdogs/ load_config.sh: host*.conf glob + all comments git_pull_execute.sh: sparse checkout glob + all comments Partnership/ssh_setup.sh: HOST_CONF path construction user_script_plug-in.sh: all script paths + per-host conf path common.sh, README.md, README-User_Script_Plug-in.md: comment refs All Partnership, Fallback, Monitors, Transcodes, Tools scripts: comment refs
372 lines
14 KiB
Bash
372 lines
14 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= SMART Health Monitor ===========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Drive SMART health monitoring for all drives on the system. Scheduled weekly
|
|
# (Sunday 7am). Queries live SMART attributes via smartctl — no persistent writes.
|
|
#
|
|
# Monitored per drive: overall SMART status (PASSED/FAILED), Reallocated_Sector_Ct
|
|
# (any > 0 is concerning), Current_Pending_Sector (any > 0 is concerning),
|
|
# Offline_Uncorrectable (any > 0 is critical), Temperature_Celsius vs thresholds,
|
|
# Power_On_Hours (informational). NVMe drives use different attribute names —
|
|
# detected and handled automatically. Silent when all drives pass.
|
|
#
|
|
# Temperature thresholds read from /boot/config/plugins/dynamix/dynamix.cfg —
|
|
# unRAID's own configured values. Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT
|
|
# from master.conf if dynamix.cfg is not found.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent runs — smartctl calls are slow.
|
|
#
|
|
# Per-Host Ignore List
|
|
# detect_hosts() aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES.
|
|
# Typically used to skip the boot USB flash drive (no meaningful SMART data).
|
|
#
|
|
# Automatic Drive Discovery
|
|
# Scans /dev/sd* and /dev/nvme* on every run — no drive list to maintain.
|
|
#
|
|
# Dynamix Temperature Thresholds
|
|
# Reads hot/max/hotssd/maxssd from dynamix.cfg so smart_health.sh and unRAID's
|
|
# dashboard use the same thresholds. Falls back to master.conf values if not found.
|
|
#
|
|
# Notifications Validated
|
|
# validate_unraid_cmd confirms smartctl and notify script are present before use.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_SMART_IGNORE_DRIVES
|
|
# Drives skipped in SMART monitoring. Aliased by detect_hosts() →
|
|
# SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive (sda).
|
|
#
|
|
# master.conf
|
|
#
|
|
# SMART_TEMP_WARN
|
|
# Fallback warn threshold in °C if dynamix.cfg not found. (default: 45)
|
|
#
|
|
# SMART_TEMP_CRIT
|
|
# Fallback critical threshold in °C if dynamix.cfg not found. (default: 55)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# smart_health.sh
|
|
# Query SMART attributes for all drives. Notify on any concerning results.
|
|
# Silent when all drives pass.
|
|
#
|
|
# smart_health.sh --dry-run
|
|
# Show which drives would be checked. No smartctl queries, no notifications.
|
|
#
|
|
# smart_health.sh --status
|
|
# Show configured ignore list and temperature thresholds. Then exit.
|
|
#
|
|
# smart_health.sh --log
|
|
# Verbose per-drive attribute output during the run.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
# Validate smartctl — required for all drive checks
|
|
validate_unraid_cmd \
|
|
"$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
|
|
"--version" "smartmontools" \
|
|
"smartctl" || {
|
|
error "smartctl not found — install smartmontools"
|
|
notify "SMART health check failed on $(hostname) — smartmontools not installed" \
|
|
"SMART Health" "warning"
|
|
exit 1
|
|
}
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES
|
|
detect_hosts
|
|
|
|
# Load temperature thresholds from dynamix.cfg — unRAID's own settings
|
|
get_unraid_temp_thresholds
|
|
|
|
log "HDD warn: ${UNRAID_DISK_HOT}°C crit: ${UNRAID_DISK_MAX}°C"
|
|
log "SSD warn: ${UNRAID_SSD_HOT}°C crit: ${UNRAID_SSD_MAX}°C"
|
|
log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_SMART HDD warn: ${UNRAID_DISK_HOT}°C"
|
|
echo "$ICON_SMART HDD crit: ${UNRAID_DISK_MAX}°C"
|
|
echo "$ICON_SMART SSD warn: ${UNRAID_SSD_HOT}°C"
|
|
echo "$ICON_SMART SSD crit: ${UNRAID_SSD_MAX}°C"
|
|
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo ""
|
|
echo "━━━ Discovered Drives ━━━"
|
|
for drive in /dev/sd? /dev/nvme?; do
|
|
[[ ! -e "$drive" ]] && continue
|
|
drive_name=$(basename "$drive")
|
|
ignored=false
|
|
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
|
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
|
done
|
|
if [[ "$ignored" == true ]]; then
|
|
echo " $ICON_WARN $drive — ignored"
|
|
else
|
|
echo " $ICON_SMART $drive — would check"
|
|
fi
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# Extract a named SMART attribute value (column 10 — raw value)
|
|
get_smart_attr() {
|
|
local drive="$1" attr="$2"
|
|
smartctl -A "$drive" 2>/dev/null | \
|
|
awk -v attr="$attr" '$2 == attr {print $10}'
|
|
}
|
|
|
|
# Get drive temperature — handles HDD (attribute) and NVMe (different output format)
|
|
get_drive_temp() {
|
|
local drive="$1"
|
|
local temp
|
|
|
|
# Standard HDD SMART attribute
|
|
temp=$(get_smart_attr "$drive" "Temperature_Celsius")
|
|
[[ -n "$temp" ]] && echo "$temp" && return
|
|
|
|
# NVMe — temperature in different section
|
|
temp=$(smartctl -A "$drive" 2>/dev/null | \
|
|
awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}')
|
|
[[ -n "$temp" ]] && echo "$temp" && return
|
|
|
|
# Fallback — any temperature line
|
|
temp=$(smartctl -A "$drive" 2>/dev/null | \
|
|
awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}')
|
|
echo "${temp:-}"
|
|
}
|
|
|
|
# Detect if a drive is SSD/NVMe (rotational=0)
|
|
is_ssd() {
|
|
local drive="$1"
|
|
local dev_name
|
|
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
|
|
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
|
|
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
|
|
# NVMe is always SSD
|
|
[[ "$drive" == *nvme* ]] && return 0
|
|
return 1
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ SMART Health Check ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
DRIVES_OK=()
|
|
DRIVES_WARN=()
|
|
DRIVES_CRIT=()
|
|
DRIVES_SKIP=()
|
|
|
|
for drive in /dev/sd? /dev/nvme?; do
|
|
[[ ! -e "$drive" ]] && continue
|
|
drive_name=$(basename "$drive")
|
|
|
|
# Check ignore list
|
|
ignored=false
|
|
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
|
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
|
done
|
|
|
|
if [[ "$ignored" == true ]]; then
|
|
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
|
|
DRIVES_SKIP+=("$drive_name")
|
|
continue
|
|
fi
|
|
|
|
echo "━━━ $ICON_SMART $drive_name ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would check $drive_name"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
# Check SMART support
|
|
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
|
warn "$drive_name — SMART not enabled or not supported — skipping"
|
|
DRIVES_SKIP+=("$drive_name")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
DRIVE_WARN=false
|
|
DRIVE_CRIT=false
|
|
|
|
# Overall SMART status
|
|
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \
|
|
grep "overall-health" | awk '{print $NF}')
|
|
case "${SMART_STATUS:-}" in
|
|
PASSED)
|
|
log "$drive_name overall status: PASSED" ;;
|
|
FAILED*)
|
|
error "$drive_name overall status: FAILED — drive may be failing"
|
|
DRIVE_CRIT=true ;;
|
|
"")
|
|
warn "$drive_name overall status: unknown — could not read SMART data" ;;
|
|
*)
|
|
warn "$drive_name overall status: $SMART_STATUS" ;;
|
|
esac
|
|
|
|
# Reallocated sectors
|
|
REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
|
|
if [[ -n "$REALLOC" ]]; then
|
|
if [[ "$REALLOC" -gt 0 ]]; then
|
|
warn "$ICON_SMART $drive_name — Reallocated sectors: $REALLOC (drive showing wear)"
|
|
DRIVE_WARN=true
|
|
else
|
|
log "$drive_name reallocated sectors: 0 ✅"
|
|
fi
|
|
fi
|
|
|
|
# Pending sectors
|
|
PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
|
|
if [[ -n "$PENDING" ]]; then
|
|
if [[ "$PENDING" -gt 0 ]]; then
|
|
warn "$ICON_SMART $drive_name — Pending sectors: $PENDING (awaiting reallocation)"
|
|
DRIVE_WARN=true
|
|
else
|
|
log "$drive_name pending sectors: 0 ✅"
|
|
fi
|
|
fi
|
|
|
|
# Uncorrectable sectors — critical threshold
|
|
UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
|
|
if [[ -n "$UNCORR" ]]; then
|
|
if [[ "$UNCORR" -gt 0 ]]; then
|
|
error "$ICON_SMART $drive_name — Uncorrectable sectors: $UNCORR — CRITICAL"
|
|
DRIVE_CRIT=true
|
|
else
|
|
log "$drive_name uncorrectable sectors: 0 ✅"
|
|
fi
|
|
fi
|
|
|
|
# Temperature — use SSD/HDD thresholds from dynamix.cfg
|
|
TEMP=$(get_drive_temp "$drive")
|
|
if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then
|
|
if is_ssd "$drive"; then
|
|
WARN_THRESH="$UNRAID_SSD_HOT"
|
|
CRIT_THRESH="$UNRAID_SSD_MAX"
|
|
DRIVE_TYPE="SSD"
|
|
else
|
|
WARN_THRESH="$UNRAID_DISK_HOT"
|
|
CRIT_THRESH="$UNRAID_DISK_MAX"
|
|
DRIVE_TYPE="HDD"
|
|
fi
|
|
|
|
if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then
|
|
error "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)"
|
|
DRIVE_CRIT=true
|
|
elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then
|
|
warn "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)"
|
|
DRIVE_WARN=true
|
|
else
|
|
log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE} ✅"
|
|
fi
|
|
fi
|
|
|
|
# Power on hours — informational only
|
|
POH=$(get_smart_attr "$drive" "Power_On_Hours")
|
|
if [[ -n "$POH" ]]; then
|
|
POH_DAYS=$(( POH / 24 ))
|
|
log "$drive_name power on hours: $POH (${POH_DAYS} days)"
|
|
fi
|
|
|
|
# Classify drive
|
|
if [[ "$DRIVE_CRIT" == true ]]; then
|
|
DRIVES_CRIT+=("$drive_name")
|
|
elif [[ "$DRIVE_WARN" == true ]]; then
|
|
DRIVES_WARN+=("$drive_name")
|
|
else
|
|
DRIVES_OK+=("$drive_name")
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]}"
|
|
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning: ${#DRIVES_WARN[@]} — ${DRIVES_WARN[*]}"
|
|
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]} — ${DRIVES_CRIT[*]}"
|
|
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped: ${#DRIVES_SKIP[@]} — ${DRIVES_SKIP[*]}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no SMART data read"
|
|
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
|
|
notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \
|
|
"SMART Health" "warning"
|
|
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
|
|
warn "Status: WARNING — ${DRIVES_WARN[*]}"
|
|
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \
|
|
"SMART Health" "warning"
|
|
else
|
|
echo "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1
|
|
exit 0 |