From bdcdfea175dfb3603c053d326c8a92fb2cc902fd Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Fri, 5 Jun 2026 19:53:15 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20build=20smart=5Flong=5Ftest.sh=20?= =?UTF-8?q?=E2=80=94=20SMART=20extended=20self-test=20for=20monthly=20main?= =?UTF-8?q?tenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests all drives sequentially (one at a time to avoid I/O saturation). Polls every 60s until complete. Notifies on failure. Respects SMART_IGNORE_DRIVES. Enable by uncommenting in MONTHLY_MAINTENANCE_SCRIPTS in master.conf. zfs_pool_scrub.sh was already fully implemented — also uncommented locally. --- Tools/smart_long_test.sh | 302 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100755 Tools/smart_long_test.sh diff --git a/Tools/smart_long_test.sh b/Tools/smart_long_test.sh new file mode 100755 index 0000000..856cc0b --- /dev/null +++ b/Tools/smart_long_test.sh @@ -0,0 +1,302 @@ +#!/bin/bash +# ============================================================================================== +# ============================= SMART Extended Self-Test ======================================= +# ============================================================================================== +# +# PURPOSE +# ───────────────────────────────────────────────────────────────────────────── +# Runs a SMART extended (long) self-test on all drives (or a specific drive), +# waits for completion, and reports results. Extended tests do a full read-scan +# of every sector — catches bad sectors that the short test skips. Monthly cadence +# via the monthly maintenance orchestrator. +# +# NVMe drives are included — smartctl supports NVMe self-test via the same +# -t long interface (NVMe 1.3+ specification). +# +# ============================================================================================== +# OPERATIONAL MODEL +# ============================================================================================== +# +# Starts long self-tests on all drives sequentially (NOT in parallel) to avoid +# saturating I/O. One drive at a time: start, wait for completion, move to next. +# Sequential order keeps test duration predictable and avoids thermal stacking. +# +# Drives already running a self-test are skipped. If a previous test did not +# finish (interrupted mid-way), it is reported and the drive is re-tested. +# +# Each test is polled every 60 seconds. Typical durations: +# HDD 2–4 TB : 60–120 min +# HDD 8–12 TB: 90–180 min +# SSD any : 5–15 min +# NVMe any : 5–10 min +# +# ============================================================================================== +# OPERATIONAL SAFEGUARDS +# ============================================================================================== +# +# Single Instance Lock +# acquire_lock prevents concurrent test runs — one long-test pass per server. +# +# Sequential Execution +# Drives tested one at a time. Parallel long-tests thrash I/O and inflate temps. +# +# SIGTERM Trap +# Poll loop exits cleanly on signal. Tests continue running in drive firmware. +# +# Tool Validation +# platform_require_cmd confirms smartctl is present before use. +# +# Silent When Clean +# Only failures and warnings produce notifications. +# +# ============================================================================================== +# CONFIGURATION +# ============================================================================================== +# +# host*.conf +# +# HOST*_SMART_IGNORE_DRIVES +# Drives skipped in SMART testing. Aliased by detect_hosts() → +# SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive. +# +# master.conf +# +# SMART_TEMP_WARN / SMART_TEMP_CRIT +# Fallback thresholds used for post-test temperature reporting if +# dynamix.cfg is not found. +# +# ============================================================================================== +# RUNTIME MODES +# ============================================================================================== +# +# smart_long_test.sh +# Run extended self-test on all drives not in SMART_IGNORE_DRIVES. +# +# smart_long_test.sh /dev/sda +# Run extended self-test on a specific drive. Bypasses the ignore list. +# +# smart_long_test.sh --status +# Show last self-test result for all drives and exit. +# +# smart_long_test.sh --dry-run +# Show which drives would be tested. No tests started. +# +# smart_long_test.sh --log +# Verbose progress output during polling. +# +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../load_config.sh" + +parse_args "$@" + +TARGET_DRIVE="${PARSED_ARGS[0]:-}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +platform_require_cmd \ + "$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \ + "--version" "smartmontools" \ + "smartctl" || { + error "smartctl not found — install smartmontools" + notify "SMART long test failed on $(hostname) — smartmontools not installed" \ + "SMART Long Test" "warning" + exit 1 + } + +platform_require_cmd \ + "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ + "" "" \ + "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" + +acquire_lock + +detect_hosts +get_unraid_temp_thresholds + +log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" +log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no tests will be started" + +trap 'warn "SMART long test script interrupted — tests continue in drive firmware"; exit 0' \ + SIGTERM SIGINT + +# ============================================================================================== +# ── Build drive list ────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +_build_drive_list() { + local drives=() + if [[ -n "$TARGET_DRIVE" ]]; then + if [[ ! -e "$TARGET_DRIVE" ]]; then + error "Drive not found: $TARGET_DRIVE" + exit 1 + fi + drives=("$TARGET_DRIVE") + else + for drive in /dev/sd? /dev/nvme?; do + [[ ! -e "$drive" ]] && continue + local drive_name + drive_name=$(basename "$drive") + local ignored=false + for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do + [[ "$drive_name" == "$ignore" ]] && ignored=true && break + done + [[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; } + if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then + log "Skipping $drive_name — SMART not enabled" + continue + fi + drives+=("$drive") + done + fi + echo "${drives[@]}" +} + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY SMART LONG TEST STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + for drive in /dev/sd? /dev/nvme?; do + [[ ! -e "$drive" ]] && continue + drive_name=$(basename "$drive") + local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \ + grep -m1 "Extended" | awk '{print $NF}') + ignored=false + for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do + [[ "$drive_name" == "$ignore" ]] && ignored=true && break + done + if [[ "$ignored" == true ]]; then + echo " $ICON_WARN $drive_name — ignored" + else + echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}" + fi + done + echo "" + echo " Ignored: ${SMART_IGNORE_DRIVES[*]:-none}" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Run Extended Tests — Sequential ━━━ +# ============================================================================================== +read -r -a DRIVES_TO_TEST <<< "$(_build_drive_list)" + +if [[ ${#DRIVES_TO_TEST[@]} -eq 0 ]]; then + warn "No drives to test — all may be on ignore list or SMART not enabled" + exit 0 +fi + +echo "" +echo "━━━ $ICON_SMART SMART Extended Self-Test — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST ${#DRIVES_TO_TEST[@]} drive(s) to test — running sequentially" +echo "" + +WINDOW_START=$(date +%s) +DRIVES_OK=() +DRIVES_FAIL=() +DRIVES_SKIP=() + +for drive in "${DRIVES_TO_TEST[@]}"; do + drive_name=$(basename "$drive") + echo "━━━ $ICON_SMART $drive_name ━━━" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would run extended self-test on $drive_name" + echo "" + continue + fi + + # Check if test already in progress + if smartctl -l selftest "$drive" 2>/dev/null | grep -q "Self-test routine in progress"; then + warn "$drive_name — self-test already in progress — skipping" + DRIVES_SKIP+=("$drive_name") + echo "" + continue + fi + + # Start extended test + if ! smartctl -t long "$drive" >/dev/null 2>&1; then + error "$drive_name — failed to start extended self-test" + DRIVES_FAIL+=("$drive_name") + echo "" + continue + fi + echo " Extended test started on $drive_name" + + # Poll until complete + DRIVE_START=$(date +%s) + while true; do + sleep 60 + + STATUS_LINE=$(smartctl -l selftest "$drive" 2>/dev/null | grep -m1 "Extended") + + if echo "$STATUS_LINE" | grep -q "Self-test routine in progress"; then + PCT=$(echo "$STATUS_LINE" | grep -oE "[0-9]+% of test remaining" || true) + log "$drive_name — test in progress ${PCT:+($PCT)}" + continue + fi + + # Test finished — determine result + DRIVE_ELAPSED=$(format_duration $(( $(date +%s) - DRIVE_START ))) + if echo "$STATUS_LINE" | grep -iq "Completed without error\|Successful"; then + echo " $ICON_DONE $drive_name — extended test passed ✅ ($DRIVE_ELAPSED)" + DRIVES_OK+=("$drive_name") + elif echo "$STATUS_LINE" | grep -iq "Failed\|failed"; then + FAIL_DETAIL=$(echo "$STATUS_LINE" | awk '{print $NF}') + error "$drive_name — extended test FAILED — $FAIL_DETAIL ($DRIVE_ELAPSED)" + DRIVES_FAIL+=("$drive_name") + elif [[ -z "$STATUS_LINE" ]]; then + warn "$drive_name — no test result found — may not support extended test" + DRIVES_SKIP+=("$drive_name") + else + warn "$drive_name — unexpected result: $STATUS_LINE ($DRIVE_ELAPSED)" + DRIVES_SKIP+=("$drive_name") + fi + echo "" + break + done +done + +WINDOW_END=$(date +%s) + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "━━━━━ $ICON_SUMMARY SMART LONG TEST SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" +echo "$ICON_SMART Drives: ${#DRIVES_TO_TEST[@]} tested" +[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${DRIVES_OK[*]}" +[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${DRIVES_FAIL[*]}" +[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${DRIVES_SKIP[*]}" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no tests started" +elif [[ ${#DRIVES_FAIL[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: FAILURES — ${DRIVES_FAIL[*]}" + notify "SMART long test failures on $(hostname) ($MY_ID) — drives: ${DRIVES_FAIL[*]}" \ + "SMART Long Test" "warning" +elif [[ ${#DRIVES_OK[@]} -gt 0 ]]; then + echo "$ICON_DONE Status: all ${#DRIVES_OK[@]} drive(s) passed ✅" +else + warn "Status: no results — all drives skipped" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && exit 1 +exit 0