feat: build smart_long_test.sh — SMART extended self-test for monthly maintenance
Tests all drives sequentially (one at a time to avoid I/O saturation). Polls every 60s until complete. Notifies on failure. Respects SMART_IGNORE_DRIVES. Enable by uncommenting in MONTHLY_MAINTENANCE_SCRIPTS in master.conf. zfs_pool_scrub.sh was already fully implemented — also uncommented locally.
This commit is contained in:
Gmer4Lfe
Executable
+302
@@ -0,0 +1,302 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ==============================================================================================
|
||||||
|
# ============================= SMART Extended Self-Test =======================================
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# PURPOSE
|
||||||
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Runs a SMART extended (long) self-test on all drives (or a specific drive),
|
||||||
|
# waits for completion, and reports results. Extended tests do a full read-scan
|
||||||
|
# of every sector — catches bad sectors that the short test skips. Monthly cadence
|
||||||
|
# via the monthly maintenance orchestrator.
|
||||||
|
#
|
||||||
|
# NVMe drives are included — smartctl supports NVMe self-test via the same
|
||||||
|
# -t long interface (NVMe 1.3+ specification).
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Starts long self-tests on all drives sequentially (NOT in parallel) to avoid
|
||||||
|
# saturating I/O. One drive at a time: start, wait for completion, move to next.
|
||||||
|
# Sequential order keeps test duration predictable and avoids thermal stacking.
|
||||||
|
#
|
||||||
|
# Drives already running a self-test are skipped. If a previous test did not
|
||||||
|
# finish (interrupted mid-way), it is reported and the drive is re-tested.
|
||||||
|
#
|
||||||
|
# Each test is polled every 60 seconds. Typical durations:
|
||||||
|
# HDD 2–4 TB : 60–120 min
|
||||||
|
# HDD 8–12 TB: 90–180 min
|
||||||
|
# SSD any : 5–15 min
|
||||||
|
# NVMe any : 5–10 min
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Single Instance Lock
|
||||||
|
# acquire_lock prevents concurrent test runs — one long-test pass per server.
|
||||||
|
#
|
||||||
|
# Sequential Execution
|
||||||
|
# Drives tested one at a time. Parallel long-tests thrash I/O and inflate temps.
|
||||||
|
#
|
||||||
|
# SIGTERM Trap
|
||||||
|
# Poll loop exits cleanly on signal. Tests continue running in drive firmware.
|
||||||
|
#
|
||||||
|
# Tool Validation
|
||||||
|
# platform_require_cmd confirms smartctl is present before use.
|
||||||
|
#
|
||||||
|
# Silent When Clean
|
||||||
|
# Only failures and warnings produce notifications.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# CONFIGURATION
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# host*.conf
|
||||||
|
#
|
||||||
|
# HOST*_SMART_IGNORE_DRIVES
|
||||||
|
# Drives skipped in SMART testing. Aliased by detect_hosts() →
|
||||||
|
# SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive.
|
||||||
|
#
|
||||||
|
# master.conf
|
||||||
|
#
|
||||||
|
# SMART_TEMP_WARN / SMART_TEMP_CRIT
|
||||||
|
# Fallback thresholds used for post-test temperature reporting if
|
||||||
|
# dynamix.cfg is not found.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# RUNTIME MODES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# smart_long_test.sh
|
||||||
|
# Run extended self-test on all drives not in SMART_IGNORE_DRIVES.
|
||||||
|
#
|
||||||
|
# smart_long_test.sh /dev/sda
|
||||||
|
# Run extended self-test on a specific drive. Bypasses the ignore list.
|
||||||
|
#
|
||||||
|
# smart_long_test.sh --status
|
||||||
|
# Show last self-test result for all drives and exit.
|
||||||
|
#
|
||||||
|
# smart_long_test.sh --dry-run
|
||||||
|
# Show which drives would be tested. No tests started.
|
||||||
|
#
|
||||||
|
# smart_long_test.sh --log
|
||||||
|
# Verbose progress output during polling.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
|
||||||
|
TARGET_DRIVE="${PARSED_ARGS[0]:-}"
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Setup ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
platform_require_cmd \
|
||||||
|
"$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
|
||||||
|
"--version" "smartmontools" \
|
||||||
|
"smartctl" || {
|
||||||
|
error "smartctl not found — install smartmontools"
|
||||||
|
notify "SMART long test failed on $(hostname) — smartmontools not installed" \
|
||||||
|
"SMART Long Test" "warning"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
platform_require_cmd \
|
||||||
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||||
|
"" "" \
|
||||||
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||||
|
|
||||||
|
acquire_lock
|
||||||
|
|
||||||
|
detect_hosts
|
||||||
|
get_unraid_temp_thresholds
|
||||||
|
|
||||||
|
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
|
log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||||||
|
|
||||||
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no tests will be started"
|
||||||
|
|
||||||
|
trap 'warn "SMART long test script interrupted — tests continue in drive firmware"; exit 0' \
|
||||||
|
SIGTERM SIGINT
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── Build drive list ──────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
_build_drive_list() {
|
||||||
|
local drives=()
|
||||||
|
if [[ -n "$TARGET_DRIVE" ]]; then
|
||||||
|
if [[ ! -e "$TARGET_DRIVE" ]]; then
|
||||||
|
error "Drive not found: $TARGET_DRIVE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
drives=("$TARGET_DRIVE")
|
||||||
|
else
|
||||||
|
for drive in /dev/sd? /dev/nvme?; do
|
||||||
|
[[ ! -e "$drive" ]] && continue
|
||||||
|
local drive_name
|
||||||
|
drive_name=$(basename "$drive")
|
||||||
|
local ignored=false
|
||||||
|
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||||||
|
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||||||
|
done
|
||||||
|
[[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
|
||||||
|
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
||||||
|
log "Skipping $drive_name — SMART not enabled"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
drives+=("$drive")
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
echo "${drives[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Status ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY SMART LONG TEST STATUS ━━━━━"
|
||||||
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
|
echo ""
|
||||||
|
for drive in /dev/sd? /dev/nvme?; do
|
||||||
|
[[ ! -e "$drive" ]] && continue
|
||||||
|
drive_name=$(basename "$drive")
|
||||||
|
local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
|
||||||
|
grep -m1 "Extended" | awk '{print $NF}')
|
||||||
|
ignored=false
|
||||||
|
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||||||
|
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||||||
|
done
|
||||||
|
if [[ "$ignored" == true ]]; then
|
||||||
|
echo " $ICON_WARN $drive_name — ignored"
|
||||||
|
else
|
||||||
|
echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
echo " Ignored: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Run Extended Tests — Sequential ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
read -r -a DRIVES_TO_TEST <<< "$(_build_drive_list)"
|
||||||
|
|
||||||
|
if [[ ${#DRIVES_TO_TEST[@]} -eq 0 ]]; then
|
||||||
|
warn "No drives to test — all may be on ignore list or SMART not enabled"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_SMART SMART Extended Self-Test — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
|
echo "$ICON_HOST ${#DRIVES_TO_TEST[@]} drive(s) to test — running sequentially"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
WINDOW_START=$(date +%s)
|
||||||
|
DRIVES_OK=()
|
||||||
|
DRIVES_FAIL=()
|
||||||
|
DRIVES_SKIP=()
|
||||||
|
|
||||||
|
for drive in "${DRIVES_TO_TEST[@]}"; do
|
||||||
|
drive_name=$(basename "$drive")
|
||||||
|
echo "━━━ $ICON_SMART $drive_name ━━━"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would run extended self-test on $drive_name"
|
||||||
|
echo ""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if test already in progress
|
||||||
|
if smartctl -l selftest "$drive" 2>/dev/null | grep -q "Self-test routine in progress"; then
|
||||||
|
warn "$drive_name — self-test already in progress — skipping"
|
||||||
|
DRIVES_SKIP+=("$drive_name")
|
||||||
|
echo ""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Start extended test
|
||||||
|
if ! smartctl -t long "$drive" >/dev/null 2>&1; then
|
||||||
|
error "$drive_name — failed to start extended self-test"
|
||||||
|
DRIVES_FAIL+=("$drive_name")
|
||||||
|
echo ""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo " Extended test started on $drive_name"
|
||||||
|
|
||||||
|
# Poll until complete
|
||||||
|
DRIVE_START=$(date +%s)
|
||||||
|
while true; do
|
||||||
|
sleep 60
|
||||||
|
|
||||||
|
STATUS_LINE=$(smartctl -l selftest "$drive" 2>/dev/null | grep -m1 "Extended")
|
||||||
|
|
||||||
|
if echo "$STATUS_LINE" | grep -q "Self-test routine in progress"; then
|
||||||
|
PCT=$(echo "$STATUS_LINE" | grep -oE "[0-9]+% of test remaining" || true)
|
||||||
|
log "$drive_name — test in progress ${PCT:+($PCT)}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Test finished — determine result
|
||||||
|
DRIVE_ELAPSED=$(format_duration $(( $(date +%s) - DRIVE_START )))
|
||||||
|
if echo "$STATUS_LINE" | grep -iq "Completed without error\|Successful"; then
|
||||||
|
echo " $ICON_DONE $drive_name — extended test passed ✅ ($DRIVE_ELAPSED)"
|
||||||
|
DRIVES_OK+=("$drive_name")
|
||||||
|
elif echo "$STATUS_LINE" | grep -iq "Failed\|failed"; then
|
||||||
|
FAIL_DETAIL=$(echo "$STATUS_LINE" | awk '{print $NF}')
|
||||||
|
error "$drive_name — extended test FAILED — $FAIL_DETAIL ($DRIVE_ELAPSED)"
|
||||||
|
DRIVES_FAIL+=("$drive_name")
|
||||||
|
elif [[ -z "$STATUS_LINE" ]]; then
|
||||||
|
warn "$drive_name — no test result found — may not support extended test"
|
||||||
|
DRIVES_SKIP+=("$drive_name")
|
||||||
|
else
|
||||||
|
warn "$drive_name — unexpected result: $STATUS_LINE ($DRIVE_ELAPSED)"
|
||||||
|
DRIVES_SKIP+=("$drive_name")
|
||||||
|
fi
|
||||||
|
echo ""
|
||||||
|
break
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
WINDOW_END=$(date +%s)
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Summary ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
echo "━━━━━ $ICON_SUMMARY SMART LONG TEST SUMMARY ━━━━━"
|
||||||
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||||
|
echo "$ICON_SMART Drives: ${#DRIVES_TO_TEST[@]} tested"
|
||||||
|
[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${DRIVES_OK[*]}"
|
||||||
|
[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${DRIVES_FAIL[*]}"
|
||||||
|
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${DRIVES_SKIP[*]}"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — no tests started"
|
||||||
|
elif [[ ${#DRIVES_FAIL[@]} -gt 0 ]]; then
|
||||||
|
echo "$ICON_ERROR Status: FAILURES — ${DRIVES_FAIL[*]}"
|
||||||
|
notify "SMART long test failures on $(hostname) ($MY_ID) — drives: ${DRIVES_FAIL[*]}" \
|
||||||
|
"SMART Long Test" "warning"
|
||||||
|
elif [[ ${#DRIVES_OK[@]} -gt 0 ]]; then
|
||||||
|
echo "$ICON_DONE Status: all ${#DRIVES_OK[@]} drive(s) passed ✅"
|
||||||
|
else
|
||||||
|
warn "Status: no results — all drives skipped"
|
||||||
|
fi
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && exit 1
|
||||||
|
exit 0
|
||||||
Reference in New Issue
Block a user