Gmer4Lfe
bdcdfea175
Tests all drives sequentially (one at a time to avoid I/O saturation). Polls every 60s until complete. Notifies on failure. Respects SMART_IGNORE_DRIVES. Enable by uncommenting in MONTHLY_MAINTENANCE_SCRIPTS in master.conf. zfs_pool_scrub.sh was already fully implemented — also uncommented locally.
303 lines
12 KiB
Bash
Executable File
303 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= SMART Extended Self-Test =======================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Runs a SMART extended (long) self-test on all drives (or a specific drive),
|
||
# waits for completion, and reports results. Extended tests do a full read-scan
|
||
# of every sector — catches bad sectors that the short test skips. Monthly cadence
|
||
# via the monthly maintenance orchestrator.
|
||
#
|
||
# NVMe drives are included — smartctl supports NVMe self-test via the same
|
||
# -t long interface (NVMe 1.3+ specification).
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Starts long self-tests on all drives sequentially (NOT in parallel) to avoid
|
||
# saturating I/O. One drive at a time: start, wait for completion, move to next.
|
||
# Sequential order keeps test duration predictable and avoids thermal stacking.
|
||
#
|
||
# Drives already running a self-test are skipped. If a previous test did not
|
||
# finish (interrupted mid-way), it is reported and the drive is re-tested.
|
||
#
|
||
# Each test is polled every 60 seconds. Typical durations:
|
||
# HDD 2–4 TB : 60–120 min
|
||
# HDD 8–12 TB: 90–180 min
|
||
# SSD any : 5–15 min
|
||
# NVMe any : 5–10 min
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock prevents concurrent test runs — one long-test pass per server.
|
||
#
|
||
# Sequential Execution
|
||
# Drives tested one at a time. Parallel long-tests thrash I/O and inflate temps.
|
||
#
|
||
# SIGTERM Trap
|
||
# Poll loop exits cleanly on signal. Tests continue running in drive firmware.
|
||
#
|
||
# Tool Validation
|
||
# platform_require_cmd confirms smartctl is present before use.
|
||
#
|
||
# Silent When Clean
|
||
# Only failures and warnings produce notifications.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# host*.conf
|
||
#
|
||
# HOST*_SMART_IGNORE_DRIVES
|
||
# Drives skipped in SMART testing. Aliased by detect_hosts() →
|
||
# SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive.
|
||
#
|
||
# master.conf
|
||
#
|
||
# SMART_TEMP_WARN / SMART_TEMP_CRIT
|
||
# Fallback thresholds used for post-test temperature reporting if
|
||
# dynamix.cfg is not found.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# smart_long_test.sh
|
||
# Run extended self-test on all drives not in SMART_IGNORE_DRIVES.
|
||
#
|
||
# smart_long_test.sh /dev/sda
|
||
# Run extended self-test on a specific drive. Bypasses the ignore list.
|
||
#
|
||
# smart_long_test.sh --status
|
||
# Show last self-test result for all drives and exit.
|
||
#
|
||
# smart_long_test.sh --dry-run
|
||
# Show which drives would be tested. No tests started.
|
||
#
|
||
# smart_long_test.sh --log
|
||
# Verbose progress output during polling.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
TARGET_DRIVE="${PARSED_ARGS[0]:-}"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
platform_require_cmd \
|
||
"$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
|
||
"--version" "smartmontools" \
|
||
"smartctl" || {
|
||
error "smartctl not found — install smartmontools"
|
||
notify "SMART long test failed on $(hostname) — smartmontools not installed" \
|
||
"SMART Long Test" "warning"
|
||
exit 1
|
||
}
|
||
|
||
platform_require_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
get_unraid_temp_thresholds
|
||
|
||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no tests will be started"
|
||
|
||
trap 'warn "SMART long test script interrupted — tests continue in drive firmware"; exit 0' \
|
||
SIGTERM SIGINT
|
||
|
||
# ==============================================================================================
|
||
# ── Build drive list ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
_build_drive_list() {
|
||
local drives=()
|
||
if [[ -n "$TARGET_DRIVE" ]]; then
|
||
if [[ ! -e "$TARGET_DRIVE" ]]; then
|
||
error "Drive not found: $TARGET_DRIVE"
|
||
exit 1
|
||
fi
|
||
drives=("$TARGET_DRIVE")
|
||
else
|
||
for drive in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$drive" ]] && continue
|
||
local drive_name
|
||
drive_name=$(basename "$drive")
|
||
local ignored=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||
done
|
||
[[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
|
||
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
||
log "Skipping $drive_name — SMART not enabled"
|
||
continue
|
||
fi
|
||
drives+=("$drive")
|
||
done
|
||
fi
|
||
echo "${drives[@]}"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY SMART LONG TEST STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
for drive in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$drive" ]] && continue
|
||
drive_name=$(basename "$drive")
|
||
local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
|
||
grep -m1 "Extended" | awk '{print $NF}')
|
||
ignored=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||
done
|
||
if [[ "$ignored" == true ]]; then
|
||
echo " $ICON_WARN $drive_name — ignored"
|
||
else
|
||
echo " $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
|
||
fi
|
||
done
|
||
echo ""
|
||
echo " Ignored: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Run Extended Tests — Sequential ━━━
|
||
# ==============================================================================================
|
||
read -r -a DRIVES_TO_TEST <<< "$(_build_drive_list)"
|
||
|
||
if [[ ${#DRIVES_TO_TEST[@]} -eq 0 ]]; then
|
||
warn "No drives to test — all may be on ignore list or SMART not enabled"
|
||
exit 0
|
||
fi
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_SMART SMART Extended Self-Test — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo "$ICON_HOST ${#DRIVES_TO_TEST[@]} drive(s) to test — running sequentially"
|
||
echo ""
|
||
|
||
WINDOW_START=$(date +%s)
|
||
DRIVES_OK=()
|
||
DRIVES_FAIL=()
|
||
DRIVES_SKIP=()
|
||
|
||
for drive in "${DRIVES_TO_TEST[@]}"; do
|
||
drive_name=$(basename "$drive")
|
||
echo "━━━ $ICON_SMART $drive_name ━━━"
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would run extended self-test on $drive_name"
|
||
echo ""
|
||
continue
|
||
fi
|
||
|
||
# Check if test already in progress
|
||
if smartctl -l selftest "$drive" 2>/dev/null | grep -q "Self-test routine in progress"; then
|
||
warn "$drive_name — self-test already in progress — skipping"
|
||
DRIVES_SKIP+=("$drive_name")
|
||
echo ""
|
||
continue
|
||
fi
|
||
|
||
# Start extended test
|
||
if ! smartctl -t long "$drive" >/dev/null 2>&1; then
|
||
error "$drive_name — failed to start extended self-test"
|
||
DRIVES_FAIL+=("$drive_name")
|
||
echo ""
|
||
continue
|
||
fi
|
||
echo " Extended test started on $drive_name"
|
||
|
||
# Poll until complete
|
||
DRIVE_START=$(date +%s)
|
||
while true; do
|
||
sleep 60
|
||
|
||
STATUS_LINE=$(smartctl -l selftest "$drive" 2>/dev/null | grep -m1 "Extended")
|
||
|
||
if echo "$STATUS_LINE" | grep -q "Self-test routine in progress"; then
|
||
PCT=$(echo "$STATUS_LINE" | grep -oE "[0-9]+% of test remaining" || true)
|
||
log "$drive_name — test in progress ${PCT:+($PCT)}"
|
||
continue
|
||
fi
|
||
|
||
# Test finished — determine result
|
||
DRIVE_ELAPSED=$(format_duration $(( $(date +%s) - DRIVE_START )))
|
||
if echo "$STATUS_LINE" | grep -iq "Completed without error\|Successful"; then
|
||
echo " $ICON_DONE $drive_name — extended test passed ✅ ($DRIVE_ELAPSED)"
|
||
DRIVES_OK+=("$drive_name")
|
||
elif echo "$STATUS_LINE" | grep -iq "Failed\|failed"; then
|
||
FAIL_DETAIL=$(echo "$STATUS_LINE" | awk '{print $NF}')
|
||
error "$drive_name — extended test FAILED — $FAIL_DETAIL ($DRIVE_ELAPSED)"
|
||
DRIVES_FAIL+=("$drive_name")
|
||
elif [[ -z "$STATUS_LINE" ]]; then
|
||
warn "$drive_name — no test result found — may not support extended test"
|
||
DRIVES_SKIP+=("$drive_name")
|
||
else
|
||
warn "$drive_name — unexpected result: $STATUS_LINE ($DRIVE_ELAPSED)"
|
||
DRIVES_SKIP+=("$drive_name")
|
||
fi
|
||
echo ""
|
||
break
|
||
done
|
||
done
|
||
|
||
WINDOW_END=$(date +%s)
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary ━━━
|
||
# ==============================================================================================
|
||
echo "━━━━━ $ICON_SUMMARY SMART LONG TEST SUMMARY ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||
echo "$ICON_SMART Drives: ${#DRIVES_TO_TEST[@]} tested"
|
||
[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${DRIVES_OK[*]}"
|
||
[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${DRIVES_FAIL[*]}"
|
||
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${DRIVES_SKIP[*]}"
|
||
echo ""
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no tests started"
|
||
elif [[ ${#DRIVES_FAIL[@]} -gt 0 ]]; then
|
||
echo "$ICON_ERROR Status: FAILURES — ${DRIVES_FAIL[*]}"
|
||
notify "SMART long test failures on $(hostname) ($MY_ID) — drives: ${DRIVES_FAIL[*]}" \
|
||
"SMART Long Test" "warning"
|
||
elif [[ ${#DRIVES_OK[@]} -gt 0 ]]; then
|
||
echo "$ICON_DONE Status: all ${#DRIVES_OK[@]} drive(s) passed ✅"
|
||
else
|
||
warn "Status: no results — all drives skipped"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && exit 1
|
||
exit 0
|