#!/bin/bash
# ==============================================================================================
# ============================= SMART Extended Self-Test =======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Runs a SMART extended (long) self-test on all drives (or a specific drive),
# waits for completion, and reports results. Extended tests do a full read-scan
# of every sector — catches bad sectors that the short test skips. Monthly cadence
# via the monthly maintenance orchestrator.
#
# NVMe drives are included — smartctl supports NVMe self-test via the same
# -t long interface (NVMe 1.3+ specification).
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Starts long self-tests on all drives sequentially (NOT in parallel) to avoid
# saturating I/O. One drive at a time: start, wait for completion, move to next.
# Sequential order keeps test duration predictable and avoids thermal stacking.
#
# Drives already running a self-test are skipped. If a previous test did not
# finish (interrupted mid-way), it is reported and the drive is re-tested.
#
# Each test is polled every 60 seconds. Typical durations:
#   HDD  2–4 TB : 60–120 min
#   HDD  8–12 TB: 90–180 min
#   SSD  any    :  5–15 min
#   NVMe any    :  5–10 min
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Single Instance Lock
#   acquire_lock prevents concurrent test runs — one long-test pass per server.
#
# Sequential Execution
#   Drives tested one at a time. Parallel long-tests thrash I/O and inflate temps.
#
# SIGTERM Trap
#   Poll loop exits cleanly on signal. Tests continue running in drive firmware.
#
# Tool Validation
#   platform_require_cmd confirms smartctl is present before use.
#
# Silent When Clean
#   Only failures and warnings produce notifications.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_SMART_IGNORE_DRIVES
#       Drives skipped in SMART testing. Aliased by detect_hosts() →
#       SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive.
#
# master.conf
#
#   SMART_TEMP_WARN / SMART_TEMP_CRIT
#       Fallback thresholds used for post-test temperature reporting if
#       dynamix.cfg is not found.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# smart_long_test.sh
#     Run extended self-test on all drives not in SMART_IGNORE_DRIVES.
#
# smart_long_test.sh /dev/sda
#     Run extended self-test on a specific drive. Bypasses the ignore list.
#
# smart_long_test.sh --status
#     Show last self-test result for all drives and exit.
#
# smart_long_test.sh --dry-run
#     Show which drives would be tested. No tests started.
#
# smart_long_test.sh --log
#     Verbose progress output during polling.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

TARGET_DRIVE="${PARSED_ARGS[0]:-}"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

platform_require_cmd \
    "$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
    "--version" "smartmontools" \
    "smartctl" || {
        error "smartctl not found — install smartmontools"
        notify "SMART long test failed on $(hostname) — smartmontools not installed" \
            "SMART Long Test" "warning"
        exit 1
    }

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts
get_unraid_temp_thresholds

log "Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
log "Ignore:       ${SMART_IGNORE_DRIVES[*]:-none}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no tests will be started"

trap 'warn "SMART long test script interrupted — tests continue in drive firmware"; exit 0' \
    SIGTERM SIGINT

# ==============================================================================================
# ── Build drive list ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
_build_drive_list() {
    local drives=()
    if [[ -n "$TARGET_DRIVE" ]]; then
        if [[ ! -e "$TARGET_DRIVE" ]]; then
            error "Drive not found: $TARGET_DRIVE"
            exit 1
        fi
        drives=("$TARGET_DRIVE")
    else
        for drive in /dev/sd? /dev/nvme?; do
            [[ ! -e "$drive" ]] && continue
            local drive_name
            drive_name=$(basename "$drive")
            local ignored=false
            for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
                [[ "$drive_name" == "$ignore" ]] && ignored=true && break
            done
            [[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
            if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
                log "Skipping $drive_name — SMART not enabled"
                continue
            fi
            drives+=("$drive")
        done
    fi
    echo "${drives[@]}"
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY SMART LONG TEST STATUS ━━━━━"
    echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""
    for drive in /dev/sd? /dev/nvme?; do
        [[ ! -e "$drive" ]] && continue
        drive_name=$(basename "$drive")
        local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
            grep -m1 "Extended" | awk '{print $NF}')
        ignored=false
        for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
            [[ "$drive_name" == "$ignore" ]] && ignored=true && break
        done
        if [[ "$ignored" == true ]]; then
            echo "  $ICON_WARN $drive_name — ignored"
        else
            echo "  $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
        fi
    done
    echo ""
    echo "  Ignored: ${SMART_IGNORE_DRIVES[*]:-none}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Run Extended Tests — Sequential ━━━
# ==============================================================================================
read -r -a DRIVES_TO_TEST <<< "$(_build_drive_list)"

if [[ ${#DRIVES_TO_TEST[@]} -eq 0 ]]; then
    warn "No drives to test — all may be on ignore list or SMART not enabled"
    exit 0
fi

echo ""
echo "━━━ $ICON_SMART SMART Extended Self-Test — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST  ${#DRIVES_TO_TEST[@]} drive(s) to test — running sequentially"
echo ""

WINDOW_START=$(date +%s)
DRIVES_OK=()
DRIVES_FAIL=()
DRIVES_SKIP=()

for drive in "${DRIVES_TO_TEST[@]}"; do
    drive_name=$(basename "$drive")
    echo "━━━ $ICON_SMART $drive_name ━━━"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would run extended self-test on $drive_name"
        echo ""
        continue
    fi

    # Check if test already in progress
    if smartctl -l selftest "$drive" 2>/dev/null | grep -q "Self-test routine in progress"; then
        warn "$drive_name — self-test already in progress — skipping"
        DRIVES_SKIP+=("$drive_name")
        echo ""
        continue
    fi

    # Start extended test
    if ! smartctl -t long "$drive" >/dev/null 2>&1; then
        error "$drive_name — failed to start extended self-test"
        DRIVES_FAIL+=("$drive_name")
        echo ""
        continue
    fi
    echo "  Extended test started on $drive_name"

    # Poll until complete
    DRIVE_START=$(date +%s)
    while true; do
        sleep 60

        STATUS_LINE=$(smartctl -l selftest "$drive" 2>/dev/null | grep -m1 "Extended")

        if echo "$STATUS_LINE" | grep -q "Self-test routine in progress"; then
            PCT=$(echo "$STATUS_LINE" | grep -oE "[0-9]+% of test remaining" || true)
            log "$drive_name — test in progress ${PCT:+($PCT)}"
            continue
        fi

        # Test finished — determine result
        DRIVE_ELAPSED=$(format_duration $(( $(date +%s) - DRIVE_START )))
        if echo "$STATUS_LINE" | grep -iq "Completed without error\|Successful"; then
            echo "  $ICON_DONE $drive_name — extended test passed ✅ ($DRIVE_ELAPSED)"
            DRIVES_OK+=("$drive_name")
        elif echo "$STATUS_LINE" | grep -iq "Failed\|failed"; then
            FAIL_DETAIL=$(echo "$STATUS_LINE" | awk '{print $NF}')
            error "$drive_name — extended test FAILED — $FAIL_DETAIL ($DRIVE_ELAPSED)"
            DRIVES_FAIL+=("$drive_name")
        elif [[ -z "$STATUS_LINE" ]]; then
            warn "$drive_name — no test result found — may not support extended test"
            DRIVES_SKIP+=("$drive_name")
        else
            warn "$drive_name — unexpected result: $STATUS_LINE ($DRIVE_ELAPSED)"
            DRIVES_SKIP+=("$drive_name")
        fi
        echo ""
        break
    done
done

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY SMART LONG TEST SUMMARY ━━━━━"
echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "$ICON_SMART  Drives:   ${#DRIVES_TO_TEST[@]} tested"
[[ ${#DRIVES_OK[@]}   -gt 0 ]] && echo "$ICON_DONE   Passed:   ${DRIVES_OK[*]}"
[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR  Failed:   ${DRIVES_FAIL[*]}"
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && echo "$ICON_WARN   Skipped:  ${DRIVES_SKIP[*]}"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no tests started"
elif [[ ${#DRIVES_FAIL[@]} -gt 0 ]]; then
    echo "$ICON_ERROR Status: FAILURES — ${DRIVES_FAIL[*]}"
    notify "SMART long test failures on $(hostname) ($MY_ID) — drives: ${DRIVES_FAIL[*]}" \
        "SMART Long Test" "warning"
elif [[ ${#DRIVES_OK[@]} -gt 0 ]]; then
    echo "$ICON_DONE Status: all ${#DRIVES_OK[@]} drive(s) passed ✅"
else
    warn "Status: no results — all drives skipped"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && exit 1
exit 0