279 lines
12 KiB
Bash
279 lines
12 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= ZFS Pool Scrub ============================================
|
|
# ==============================================================================================
|
|
# Triggers a ZFS scrub on all pools (or a specific pool) and waits for completion.
|
|
# Sends a notification when scrub completes with a summary of any errors found.
|
|
#
|
|
# ── WHAT ZFS SCRUB DOES ───────────────────────────────────────────────────────────────────────
|
|
# Reads every block on every pool and verifies checksums against the stored hash.
|
|
# Catches silent data corruption that would otherwise only surface when you read the
|
|
# corrupted data — by then it may be too late for redundancy to help.
|
|
#
|
|
# Scrub is safe to run while the pool is in use — it does not interrupt normal I/O.
|
|
# It does consume I/O bandwidth — run during off-peak hours or maintenance windows.
|
|
# Monthly is recommended for all pools. Quarterly minimum for large pools.
|
|
#
|
|
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
|
# Starts scrub on each pool then polls every 60 seconds until all complete.
|
|
# Progress shown via warn() every poll (visible) when scrub is running.
|
|
# Safe to leave running or interrupt — scrub continues even if script is stopped.
|
|
# On completion reports errors per pool and notifies if any found.
|
|
#
|
|
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
|
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
|
|
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped (single-disk VMs, temp pools etc.)
|
|
# unless specified explicitly as a positional argument.
|
|
#
|
|
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
|
# acquire_lock — prevents concurrent scrub starts on same server
|
|
# detect_hosts() — correct pool ignore list per host
|
|
# validate_unraid_cmd — zpool and notify validated before use
|
|
# Scrub-in-progress check — skips pools already scrubbing rather than erroring
|
|
# SIGTERM trap — poll loop exits cleanly on signal
|
|
# Silent when clean — only errors produce visible output and notification
|
|
#
|
|
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
|
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from automatic scrub
|
|
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# zfs_pool_scrub.sh — scrub all non-ignored pools
|
|
# zfs_pool_scrub.sh poolname — scrub specific pool (bypasses ignore list)
|
|
# zfs_pool_scrub.sh --status — show scrub status for all pools
|
|
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
|
|
# zfs_pool_scrub.sh --log — verbose progress output
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
TARGET_POOL="${PARSED_ARGS[0]:-}"
|
|
SCRUB_RUNNING=true
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \
|
|
"--version" "" \
|
|
"zpool" || {
|
|
error "ZFS not available on this system — zpool not found"
|
|
exit 1
|
|
}
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
|
detect_hosts
|
|
|
|
# Build ignore pool map
|
|
declare -A IGNORE_MAP
|
|
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
|
[[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1
|
|
done
|
|
|
|
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
|
|
|
|
# SIGTERM trap — exit poll loop cleanly
|
|
trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \
|
|
SIGTERM SIGINT
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
while IFS= read -r pool; do
|
|
[[ -z "$pool" ]] && continue
|
|
SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
|
IGNORED=""
|
|
[[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)"
|
|
echo " $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}"
|
|
done < <(zpool list -H -o name 2>/dev/null)
|
|
echo ""
|
|
echo " Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── Build pool list ────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
POOLS_TO_SCRUB=()
|
|
|
|
if [[ -n "$TARGET_POOL" ]]; then
|
|
# Specific pool — bypass ignore list, validate exists
|
|
if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then
|
|
error "Pool not found: $TARGET_POOL"
|
|
exit 1
|
|
fi
|
|
POOLS_TO_SCRUB=("$TARGET_POOL")
|
|
else
|
|
# All pools — skip ignored ones
|
|
while IFS= read -r pool; do
|
|
[[ -z "$pool" ]] && continue
|
|
if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then
|
|
log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
|
|
continue
|
|
fi
|
|
POOLS_TO_SCRUB+=("$pool")
|
|
done < <(zpool list -H -o name 2>/dev/null)
|
|
fi
|
|
|
|
if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then
|
|
warn "No pools to scrub — all pools may be on the ignore list"
|
|
warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
|
exit 0
|
|
fi
|
|
|
|
log "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Start Scrubs ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━"
|
|
START=$(date +%s)
|
|
|
|
STARTED=()
|
|
SKIPPED_POOLS=()
|
|
|
|
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
|
|
|
# Check if scrub already in progress
|
|
ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
|
|
if [[ "$ALREADY" -gt 0 ]]; then
|
|
warn "$pool — scrub already in progress — joining existing scrub"
|
|
STARTED+=("$pool")
|
|
continue
|
|
fi
|
|
|
|
log "Starting scrub on $pool..."
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would scrub: $pool"
|
|
STARTED+=("$pool")
|
|
elif zpool scrub "$pool" 2>/dev/null; then
|
|
log "$pool scrub started ✅"
|
|
STARTED+=("$pool")
|
|
else
|
|
error "Failed to start scrub on $pool"
|
|
SKIPPED_POOLS+=("$pool")
|
|
fi
|
|
done
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
warn "DRY RUN — no scrubs started"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ${#STARTED[@]} -eq 0 ]]; then
|
|
error "No scrubs were started — check pool status"
|
|
exit 1
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Poll Until Complete ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
|
|
log "Polling every 60 seconds — scrubs may take hours on large pools"
|
|
log "Safe to interrupt — scrubs continue in background if script is stopped"
|
|
|
|
while [[ "$SCRUB_RUNNING" == true ]]; do
|
|
sleep 60
|
|
|
|
STILL_RUNNING=false
|
|
for pool in "${STARTED[@]}"; do
|
|
IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
|
grep "scan:" | grep -c "in progress" || true)
|
|
if [[ "$IN_PROGRESS" -gt 0 ]]; then
|
|
STILL_RUNNING=true
|
|
# Show progress — always visible so user knows it's running
|
|
PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
|
grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done")
|
|
REPAIRED=$(zpool status "$pool" 2>/dev/null | \
|
|
grep "scan:" | grep -oE "[0-9]+ repaired")
|
|
warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}"
|
|
fi
|
|
done
|
|
|
|
[[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false
|
|
done
|
|
|
|
END=$(date +%s)
|
|
warn "All scrubs complete — $(format_duration $(( END - START )))"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Results ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_ZFS Scrub Results ━━━"
|
|
|
|
POOLS_OK=()
|
|
POOLS_ERRORS=()
|
|
|
|
for pool in "${STARTED[@]}"; do
|
|
SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
|
ERRORS=$(zpool status "$pool" 2>/dev/null | \
|
|
grep "errors:" | grep -v "No known data errors")
|
|
|
|
if [[ -n "$ERRORS" ]]; then
|
|
error "$pool — ERRORS FOUND"
|
|
error " $SCAN_LINE"
|
|
error " $ERRORS"
|
|
POOLS_ERRORS+=("$pool")
|
|
else
|
|
log "$pool — $SCAN_LINE"
|
|
POOLS_OK+=("$pool")
|
|
fi
|
|
done
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_ZFS Pools: ${#POOLS_TO_SCRUB[@]} to scrub"
|
|
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
|
|
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
|
|
[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
|
|
if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}"
|
|
notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \
|
|
"ZFS Scrub" "warning"
|
|
elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then
|
|
log "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1
|
|
exit 0 |