massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+175
-91
@@ -1,84 +1,128 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- ZFS Pool Scrub ---------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= ZFS Pool Scrub ============================================
|
||||
# ==============================================================================================
|
||||
# Triggers a ZFS scrub on all pools (or a specific pool) and waits for completion.
|
||||
# Sends a notification when scrub completes with a summary of any errors found.
|
||||
#
|
||||
# ZFS scrub reads every block on every pool and verifies checksums — it catches
|
||||
# silent data corruption that would otherwise only surface when you try to read
|
||||
# the corrupted data. Running monthly is recommended for all ZFS pools.
|
||||
# ── WHAT ZFS SCRUB DOES ───────────────────────────────────────────────────────────────────────
|
||||
# Reads every block on every pool and verifies checksums against the stored hash.
|
||||
# Catches silent data corruption that would otherwise only surface when you read the
|
||||
# corrupted data — by then it may be too late for redundancy to help.
|
||||
#
|
||||
# Usage:
|
||||
# zfs_pool_scrub.sh — scrub all pools
|
||||
# zfs_pool_scrub.sh poolname — scrub specific pool only
|
||||
# zfs_pool_scrub.sh --status — show scrub status for all pools
|
||||
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
|
||||
# Scrub is safe to run while the pool is in use — it does not interrupt normal I/O.
|
||||
# It does consume I/O bandwidth — run during off-peak hours or maintenance windows.
|
||||
# Monthly is recommended for all pools. Quarterly minimum for large pools.
|
||||
#
|
||||
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped unless specified explicitly.
|
||||
# Scrub runs in background — script polls until complete then reports.
|
||||
# Safe to run while the pool is in use — scrub does not interrupt normal I/O.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Starts scrub on each pool then polls every 60 seconds until all complete.
|
||||
# Progress shown via warn() every poll (visible) when scrub is running.
|
||||
# Safe to leave running or interrupt — scrub continues even if script is stopped.
|
||||
# On completion reports errors per pool and notifies if any found.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
|
||||
# Pools in ZFS_REPORT_IGNORE_POOLS are skipped (single-disk VMs, temp pools etc.)
|
||||
# unless specified explicitly as a positional argument.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent scrub starts on same server
|
||||
# detect_hosts() — correct pool ignore list per host
|
||||
# validate_unraid_cmd — zpool and notify validated before use
|
||||
# Scrub-in-progress check — skips pools already scrubbing rather than erroring
|
||||
# SIGTERM trap — poll loop exits cleanly on signal
|
||||
# Silent when clean — only errors produce visible output and notification
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from automatic scrub
|
||||
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# zfs_pool_scrub.sh — scrub all non-ignored pools
|
||||
# zfs_pool_scrub.sh poolname — scrub specific pool (bypasses ignore list)
|
||||
# zfs_pool_scrub.sh --status — show scrub status for all pools
|
||||
# zfs_pool_scrub.sh --dry-run — show what would be scrubbed
|
||||
# zfs_pool_scrub.sh --log — verbose progress output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
TARGET_POOL="${PARSED_ARGS[0]:-}"
|
||||
SCRUB_RUNNING=true
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
validate_unraid_cmd \
|
||||
"$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \
|
||||
"--version" "" \
|
||||
"zpool" || {
|
||||
error "ZFS not available on this system — zpool not found"
|
||||
exit 1
|
||||
}
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v zpool >/dev/null 2>&1; then
|
||||
error "ZFS not available on this system"
|
||||
exit 1
|
||||
fi
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
||||
detect_hosts
|
||||
|
||||
success "ZFS available"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
|
||||
|
||||
# Build ignore map
|
||||
# Build ignore pool map
|
||||
declare -A IGNORE_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
|
||||
|
||||
# SIGTERM trap — exit poll loop cleanly
|
||||
trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \
|
||||
SIGTERM SIGINT
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SCRUB STATUS ━━━━━"
|
||||
zpool list -H -o name 2>/dev/null | while read -r pool; do
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
while IFS= read -r pool; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
||||
echo " $ICON_ZFS $pool — $SCAN"
|
||||
done
|
||||
IGNORED=""
|
||||
[[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)"
|
||||
echo " $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}"
|
||||
done < <(zpool list -H -o name 2>/dev/null)
|
||||
echo ""
|
||||
echo " Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build pool list to scrub
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── Build pool list ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
POOLS_TO_SCRUB=()
|
||||
|
||||
if [[ -n "$TARGET_POOL" ]]; then
|
||||
# Specific pool requested — validate it exists
|
||||
# Specific pool — bypass ignore list, validate exists
|
||||
if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then
|
||||
error "Pool not found: $TARGET_POOL"
|
||||
exit 1
|
||||
@@ -89,7 +133,7 @@ else
|
||||
while IFS= read -r pool; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then
|
||||
info "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
|
||||
log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
|
||||
continue
|
||||
fi
|
||||
POOLS_TO_SCRUB+=("$pool")
|
||||
@@ -97,99 +141,139 @@ else
|
||||
fi
|
||||
|
||||
if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then
|
||||
warn "No pools to scrub"
|
||||
warn "No pools to scrub — all pools may be on the ignore list"
|
||||
warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
info "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
|
||||
log "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_ZFS Start Scrubs ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Start Scrubs ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS Starting ZFS Scrubs ━━━"
|
||||
echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━"
|
||||
START=$(date +%s)
|
||||
|
||||
STARTED=()
|
||||
SKIPPED_POOLS=()
|
||||
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
info "$ICON_ZFS Starting scrub on $pool..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
zpool scrub "$pool" 2>/dev/null && \
|
||||
success "$pool scrub started" || \
|
||||
error "Failed to start scrub on $pool"
|
||||
else
|
||||
|
||||
# Check if scrub already in progress
|
||||
ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$ALREADY" -gt 0 ]]; then
|
||||
warn "$pool — scrub already in progress — joining existing scrub"
|
||||
STARTED+=("$pool")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Starting scrub on $pool..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would scrub: $pool"
|
||||
STARTED+=("$pool")
|
||||
elif zpool scrub "$pool" 2>/dev/null; then
|
||||
log "$pool scrub started ✅"
|
||||
STARTED+=("$pool")
|
||||
else
|
||||
error "Failed to start scrub on $pool"
|
||||
SKIPPED_POOLS+=("$pool")
|
||||
fi
|
||||
done
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
||||
echo "$ICON_WARN Status: DRY RUN — no scrubs started"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
warn "DRY RUN — no scrubs started"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
}
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Poll until complete ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ ${#STARTED[@]} -eq 0 ]]; then
|
||||
error "No scrubs were started — check pool status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Poll Until Complete ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_TIME Waiting for scrubs to complete ━━━"
|
||||
info "Polling every 60 seconds — this may take a while on large pools"
|
||||
info "Safe to leave running — scrub continues even if this script is stopped"
|
||||
echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
|
||||
log "Polling every 60 seconds — scrubs may take hours on large pools"
|
||||
log "Safe to interrupt — scrubs continue in background if script is stopped"
|
||||
|
||||
STILL_RUNNING=true
|
||||
while [[ "$STILL_RUNNING" == true ]]; do
|
||||
while [[ "$SCRUB_RUNNING" == true ]]; do
|
||||
sleep 60
|
||||
|
||||
STILL_RUNNING=false
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
STATUS=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$STATUS" -gt 0 ]]; then
|
||||
for pool in "${STARTED[@]}"; do
|
||||
IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -c "in progress" || true)
|
||||
if [[ "$IN_PROGRESS" -gt 0 ]]; then
|
||||
STILL_RUNNING=true
|
||||
REPAIRED=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -oE "[0-9]+ repaired")
|
||||
log "$pool — scrub in progress ${REPAIRED:+($REPAIRED)}"
|
||||
# Show progress — always visible so user knows it's running
|
||||
PROGRESS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done")
|
||||
REPAIRED=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "scan:" | grep -oE "[0-9]+ repaired")
|
||||
warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
success "All scrubs complete"
|
||||
warn "All scrubs complete — $(format_duration $(( END - START )))"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Results ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Results ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS Scrub Results ━━━"
|
||||
|
||||
POOLS_OK=()
|
||||
POOLS_ERRORS=()
|
||||
|
||||
for pool in "${POOLS_TO_SCRUB[@]}"; do
|
||||
for pool in "${STARTED[@]}"; do
|
||||
SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:")
|
||||
ERRORS=$(zpool status "$pool" 2>/dev/null | grep "errors:" | grep -v "No known data errors")
|
||||
ERRORS=$(zpool status "$pool" 2>/dev/null | \
|
||||
grep "errors:" | grep -v "No known data errors")
|
||||
|
||||
if [[ -n "$ERRORS" ]]; then
|
||||
error "$pool — $SCAN_LINE"
|
||||
error "$pool — $ERRORS"
|
||||
error "$pool — ERRORS FOUND"
|
||||
error " $SCAN_LINE"
|
||||
error " $ERRORS"
|
||||
POOLS_ERRORS+=("$pool")
|
||||
else
|
||||
success "$pool — $SCAN_LINE"
|
||||
log "$pool — $SCAN_LINE"
|
||||
POOLS_OK+=("$pool")
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
|
||||
echo "$ICON_ZFS Pools scrubbed: ${#POOLS_TO_SCRUB[@]}"
|
||||
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
|
||||
echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_ZFS Pools: ${#POOLS_TO_SCRUB[@]} to scrub"
|
||||
echo "$ICON_SUCCESS Clean: ${#POOLS_OK[@]}"
|
||||
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR Errors: ${#POOLS_ERRORS[@]}"
|
||||
[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}"
|
||||
notify "ZFS scrub complete on $(hostname) — ERRORS found in pools: ${POOLS_ERRORS[*]}" "ZFS Scrub" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL POOLS CLEAN"
|
||||
notify "ZFS scrub complete on $(hostname) — ${#POOLS_OK[@]} pools clean in $(format_duration $((END - START)))" "ZFS Scrub" "normal"
|
||||
notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \
|
||||
"ZFS Scrub" "warning"
|
||||
elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then
|
||||
log "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user