massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+135
-102
@@ -1,93 +1,133 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- ZFS Memory Snapshot ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= ZFS Memory Snapshot ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly ZFS pool health and memory diagnostic report.
|
||||
# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage
|
||||
# and kernel pressure into a single report. Informational only — no action taken.
|
||||
# system_watchdog.sh handles threshold-based intervention.
|
||||
#
|
||||
# ── WHAT IT REPORTS ───────────────────────────────────────────────────────────────────────────
|
||||
# ZFS pool health — status, state, errors per pool (excluding ignored pools)
|
||||
# ARC statistics — current size, max, utilization %, metadata pressure
|
||||
# Memory status — total/free/available RAM vs thresholds
|
||||
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage
|
||||
# Kernel pressure — vmstat snapshot (3 samples)
|
||||
#
|
||||
# ── OUTPUT ────────────────────────────────────────────────────────────────────────────────────
|
||||
# Output goes to both console and ZFS_REPORT_LOG for later review.
|
||||
# In dry-run mode — console only, nothing written to log.
|
||||
# Notifies if any warning thresholds are exceeded.
|
||||
# Silent when all healthy — only problems produce output.
|
||||
#
|
||||
# Pools listed in ZFS_REPORT_IGNORE_POOLS are excluded from health reporting.
|
||||
# Useful for pools expected to run at high usage (docker, cache etc.)
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
|
||||
# Each server ignores its own single-disk ZFS array pools — not the peer's.
|
||||
#
|
||||
# All configuration in Master.conf under ZFS Memory Snapshot section.
|
||||
# Supports --dry-run (preview only, no log write) and --status.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — zpool + docker stats are slow, prevent duplicates
|
||||
# detect_hosts() — correct pool ignore list per host
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# DOCKER_TIMEOUT — docker stats protected against hung daemon
|
||||
# ZFS not available — skips pool and ARC sections gracefully
|
||||
# Docker not available — skips container section gracefully
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from health reporting
|
||||
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# ZFS_REPORT_LOG — log file path for weekly reports
|
||||
# ZFS_REPORT_ARC_WARN_PCT — warn if ARC using more than this % of max
|
||||
# ZFS_REPORT_FREE_WARN_GB — warn if less than this GB free RAM
|
||||
# ZFS_REPORT_AVAIL_WARN_GB — warn if less than this GB available RAM
|
||||
# ZFS_REPORT_DOCKER_TOP — how many top Docker containers to show
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# zfs_memory_snapshot.sh — normal report (writes to log)
|
||||
# zfs_memory_snapshot.sh --dry-run — console only, no log write
|
||||
# zfs_memory_snapshot.sh --log — verbose output
|
||||
# zfs_memory_snapshot.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Monitor/report script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
||||
detect_hosts
|
||||
|
||||
# Build ignore pool lookup map — O(1) check per pool
|
||||
declare -A IGNORE_POOL_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
|
||||
# Tee output to log file unless dry run
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
|
||||
exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# Build ignore list for quick lookup
|
||||
declare -A IGNORE_POOL_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
|
||||
info "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
|
||||
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
|
||||
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
|
||||
echo "$ICON_MEM Avail RAM warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
|
||||
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
|
||||
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
|
||||
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
|
||||
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
|
||||
echo "$ICON_MEM Avail warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
|
||||
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
|
||||
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Tracking
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Report ━━━
|
||||
# ==============================================================================================
|
||||
WARNINGS=()
|
||||
START=$(date +%s)
|
||||
DATE=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE"
|
||||
echo " $ICON_HOST Host: $(hostname)"
|
||||
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_ZFS ZFS Pool Health ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── ZFS Pool Health ───────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"
|
||||
|
||||
@@ -95,13 +135,11 @@ if ! command -v zpool >/dev/null 2>&1; then
|
||||
warn "ZFS not available on this system — skipping pool checks"
|
||||
else
|
||||
# Pool status — filtered to key lines, ignoring specified pools
|
||||
info "Pool status:"
|
||||
CURRENT_POOL=""
|
||||
while IFS= read -r line; do
|
||||
# Extract pool name from "pool: poolname" lines
|
||||
if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
|
||||
CURRENT_POOL="${BASH_REMATCH[1]// /}"
|
||||
fi
|
||||
# Skip lines belonging to ignored pools
|
||||
[[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
|
||||
echo " $line"
|
||||
done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")
|
||||
@@ -109,41 +147,36 @@ else
|
||||
echo ""
|
||||
|
||||
# Pool list — filter out ignored pools
|
||||
info "Pool overview:"
|
||||
zpool list 2>/dev/null | while IFS= read -r line; do
|
||||
# Always show header line
|
||||
if [[ "$line" == NAME* ]]; then
|
||||
echo " $line"
|
||||
continue
|
||||
fi
|
||||
# Extract pool name (first field)
|
||||
pool_name=$(echo "$line" | awk '{print $1}')
|
||||
[[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
|
||||
echo " $line"
|
||||
done
|
||||
|
||||
# Check for unhealthy pools — excluding ignored ones
|
||||
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | while IFS=$'\t' read -r name health; do
|
||||
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
|
||||
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
|
||||
done)
|
||||
# Check for unhealthy non-ignored pools
|
||||
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \
|
||||
while IFS=$'\t' read -r name health; do
|
||||
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
|
||||
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
|
||||
done)
|
||||
|
||||
if [[ -n "$UNHEALTHY" ]]; then
|
||||
error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
|
||||
WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
|
||||
else
|
||||
success "All monitored ZFS pools are ONLINE"
|
||||
log "All monitored ZFS pools are ONLINE ✅"
|
||||
fi
|
||||
|
||||
# Show ignored pools
|
||||
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
|
||||
info "Ignored pools (not reported): ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_ZFS ARC Statistics ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── ARC Statistics ────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS ARC Statistics ━━━"
|
||||
|
||||
@@ -170,14 +203,16 @@ else
|
||||
warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
|
||||
WARNINGS+=("ARC high: ${ARC_PCT}%")
|
||||
else
|
||||
success "ARC utilization ${ARC_PCT}% — within threshold (${ZFS_REPORT_ARC_WARN_PCT}%)"
|
||||
log "ARC utilization ${ARC_PCT}% — within threshold ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
info "Metadata pressure:"
|
||||
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \
|
||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \
|
||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
|
||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
|
||||
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
|
||||
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
|
||||
@@ -187,17 +222,15 @@ else
|
||||
echo " $ICON_ZFS Metadata Misses: ${META_MISSES}"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_MEM Memory Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Memory Status ─────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_MEM Memory Status ━━━"
|
||||
|
||||
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
|
||||
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
|
||||
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
|
||||
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
|
||||
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
|
||||
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
|
||||
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
|
||||
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
|
||||
|
||||
echo " $ICON_MEM Total RAM: $TOTAL_HUMAN"
|
||||
echo " $ICON_MEM Free RAM: $FREE_HUMAN"
|
||||
@@ -207,42 +240,38 @@ if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
|
||||
warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
|
||||
WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
|
||||
else
|
||||
success "Free RAM ${FREE_HUMAN} — within threshold (${ZFS_REPORT_FREE_WARN_GB}GB)"
|
||||
log "Free RAM ${FREE_HUMAN} — within threshold ✅"
|
||||
fi
|
||||
|
||||
if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
|
||||
warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
|
||||
WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
|
||||
else
|
||||
success "Available RAM ${AVAIL_HUMAN} — within threshold (${ZFS_REPORT_AVAIL_WARN_GB}GB)"
|
||||
log "Available RAM ${AVAIL_HUMAN} — within threshold ✅"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Docker Memory ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Docker Memory ─────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
warn "Docker not available — skipping container memory section"
|
||||
else
|
||||
docker stats --no-stream \
|
||||
timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
||||
--format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
|
||||
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \
|
||||
while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Kernel Pressure ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Kernel Pressure ───────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"
|
||||
|
||||
if ! command -v vmstat >/dev/null 2>&1; then
|
||||
warn "vmstat not available — skipping kernel pressure section"
|
||||
else
|
||||
info "vmstat snapshot (3 samples):"
|
||||
vmstat 1 3 2>/dev/null | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
@@ -250,23 +279,27 @@ fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
|
||||
[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
|
||||
echo "$ICON_ZFS Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
||||
success "Report complete — no warnings"
|
||||
log "$ICON_DONE All checks within thresholds ✅"
|
||||
else
|
||||
echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
|
||||
for w in "${WARNINGS[@]}"; do
|
||||
echo " $ICON_WARN $w"
|
||||
done
|
||||
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning"
|
||||
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \
|
||||
"ZFS Report" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user