394 lines
18 KiB
Bash
394 lines
18 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= ZFS Memory Snapshot ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Weekly ZFS pool health and memory diagnostic report. Scheduled Sunday 6am —
|
|
# first in the Sunday monitoring block, before other scripts run. Informational
|
|
# only — system_watchdog.sh handles threshold-based intervention.
|
|
#
|
|
# Combines ZFS pool status, ARC statistics, Docker memory usage, and kernel
|
|
# memory pressure into a single snapshot. In normal mode output goes to both
|
|
# console (for User Scripts output log) and ZFS_REPORT_LOG for week-over-week
|
|
# comparison. In --dry-run mode, console only — nothing written to the log.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Five report sections (each skips gracefully if its data source is unavailable):
|
|
#
|
|
# ZFS pool health — status, state, errors per pool. Pools in
|
|
# ZFS_REPORT_IGNORE_POOLS excluded from the report
|
|
# (still fully monitored by unRAID — report-only exclusion).
|
|
# ARC statistics — current ARC vs max, metadata pressure, hit rate.
|
|
# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT, or if
|
|
# ARC headroom (max - current) drops below ZFS_REPORT_ARC_FREE_WARN_GB.
|
|
# Memory status — total, free, available RAM (informational only — see note below).
|
|
# Warns if available < ZFS_REPORT_AVAIL_WARN_GB.
|
|
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
|
|
# Useful for spotting containers approaching watchdog limits.
|
|
# Kernel pressure — vmstat snapshot (3 samples).
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Informational, Not Interventional
|
|
# This script reports — it does not act. system_watchdog.sh handles
|
|
# threshold-based intervention. Keeping the roles separate means the report
|
|
# is never suppressed by the same logic that triggers remediation.
|
|
#
|
|
# Week-Over-Week Comparison
|
|
# Output is written to ZFS_REPORT_LOG so the same snapshot can be reviewed
|
|
# across weeks. Memory pressure and ARC creep are slow — a single run is
|
|
# rarely conclusive; the trend across weeks is what matters.
|
|
#
|
|
# Section Independence
|
|
# Each of the five report sections guards its own data source. ZFS not
|
|
# available, Docker not responding — those sections skip, the rest still run.
|
|
# A partial report is more useful than no report.
|
|
#
|
|
# ARC Headroom, Not System Free RAM
|
|
# ZFS ARC deliberately grows to use most of the RAM the system isn't otherwise
|
|
# using — that's the point of a page cache. System-wide "free" RAM being low is
|
|
# therefore normal and not a signal of anything, so the memory warning is based
|
|
# on ARC headroom (ARC_MAX - ARC_CURRENT) instead — how much room ARC itself has
|
|
# left before it hits its configured ceiling. "Available" RAM (which accounts for
|
|
# reclaimable cache) is still checked separately as a true system-pressure signal.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents duplicate runs — zpool and docker stats are slow.
|
|
#
|
|
# Per-Host Pool Ignore List
|
|
# detect_hosts() aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
|
|
# Single-disk JBOD members excluded from report noise per server.
|
|
#
|
|
# ZFS Availability Guard
|
|
# Skips pool and ARC sections gracefully if ZFS is not available on this server.
|
|
#
|
|
# Docker Availability Guard
|
|
# Skips container memory section gracefully if Docker is not responding.
|
|
#
|
|
# Docker Stats Timeout
|
|
# DOCKER_TIMEOUT caps docker stats calls. A hung daemon does not block the report.
|
|
#
|
|
# ==============================================================================================
|
|
# STATE FILES
|
|
# ==============================================================================================
|
|
#
|
|
# ZFS_REPORT_LOG — /var/log/zfs-weekly-health.log (tmpfs, resets on reboot)
|
|
# Weekly report written here for comparison across runs. Open the log to
|
|
# see pool health trend week over week without remembering last week's values.
|
|
# In dry-run mode, console only — nothing written.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_ZFS_REPORT_IGNORE_POOLS
|
|
# Pools excluded from health reporting. Single-disk JBOD members generate
|
|
# expected high-usage warnings — exclude them to reduce report noise.
|
|
# Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS.
|
|
#
|
|
# master.conf
|
|
#
|
|
# ZFS_REPORT_LOG
|
|
# Log file path for weekly reports. (default: /var/log/zfs-weekly-health.log)
|
|
#
|
|
# ZFS_REPORT_ARC_WARN_PCT
|
|
# Warn if ARC is using more than this percentage of its configured max. (default: 90)
|
|
#
|
|
# ZFS_REPORT_ARC_FREE_WARN_GB
|
|
# Warn if ARC headroom (ARC_MAX - ARC_CURRENT) drops below this many GB. (default: 10)
|
|
#
|
|
# ZFS_REPORT_AVAIL_WARN_GB
|
|
# Warn if available RAM is below this threshold in GB. (default: 20)
|
|
#
|
|
# ZFS_REPORT_DOCKER_TOP
|
|
# Number of top Docker containers by memory usage to include. (default: 10)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# zfs_memory_snapshot.sh
|
|
# Generate report, write to ZFS_REPORT_LOG and console. Notify on warnings.
|
|
#
|
|
# zfs_memory_snapshot.sh --dry-run
|
|
# Generate report to console only. No log write, no notifications.
|
|
#
|
|
# zfs_memory_snapshot.sh --status
|
|
# Show pool ignore list and threshold configuration. Then exit.
|
|
#
|
|
# zfs_memory_snapshot.sh --log
|
|
# Verbose output during report generation.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
DOCKER_TIMEOUT=15
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
|
detect_hosts
|
|
|
|
# Build ignore pool lookup map — O(1) check per pool
|
|
declare -A IGNORE_POOL_MAP
|
|
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
|
|
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
|
|
done
|
|
|
|
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
|
log "$ICON_GEAR Config: arc-warn=${ZFS_REPORT_ARC_WARN_PCT}% arc-free-warn=${ZFS_REPORT_ARC_FREE_WARN_GB}GB avail-warn=${ZFS_REPORT_AVAIL_WARN_GB}GB docker-top=${ZFS_REPORT_DOCKER_TOP}"
|
|
|
|
# Tee output to log file unless dry run
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
|
|
exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
|
|
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
|
|
echo "$ICON_ZFS ARC free warn: ${ZFS_REPORT_ARC_FREE_WARN_GB}GB"
|
|
echo "$ICON_MEM Avail warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
|
|
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
|
|
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Report ━━━
|
|
# ==============================================================================================
|
|
WARNINGS=()
|
|
START=$(date +%s)
|
|
DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
|
|
|
echo ""
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE"
|
|
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# ── ZFS Pool Health ───────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"
|
|
|
|
if ! command -v zpool >/dev/null 2>&1; then
|
|
warn "ZFS not available on this system — skipping pool checks"
|
|
else
|
|
# Pool status — filtered to key lines, ignoring specified pools
|
|
CURRENT_POOL=""
|
|
while IFS= read -r line; do
|
|
if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
|
|
CURRENT_POOL="${BASH_REMATCH[1]// /}"
|
|
fi
|
|
[[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
|
|
echo " $line"
|
|
done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")
|
|
|
|
echo ""
|
|
|
|
# Pool list — filter out ignored pools
|
|
zpool list 2>/dev/null | while IFS= read -r line; do
|
|
if [[ "$line" == NAME* ]]; then
|
|
echo " $line"
|
|
continue
|
|
fi
|
|
pool_name=$(echo "$line" | awk '{print $1}')
|
|
[[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
|
|
echo " $line"
|
|
done
|
|
|
|
# Check for unhealthy non-ignored pools
|
|
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \
|
|
while IFS=$'\t' read -r name health; do
|
|
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
|
|
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
|
|
done)
|
|
|
|
if [[ -n "$UNHEALTHY" ]]; then
|
|
error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
|
|
WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
|
|
else
|
|
echo "All monitored ZFS pools are ONLINE ✅"
|
|
fi
|
|
|
|
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
|
|
log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
|
fi
|
|
fi
|
|
|
|
# ── ARC Statistics ────────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_ZFS ARC Statistics ━━━"
|
|
|
|
if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then
|
|
warn "ZFS arcstats not available — skipping ARC section"
|
|
else
|
|
# zfs_arc_max reads 0 when it has been left at the default, which is a value rather than a
|
|
# failure — so the || fallback never fires for the case that actually needs it, exactly like a
|
|
# grep -c that prints 0 and exits 1. Zero here would reach the ARC_PCT division below, and awk
|
|
# treats division by zero as fatal: it prints nothing, ARC_PCT comes back empty, and the whole
|
|
# ARC section reports blanks. c_max is the cap the kernel is really enforcing either way.
|
|
ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || echo 0)
|
|
ARC_MAX="${ARC_MAX//[^0-9]/}"
|
|
if [[ "${ARC_MAX:-0}" -eq 0 ]]; then
|
|
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
|
|
ARC_MAX="${ARC_MAX:-0}"
|
|
fi
|
|
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
|
|
|
ARC_MAX_GB=$(bytes_to_gb "$ARC_MAX" 1)
|
|
ARC_CUR_GB=$(bytes_to_gb "$ARC_SIZE" 1)
|
|
ARC_META_GB=$(bytes_to_gb "$ARC_META_USED" 1)
|
|
ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
|
|
ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
|
|
ARC_FREE_GB=$(awk "BEGIN {printf \"%d\", ($ARC_MAX - $ARC_SIZE) / 1073741824}")
|
|
|
|
echo " $ICON_ZFS ARC Max: ${ARC_MAX_GB}GB"
|
|
echo " $ICON_ZFS ARC Current: ${ARC_CUR_GB}GB"
|
|
echo " $ICON_ZFS ARC Meta Used: ${ARC_META_GB}GB"
|
|
echo " $ICON_ZFS ARC Utilization: ${ARC_PCT}%"
|
|
echo " $ICON_ZFS ARC Free: ${ARC_FREE_GB}GB"
|
|
|
|
if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then
|
|
warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
|
|
WARNINGS+=("ARC high: ${ARC_PCT}%")
|
|
else
|
|
log "ARC utilization ${ARC_PCT}% — within threshold ✅"
|
|
fi
|
|
|
|
if [[ "$ARC_FREE_GB" -lt "$ZFS_REPORT_ARC_FREE_WARN_GB" ]]; then
|
|
warn "ARC free ${ARC_FREE_GB}GB — below ${ZFS_REPORT_ARC_FREE_WARN_GB}GB threshold"
|
|
WARNINGS+=("ARC headroom low: ${ARC_FREE_GB}GB")
|
|
else
|
|
log "ARC free ${ARC_FREE_GB}GB — within threshold ✅"
|
|
fi
|
|
|
|
echo ""
|
|
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \
|
|
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
|
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \
|
|
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
|
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
|
|
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
|
|
|
MRU_GB=$(bytes_to_gb "$META_MRU_GHOST")
|
|
MFU_GB=$(bytes_to_gb "$META_MFU_GHOST")
|
|
|
|
echo " $ICON_ZFS MRU Ghost: ${MRU_GB}GB"
|
|
echo " $ICON_ZFS MFU Ghost: ${MFU_GB}GB"
|
|
echo " $ICON_ZFS Metadata Misses: ${META_MISSES}"
|
|
fi
|
|
|
|
# ── Memory Status ─────────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_MEM Memory Status ━━━"
|
|
|
|
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
|
|
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
|
|
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
|
|
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
|
|
|
|
echo " $ICON_MEM Total RAM: $TOTAL_HUMAN"
|
|
echo " $ICON_MEM Free RAM: $FREE_HUMAN (informational — ARC intentionally uses most of this)"
|
|
echo " $ICON_MEM Available RAM: $AVAIL_HUMAN"
|
|
|
|
if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
|
|
warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
|
|
WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
|
|
else
|
|
log "Available RAM ${AVAIL_HUMAN} — within threshold ✅"
|
|
fi
|
|
|
|
# ── Docker Memory ─────────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
warn "Docker not available — skipping container memory section"
|
|
else
|
|
timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
|
--format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
|
|
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \
|
|
while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
fi
|
|
|
|
# ── Kernel Pressure ───────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"
|
|
|
|
if ! command -v vmstat >/dev/null 2>&1; then
|
|
warn "vmstat not available — skipping kernel pressure section"
|
|
else
|
|
vmstat 1 3 2>/dev/null | while IFS= read -r line; do
|
|
echo " $line"
|
|
done
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
|
|
[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
|
|
log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
|
echo ""
|
|
|
|
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE All checks within thresholds ✅"
|
|
else
|
|
echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
|
|
for w in "${WARNINGS[@]}"; do
|
|
echo " $ICON_WARN $w"
|
|
done
|
|
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \
|
|
"ZFS Report" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
|
|
exit 0
|