Varaverk/Monitors/zfs_memory_snapshot.sh

#!/bin/bash
# ==============================================================================================
# ============================= ZFS Memory Snapshot ============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Weekly ZFS pool health and memory diagnostic report. Scheduled Sunday 6am —
# first in the Sunday monitoring block, before other scripts run. Informational
# only — system_watchdog.sh handles threshold-based intervention.
#
# Combines ZFS pool status, ARC statistics, Docker memory usage, and kernel
# memory pressure into a single snapshot. Output goes to both console (for User
# Scripts output log) and ZFS_REPORT_LOG for week-over-week comparison.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Five report sections (each skips gracefully if its data source is unavailable):
#
#   ZFS pool health   — status, state, errors per pool. Pools in
#                       ZFS_REPORT_IGNORE_POOLS excluded from the report
#                       (still fully monitored by unRAID — report-only exclusion).
#   ARC statistics    — current ARC vs max, metadata pressure, hit rate.
#                       Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT.
#   Memory status     — total, free, available RAM.
#                       Warns if free < ZFS_REPORT_FREE_WARN_GB or
#                       available < ZFS_REPORT_AVAIL_WARN_GB.
#   Docker memory     — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
#                       Useful for spotting containers approaching watchdog limits.
#   Kernel pressure   — vmstat snapshot (3 samples).
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Single Instance Lock
#   acquire_lock prevents duplicate runs — zpool and docker stats are slow.
#
# Per-Host Pool Ignore List
#   detect_hosts() aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
#   Single-disk JBOD members excluded from report noise per server.
#
# ZFS Availability Guard
#   Skips pool and ARC sections gracefully if ZFS is not available on this server.
#
# Docker Availability Guard
#   Skips container memory section gracefully if Docker is not responding.
#
# Docker Stats Timeout
#   DOCKER_TIMEOUT caps docker stats calls. A hung daemon does not block the report.
#
# Notification Validated
#   validate_unraid_cmd confirms the notify script is present before use.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
#   ZFS_REPORT_LOG   — /var/log/zfs-weekly-health.log (tmpfs, resets on reboot)
#       Weekly report written here for comparison across runs. Open the log to
#       see pool health trend week over week without remembering last week's values.
#       In dry-run mode, console only — nothing written.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master_host*.conf
#
#   HOST*_ZFS_REPORT_IGNORE_POOLS
#       Pools excluded from health reporting. Single-disk JBOD members generate
#       expected high-usage warnings — exclude them to reduce report noise.
#       Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS.
#
# master.conf
#
#   ZFS_REPORT_LOG
#       Log file path for weekly reports. (default: /var/log/zfs-weekly-health.log)
#
#   ZFS_REPORT_ARC_WARN_PCT
#       Warn if ARC is using more than this percentage of its configured max. (default: 90)
#
#   ZFS_REPORT_FREE_WARN_GB
#       Warn if free RAM is below this threshold in GB. (default: 10)
#
#   ZFS_REPORT_AVAIL_WARN_GB
#       Warn if available RAM is below this threshold in GB. (default: 20)
#
#   ZFS_REPORT_DOCKER_TOP
#       Number of top Docker containers by memory usage to include. (default: 10)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# zfs_memory_snapshot.sh
#     Generate report, write to ZFS_REPORT_LOG and console. Notify on warnings.
#
# zfs_memory_snapshot.sh --dry-run
#     Generate report to console only. No log write, no notifications.
#
# zfs_memory_snapshot.sh --status
#     Show pool ignore list and threshold configuration. Then exit.
#
# zfs_memory_snapshot.sh --log
#     Verbose output during report generation.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Monitor/report script — output is the point
SILENT_MODE=false

parse_args "$@"

DOCKER_TIMEOUT=15

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
detect_hosts

# Build ignore pool lookup map — O(1) check per pool
declare -A IGNORE_POOL_MAP
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
    [[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
done

log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"

# Tee output to log file unless dry run
if [[ "$DRY_RUN" == false ]]; then
    mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
    exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_ZFS        Log file:       $ZFS_REPORT_LOG"
    echo "$ICON_ZFS        ARC warn:       ${ZFS_REPORT_ARC_WARN_PCT}%"
    echo "$ICON_MEM        Free RAM warn:  ${ZFS_REPORT_FREE_WARN_GB}GB"
    echo "$ICON_MEM        Avail warn:     ${ZFS_REPORT_AVAIL_WARN_GB}GB"
    echo "$ICON_CONTAINERS Docker top:     $ZFS_REPORT_DOCKER_TOP"
    echo "$ICON_ZFS        Ignore pools:   ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
    echo "$ICON_GEAR       Dry Run:        $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Report ━━━
# ==============================================================================================
WARNINGS=()
START=$(date +%s)
DATE=$(date '+%Y-%m-%d %H:%M:%S')

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_ZFS  ZFS WEEKLY HEALTH REPORT — $DATE"
echo "  $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# ── ZFS Pool Health ───────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"

if ! command -v zpool >/dev/null 2>&1; then
    warn "ZFS not available on this system — skipping pool checks"
else
    # Pool status — filtered to key lines, ignoring specified pools
    CURRENT_POOL=""
    while IFS= read -r line; do
        if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
            CURRENT_POOL="${BASH_REMATCH[1]// /}"
        fi
        [[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
        echo "  $line"
    done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")

    echo ""

    # Pool list — filter out ignored pools
    zpool list 2>/dev/null | while IFS= read -r line; do
        if [[ "$line" == NAME* ]]; then
            echo "  $line"
            continue
        fi
        pool_name=$(echo "$line" | awk '{print $1}')
        [[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
        echo "  $line"
    done

    # Check for unhealthy non-ignored pools
    UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \
        while IFS=$'\t' read -r name health; do
            [[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
            [[ "$health" != "ONLINE" ]] && echo "$name: $health"
        done)

    if [[ -n "$UNHEALTHY" ]]; then
        error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
        WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
    else
        log "All monitored ZFS pools are ONLINE ✅"
    fi

    if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
        log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
    fi
fi

# ── ARC Statistics ────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_ZFS ARC Statistics ━━━"

if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then
    warn "ZFS arcstats not available — skipping ARC section"
else
    ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || \
              awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)

    ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
    ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
    ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
    ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
    ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")

    echo "  $ICON_ZFS ARC Max:         ${ARC_MAX_GB}GB"
    echo "  $ICON_ZFS ARC Current:     ${ARC_CUR_GB}GB"
    echo "  $ICON_ZFS ARC Meta Used:   ${ARC_META_GB}GB"
    echo "  $ICON_ZFS ARC Utilization: ${ARC_PCT}%"

    if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then
        warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
        WARNINGS+=("ARC high: ${ARC_PCT}%")
    else
        log "ARC utilization ${ARC_PCT}% — within threshold ✅"
    fi

    echo ""
    META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \
        /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
    META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \
        /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
    META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
        /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)

    MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
    MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")

    echo "  $ICON_ZFS MRU Ghost:       ${MRU_GB}GB"
    echo "  $ICON_ZFS MFU Ghost:       ${MFU_GB}GB"
    echo "  $ICON_ZFS Metadata Misses: ${META_MISSES}"
fi

# ── Memory Status ─────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_MEM Memory Status ━━━"

FREE_HUMAN=$(free -h  | awk '/Mem:/ {print $4}')
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
FREE_GB=$(free -g     | awk '/Mem:/ {print $4}')
AVAIL_GB=$(free -g    | awk '/Mem:/ {print $7}')

echo "  $ICON_MEM Total RAM:      $TOTAL_HUMAN"
echo "  $ICON_MEM Free RAM:       $FREE_HUMAN"
echo "  $ICON_MEM Available RAM:  $AVAIL_HUMAN"

if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
    warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
    WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
else
    log "Free RAM ${FREE_HUMAN} — within threshold ✅"
fi

if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
    warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
    WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
else
    log "Available RAM ${AVAIL_HUMAN} — within threshold ✅"
fi

# ── Docker Memory ─────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"

if ! command -v docker >/dev/null 2>&1; then
    warn "Docker not available — skipping container memory section"
else
    timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
        --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
        2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \
        while IFS= read -r line; do
            echo "  $line"
        done
fi

# ── Kernel Pressure ───────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"

if ! command -v vmstat >/dev/null 2>&1; then
    warn "vmstat not available — skipping kernel pressure section"
else
    vmstat 1 3 2>/dev/null | while IFS= read -r line; do
        echo "  $line"
    done
fi

END=$(date +%s)

# ── Summary ───────────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
echo "$ICON_ZFS     Log:      $ZFS_REPORT_LOG"
[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
    log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
echo ""

if [[ ${#WARNINGS[@]} -eq 0 ]]; then
    log "$ICON_DONE All checks within thresholds ✅"
else
    echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
    for w in "${WARNINGS[@]}"; do
        echo "  $ICON_WARN $w"
    done
    notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \
        "ZFS Report" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
exit 0