From ba07634a885733666fcb93a635976521193a81b2 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 11 Jul 2026 17:27:59 -0400 Subject: [PATCH] Replace system free-RAM warning with ARC headroom check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The memory warning used plain system-wide 'free' RAM (via free -g), a carryover from before this script was ZFS-specific. ZFS ARC deliberately consumes most otherwise-unused RAM, so 'free' being low is normal and not a meaningful signal — it fired a false alarm on 2026-07-05 (free 5.8Gi, but available a healthy 45Gi). Replaced ZFS_REPORT_FREE_WARN_GB with ZFS_REPORT_ARC_FREE_WARN_GB: warns when ARC headroom (ARC_MAX - ARC_CURRENT) drops below threshold, which is what actually indicates ARC is running out of room to grow. Available RAM check is unchanged — it's a legitimate system-pressure signal on its own. --- Deployment/master.conf.template | 2 +- Monitors/zfs_memory_snapshot.sh | 43 ++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 8cc76ce..7983113 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -1311,7 +1311,7 @@ # Weekly ZFS pool health and memory diagnostic report — informational only. ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max - ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM + ZFS_REPORT_ARC_FREE_WARN_GB=10 # warn if ARC headroom (max - current) drops below this GB ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory # Pool ignore list defined per host in host*.conf — pool names are server-specific: diff --git a/Monitors/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh index 1fd2c8d..216c807 100755 --- a/Monitors/zfs_memory_snapshot.sh +++ b/Monitors/zfs_memory_snapshot.sh @@ -33,6 +33,14 @@ # available, Docker not responding — those sections skip, the rest still run. # A partial report is more useful than no report. # +# ARC Headroom, Not System Free RAM +# ZFS ARC deliberately grows to use most of the RAM the system isn't otherwise +# using — that's the point of a page cache. System-wide "free" RAM being low is +# therefore normal and not a signal of anything, so the memory warning is based +# on ARC headroom (ARC_MAX - ARC_CURRENT) instead — how much room ARC itself has +# left before it hits its configured ceiling. "Available" RAM (which accounts for +# reclaimable cache) is still checked separately as a true system-pressure signal. +# # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== @@ -43,10 +51,10 @@ # ZFS_REPORT_IGNORE_POOLS excluded from the report # (still fully monitored by unRAID — report-only exclusion). # ARC statistics — current ARC vs max, metadata pressure, hit rate. -# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT. -# Memory status — total, free, available RAM. -# Warns if free < ZFS_REPORT_FREE_WARN_GB or -# available < ZFS_REPORT_AVAIL_WARN_GB. +# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT, or if +# ARC headroom (max - current) drops below ZFS_REPORT_ARC_FREE_WARN_GB. +# Memory status — total, free, available RAM (informational only — see note below). +# Warns if available < ZFS_REPORT_AVAIL_WARN_GB. # Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage. # Useful for spotting containers approaching watchdog limits. # Kernel pressure — vmstat snapshot (3 samples). @@ -102,8 +110,8 @@ # ZFS_REPORT_ARC_WARN_PCT # Warn if ARC is using more than this percentage of its configured max. (default: 90) # -# ZFS_REPORT_FREE_WARN_GB -# Warn if free RAM is below this threshold in GB. (default: 10) +# ZFS_REPORT_ARC_FREE_WARN_GB +# Warn if ARC headroom (ARC_MAX - ARC_CURRENT) drops below this many GB. (default: 10) # # ZFS_REPORT_AVAIL_WARN_GB # Warn if available RAM is below this threshold in GB. (default: 20) @@ -159,7 +167,7 @@ done log "Identity: $MY_ID ($LOCAL_SERVER_NAME)" log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" -log "$ICON_GEAR Config: arc-warn=${ZFS_REPORT_ARC_WARN_PCT}% free-warn=${ZFS_REPORT_FREE_WARN_GB}GB avail-warn=${ZFS_REPORT_AVAIL_WARN_GB}GB docker-top=${ZFS_REPORT_DOCKER_TOP}" +log "$ICON_GEAR Config: arc-warn=${ZFS_REPORT_ARC_WARN_PCT}% arc-free-warn=${ZFS_REPORT_ARC_FREE_WARN_GB}GB avail-warn=${ZFS_REPORT_AVAIL_WARN_GB}GB docker-top=${ZFS_REPORT_DOCKER_TOP}" # Tee output to log file unless dry run if [[ "$DRY_RUN" == false ]]; then @@ -178,7 +186,7 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG" echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%" - echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB" + echo "$ICON_ZFS ARC free warn: ${ZFS_REPORT_ARC_FREE_WARN_GB}GB" echo "$ICON_MEM Avail warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB" echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP" echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}" @@ -266,11 +274,13 @@ else ARC_META_GB=$(bytes_to_gb "$ARC_META_USED" 1) ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}") ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT") + ARC_FREE_GB=$(awk "BEGIN {printf \"%d\", ($ARC_MAX - $ARC_SIZE) / 1073741824}") echo " $ICON_ZFS ARC Max: ${ARC_MAX_GB}GB" echo " $ICON_ZFS ARC Current: ${ARC_CUR_GB}GB" echo " $ICON_ZFS ARC Meta Used: ${ARC_META_GB}GB" echo " $ICON_ZFS ARC Utilization: ${ARC_PCT}%" + echo " $ICON_ZFS ARC Free: ${ARC_FREE_GB}GB" if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold" @@ -279,6 +289,13 @@ else log "ARC utilization ${ARC_PCT}% — within threshold ✅" fi + if [[ "$ARC_FREE_GB" -lt "$ZFS_REPORT_ARC_FREE_WARN_GB" ]]; then + warn "ARC free ${ARC_FREE_GB}GB — below ${ZFS_REPORT_ARC_FREE_WARN_GB}GB threshold" + WARNINGS+=("ARC headroom low: ${ARC_FREE_GB}GB") + else + log "ARC free ${ARC_FREE_GB}GB — within threshold ✅" + fi + echo "" META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \ /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) @@ -302,20 +319,12 @@ echo "━━━ $ICON_MEM Memory Status ━━━" FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}') AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}') TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}') -FREE_GB=$(free -g | awk '/Mem:/ {print $4}') AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}') echo " $ICON_MEM Total RAM: $TOTAL_HUMAN" -echo " $ICON_MEM Free RAM: $FREE_HUMAN" +echo " $ICON_MEM Free RAM: $FREE_HUMAN (informational — ARC intentionally uses most of this)" echo " $ICON_MEM Available RAM: $AVAIL_HUMAN" -if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then - warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold" - WARNINGS+=("Low free RAM: ${FREE_HUMAN}") -else - log "Free RAM ${FREE_HUMAN} — within threshold ✅" -fi - if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold" WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")