#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- ZFS Memory Snapshot ----------------------------------------
# -----------------------------------------------------------------------------------------------
# Weekly ZFS pool health and memory diagnostic report.
# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage
# and kernel pressure into a single report. Informational only — no action taken.
# system_watchdog.sh handles threshold-based intervention.
#
# Output goes to both console and ZFS_REPORT_LOG for later review.
# Notifies if any warning thresholds are exceeded.
#
# Pools listed in ZFS_REPORT_IGNORE_POOLS are excluded from health reporting.
# Useful for pools expected to run at high usage (docker, cache etc.)
#
# All configuration in Master.conf under ZFS Memory Snapshot section.
# Supports --dry-run (preview only, no log write) and --status.
# -----------------------------------------------------------------------------------------------

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"

parse_args "$@"

# Tee output to log file unless dry run
if [[ "$DRY_RUN" == false ]]; then
    mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
    exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"

if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

success "Running as root"

# Build ignore list for quick lookup
declare -A IGNORE_POOL_MAP
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
    [[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
done

if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
    info "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_ZFS  Log file:         $ZFS_REPORT_LOG"
    echo "$ICON_ZFS  ARC warn:         ${ZFS_REPORT_ARC_WARN_PCT}%"
    echo "$ICON_MEM  Free RAM warn:    ${ZFS_REPORT_FREE_WARN_GB}GB"
    echo "$ICON_MEM  Avail RAM warn:   ${ZFS_REPORT_AVAIL_WARN_GB}GB"
    echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
    echo "$ICON_ZFS  Ignore pools:     ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
    echo "$ICON_GEAR Dry Run:          $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"

# -----------------------------------------------------------------------------------------------
# Tracking
# -----------------------------------------------------------------------------------------------
WARNINGS=()
START=$(date +%s)
DATE=$(date +"%Y-%m-%d %H:%M:%S")

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  $ICON_ZFS  ZFS WEEKLY HEALTH REPORT — $DATE"
echo "  $ICON_HOST Host: $(hostname)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS ZFS Pool Health ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"

if ! command -v zpool >/dev/null 2>&1; then
    warn "ZFS not available on this system — skipping pool checks"
else
    # Pool status — filtered to key lines, ignoring specified pools
    info "Pool status:"
    while IFS= read -r line; do
        # Extract pool name from "pool: poolname" lines
        if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
            CURRENT_POOL="${BASH_REMATCH[1]// /}"
        fi
        # Skip lines belonging to ignored pools
        [[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
        echo "  $line"
    done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")

    echo ""

    # Pool list — filter out ignored pools
    info "Pool overview:"
    zpool list 2>/dev/null | while IFS= read -r line; do
        # Always show header line
        if [[ "$line" == NAME* ]]; then
            echo "  $line"
            continue
        fi
        # Extract pool name (first field)
        pool_name=$(echo "$line" | awk '{print $1}')
        [[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
        echo "  $line"
    done

    # Check for unhealthy pools — excluding ignored ones
    UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | while IFS=$'\t' read -r name health; do
        [[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
        [[ "$health" != "ONLINE" ]] && echo "$name: $health"
    done)

    if [[ -n "$UNHEALTHY" ]]; then
        error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
        WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
    else
        success "All monitored ZFS pools are ONLINE"
    fi

    # Show ignored pools
    if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
        info "Ignored pools (not reported): ${ZFS_REPORT_IGNORE_POOLS[*]}"
    fi
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_ZFS ARC Statistics ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_ZFS ARC Statistics ━━━"

if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then
    warn "ZFS arcstats not available — skipping ARC section"
else
    ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || \
              awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
    ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)

    ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
    ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
    ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
    ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
    ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")

    echo "  $ICON_ZFS ARC Max:         ${ARC_MAX_GB}GB"
    echo "  $ICON_ZFS ARC Current:     ${ARC_CUR_GB}GB"
    echo "  $ICON_ZFS ARC Meta Used:   ${ARC_META_GB}GB"
    echo "  $ICON_ZFS ARC Utilization: ${ARC_PCT}%"

    if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then
        warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
        WARNINGS+=("ARC high: ${ARC_PCT}%")
    else
        success "ARC utilization ${ARC_PCT}% — within threshold (${ZFS_REPORT_ARC_WARN_PCT}%)"
    fi

    echo ""
    info "Metadata pressure:"
    META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
    META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
    META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)

    MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
    MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")

    echo "  $ICON_ZFS MRU Ghost:       ${MRU_GB}GB"
    echo "  $ICON_ZFS MFU Ghost:       ${MFU_GB}GB"
    echo "  $ICON_ZFS Metadata Misses: ${META_MISSES}"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_MEM Memory Status ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_MEM Memory Status ━━━"

FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')

echo "  $ICON_MEM Total RAM:      $TOTAL_HUMAN"
echo "  $ICON_MEM Free RAM:       $FREE_HUMAN"
echo "  $ICON_MEM Available RAM:  $AVAIL_HUMAN"

if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
    warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
    WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
else
    success "Free RAM ${FREE_HUMAN} — within threshold (${ZFS_REPORT_FREE_WARN_GB}GB)"
fi

if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
    warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
    WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
else
    success "Available RAM ${AVAIL_HUMAN} — within threshold (${ZFS_REPORT_AVAIL_WARN_GB}GB)"
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CONTAINERS Docker Memory ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"

if ! command -v docker >/dev/null 2>&1; then
    warn "Docker not available — skipping container memory section"
else
    docker stats --no-stream \
        --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
        2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do
        echo "  $line"
    done
fi

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Kernel Pressure ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"

if ! command -v vmstat >/dev/null 2>&1; then
    warn "vmstat not available — skipping kernel pressure section"
else
    info "vmstat snapshot (3 samples):"
    vmstat 1 3 2>/dev/null | while IFS= read -r line; do
        echo "  $line"
    done
fi

END=$(date +%s)

# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
echo "$ICON_TIME Duration:  $(format_duration $((END - START)))"
echo "$ICON_ZFS  Log:       $ZFS_REPORT_LOG"
[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
    echo "$ICON_ZFS  Ignored:   ${ZFS_REPORT_IGNORE_POOLS[*]}"
echo ""
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
    success "Report complete — no warnings"
else
    echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
    for w in "${WARNINGS[@]}"; do
        echo "  $ICON_WARN $w"
    done
    notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"