#!/bin/bash
# ==============================================================================================
# ============================= System Tuning Monitor ==========================================
# ==============================================================================================
# Tracks inotify and php-fpm usage over time.
# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary.
# Schedule: 0 */6 * * * (every 6 hours via User Scripts)
#
# ── WHAT IT TRACKS ────────────────────────────────────────────────────────────────────────────
#   inotify instances:
#     Current in use vs kernel limit
#     % utilization — warns above INOTIFY_WARN_PCT (default 80%)
#     Top 5 consumers by instance count
#     Symptom of exhaustion: containers miss file events, downloads not detected,
#     Live TV stutter, library not updated
#
#   php-fpm workers:
#     Active workers vs PHP_MAX_CHILDREN limit
#     % utilization — warns above PHP_FPM_WARN_PCT (default 80%)
#     Symptom: unRAID WebGUI slowdowns or timeouts under load
#
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
#   DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
#   Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size.
#
# ── WHAT THE WEEKLY REPORT SHOWS ──────────────────────────────────────────────────────────────
#   inotify: peak, average, warning count over the week
#   php-fpm: peak workers, average workers, warning count over the week
#
# ── SILENT BY DEFAULT ─────────────────────────────────────────────────────────────────────────
# Background snapshot script — no output when healthy.
# Warns to stderr when thresholds exceeded — visible in User Scripts output log.
# Does NOT notify on every snapshot — only when threshold exceeded.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# Each server writes to its own DATA_DIR — no collision between servers.
# MY_ID included in warning output for clarity in shared notification channels.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   acquire_lock     — prevents overlapping 6-hour snapshots
#   root check       — /proc/*/fd requires root access
#   atomic log write — tmp file + mv prevents partial writes on trim
#   validate_unraid  — notify script validated before use
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   INOTIFY_WARN_PCT      — warn threshold % (default 80)
#   PHP_FPM_WARN_PCT      — warn threshold % (default 80)
#   PHP_MAX_CHILDREN      — max php-fpm workers (set by php_fpm_max_children.sh)
#   TUNING_MONITOR_LOG    — log file path
#   TUNING_LOG_RETENTION  — days before old entries purged (default 30)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   system_tuning_monitor.sh              — normal snapshot run
#   system_tuning_monitor.sh --dry-run    — measure and show, no log write
#   system_tuning_monitor.sh --log        — verbose output
#   system_tuning_monitor.sh --status     — show config and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root — /proc/*/fd requires root access"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

# detect_hosts() sets MY_ID — used in warning output
detect_hosts

DATE=$(date '+%Y-%m-%d')
TIME=$(date '+%H:%M')
INOTIFY_WARN=0
PHPFPM_WARN=0

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST   Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR   inotify warn:   ${INOTIFY_WARN_PCT:-80}%"
    echo "$ICON_GEAR   php-fpm warn:   ${PHP_FPM_WARN_PCT:-80}%"
    echo "$ICON_GEAR   php-fpm max:    ${PHP_MAX_CHILDREN:-250}"
    echo "$ICON_GEAR   Log file:       ${TUNING_MONITOR_LOG:-not set}"
    echo "$ICON_GEAR   Retention:      ${TUNING_LOG_RETENTION:-30} days"
    echo "$ICON_GEAR   Dry Run:        $DRY_RUN"

    if [[ -f "$TUNING_MONITOR_LOG" ]]; then
        ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
        OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
        NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
        echo "$ICON_MONITOR   Log entries:    $ENTRY_COUNT ($OLDEST → $NEWEST)"
    else
        echo "$ICON_MONITOR   Log entries:    none yet"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"

# ==============================================================================================
# ━━━ inotify ━━━
# ==============================================================================================
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
INOTIFY_USED="${INOTIFY_USED:-0}"

if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
    INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
else
    INOTIFY_PCT=0
fi

[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1

# Top 5 inotify consumers
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
    awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
    while read -r count pid; do
        comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
        echo "${count}×${comm}"
    done | tr '\n' ',' | sed 's/,$//')

if [[ "$INOTIFY_WARN" -eq 1 ]]; then
    warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
    warn "Top consumers: ${INOTIFY_TOP:-unknown}"
    warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
    notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
        "System Tuning" "warning"
else
    log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
    log "inotify top consumers: ${INOTIFY_TOP:-none}"
fi

# ==============================================================================================
# ━━━ php-fpm ━━━
# ==============================================================================================
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"

PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"

if [[ "$PHPFPM_MAX" -gt 0 ]]; then
    PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
else
    PHPFPM_PCT=0
fi

[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1

if [[ "$PHPFPM_WARN" -eq 1 ]]; then
    warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
    warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
    notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
        "System Tuning" "warning"
else
    log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
fi

# ==============================================================================================
# ━━━ Write Snapshot ━━━
# ==============================================================================================
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
    warn "TUNING_MONITOR_LOG not set — snapshot not written"
    exit 0
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — snapshot not written"
    warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
    exit 0
fi

mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"

# Trim old entries — atomic write via temp file
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
    CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
    awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
        "$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
        mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
fi

# Append snapshot
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
    >> "$TUNING_MONITOR_LOG"

log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"