#!/bin/bash
# ==============================================================================================
# ============================= System Tuning Monitor ==========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# inotify and PHP-FPM utilisation tracking via time-series snapshots. Scheduled
# every 6 hours (0 */6 * * *). Writes one bounded log entry per run to
# TUNING_MONITOR_LOG. weekly_health_digest.sh reads this log to report peak,
# average, and warning counts over the week.
#
# Background snapshot script — no output when healthy. Warns (and notifies) only
# when INOTIFY_WARN_PCT or PHP_FPM_WARN_PCT thresholds are exceeded.
#
# inotify exhaustion symptoms: downloads complete but arrs don't detect them,
# live TV stutter, library updates stop. PHP-FPM exhaustion symptoms: unRAID
# WebGUI slowdowns or timeouts under load.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Each run snapshots:
#   inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit.
#            Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT.
#   php-fpm: active workers vs PHP_MAX_CHILDREN limit.
#            Warns above PHP_FPM_WARN_PCT.
#
# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days):
#   DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|
#   PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts
# warnings over the week to show trend severity.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Snapshot for Trend, Not Just Alert
#   Exhaustion events are rarely instant — they build over hours or days.
#   Logging every 6 hours builds a trend that weekly_health_digest.sh can
#   surface as a warning count, catching gradual pressure before it becomes
#   an outage.
#
# Bounded Log Size
#   Log entries are trimmed to TUNING_LOG_RETENTION days on every write.
#   The log never grows unbounded regardless of how long the server runs.
#
# Silent When Healthy
#   No output, no notification on a clean run. Threshold breach is the only
#   signal — routine snapshots below the threshold produce nothing.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Single Instance Lock
#   acquire_lock prevents overlapping 6-hour snapshot runs.
#
# Root Enforcement
#   /proc/*/fd enumeration for inotify consumer counting requires root access.
#
# Atomic Log Write
#   Trim uses tmp file + mv — partial writes during log rotation cannot corrupt
#   the accumulated history.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
#   TUNING_MONITOR_LOG   — DATA_DIR/tuning_monitor.db
#       One line per 6-hour snapshot. Trimmed to TUNING_LOG_RETENTION days on
#       each write — bounded size. Read by weekly_health_digest.sh for trend
#       reporting. Resets if DATA_DIR is cleared.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   INOTIFY_WARN_PCT
#       Warn if inotify instances exceed this percentage of the kernel limit. (default: 80)
#
#   PHP_FPM_WARN_PCT
#       Warn if php-fpm active workers exceed this percentage of PHP_MAX_CHILDREN. (default: 80)
#
#   PHP_MAX_CHILDREN
#       Maximum php-fpm workers — set by php_fpm_max_children.sh in System_Essentials.
#
#   TUNING_MONITOR_LOG
#       Log file path. (default: DATA_DIR/tuning_monitor.db)
#
#   TUNING_LOG_RETENTION
#       Days before old entries are purged. (default: 30)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# system_tuning_monitor.sh
#     Take snapshot. Write to log. Warn (and notify) if thresholds exceeded.
#     Silent when healthy.
#
# system_tuning_monitor.sh --dry-run
#     Measure inotify and PHP-FPM utilisation and display results. No log write.
#
# system_tuning_monitor.sh --status
#     Show thresholds, log path, and retention. Then exit.
#
# system_tuning_monitor.sh --log
#     Verbose output during the snapshot including top inotify consumers.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root — /proc/*/fd requires root access"
    exit 1
fi


acquire_lock

# detect_hosts() sets MY_ID — used in warning output
detect_hosts

log "$ICON_GEAR Config: inotify-warn=${INOTIFY_WARN_PCT:-80}% php-fpm-warn=${PHP_FPM_WARN_PCT:-80}% max-workers=${PHP_MAX_CHILDREN:-250} retention=${TUNING_LOG_RETENTION:-30}d"

DATE=$(date '+%Y-%m-%d')
TIME=$(date '+%H:%M')
INOTIFY_WARN=0
PHPFPM_WARN=0

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST   Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR   inotify warn:   ${INOTIFY_WARN_PCT:-80}%"
    echo "$ICON_GEAR   php-fpm warn:   ${PHP_FPM_WARN_PCT:-80}%"
    echo "$ICON_GEAR   php-fpm max:    ${PHP_MAX_CHILDREN:-250}"
    echo "$ICON_GEAR   Log file:       ${TUNING_MONITOR_LOG:-not set}"
    echo "$ICON_GEAR   Retention:      ${TUNING_LOG_RETENTION:-30} days"
    echo "$ICON_GEAR   Dry Run:        $DRY_RUN"

    if [[ -f "$TUNING_MONITOR_LOG" ]]; then
        ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
        OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
        NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
        echo "$ICON_MONITOR   Log entries:    $ENTRY_COUNT ($OLDEST → $NEWEST)"
    else
        echo "$ICON_MONITOR   Log entries:    none yet"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"

# ==============================================================================================
# ━━━ inotify ━━━
# ==============================================================================================
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
INOTIFY_USED="${INOTIFY_USED:-0}"

if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
    INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
else
    INOTIFY_PCT=0
fi

[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1

# Top 5 inotify consumers
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
    awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
    while read -r count pid; do
        comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
        echo "${count}×${comm}"
    done | tr '\n' ',' | sed 's/,$//')

if [[ "$INOTIFY_WARN" -eq 1 ]]; then
    warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
    warn "Top consumers: ${INOTIFY_TOP:-unknown}"
    warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
    notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
        "System Tuning" "warning"
else
    log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
    log "inotify top consumers: ${INOTIFY_TOP:-none}"
fi

# ==============================================================================================
# ━━━ php-fpm ━━━
# ==============================================================================================
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"

PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || true)
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"

if [[ "$PHPFPM_MAX" -gt 0 ]]; then
    PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
else
    PHPFPM_PCT=0
fi

[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1

if [[ "$PHPFPM_WARN" -eq 1 ]]; then
    warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
    warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
    notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
        "System Tuning" "warning"
else
    log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
fi

# ==============================================================================================
# ━━━ Write Snapshot ━━━
# ==============================================================================================
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
    warn "TUNING_MONITOR_LOG not set — snapshot not written"
    exit 0
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — snapshot not written"
    warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
    exit 0
fi

mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"

# Trim old entries — atomic write via temp file
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
    CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
    awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
        "$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
        mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
fi

# Append snapshot
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
    >> "$TUNING_MONITOR_LOG"

echo "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
log "Entry: ${DATE}|${TIME}|${INOTIFY_USED}/${INOTIFY_LIMIT}(${INOTIFY_PCT}%,warn=${INOTIFY_WARN})|${PHPFPM_ACTIVE}/${PHPFPM_MAX}(${PHPFPM_PCT}%,warn=${PHPFPM_WARN})"