#!/bin/bash
# ==============================================================================================
# ================================= inotify Tuning ============================================
# ==============================================================================================
# Raises Linux inotify limits at array start to prevent exhaustion across the container stack.
# Run once at array start via ARRAY_START_SCRIPTS in master.conf.
# Settings are lost on reboot — this script reapplies them on every array start.
#
# ── THREE INOTIFY LIMITS ──────────────────────────────────────────────────────────────────────
#   max_user_instances  — max number of independent inotify file descriptor objects per user
#                         Each container that calls inotify_init() consumes one instance
#                         Default 128 — exhausted quickly with 20+ active containers
#
#   max_user_watches    — SHARED budget across ALL users and containers on the system
#                         Each watched file or directory costs one watch from this pool
#                         Default 8192 — VSCode alone can need 50K-200K for large workspaces
#
#   max_queued_events   — max events buffered before kernel starts dropping them
#                         Low value = events silently lost during high-activity periods
#                         Default 16384 — sufficient for most setups
#
# ── WHY VSCODE THROWS "UNABLE TO WATCH FOR FILE CHANGES" ─────────────────────────────────────
# VSCode (and Code-Server in Docker) opens one inotify watch per file in the workspace.
# A typical project with node_modules can easily have 100K-200K files.
# All containers on the host share max_user_watches — the combined usage of:
#   Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server, AdGuard, all other arrs
#   easily exceeds 524288 (512K) watches on a busy server.
# Raising to 1048576 (1M) gives sufficient headroom — safe on 128GB RAM (~128MB kernel use).
#
# ── STARTUP ORDER MATTERS ─────────────────────────────────────────────────────────────────────
# inotify_tuning.sh must run BEFORE containers that watch files start.
# In ARRAY_START_SCRIPTS order: inotify_tuning.sh first, then container-starting scripts.
# If Code-Server starts before limits are raised it inherits the old (low) limits.
# Code-Server restart fixes this: limits are kernel-wide, not process-bound at start.
# So if Code-Server is already running: docker restart Code-Server after this script runs.
#
# ── CONSUMERS ON THIS STACK ───────────────────────────────────────────────────────────────────
#   Emby           — watches all media library paths (1 watch per folder)
#   Sonarr         — watches TV_Shows folder tree
#   Radarr         — watches Movies folder tree
#   Lidarr         — watches Music folder tree
#   Nextcloud      — watches data directory for changes
#   Code-Server    — watches entire workspace (can be 50K-200K with node_modules)
#   AdGuard Home   — watches config directory
#   + all other containers using inotify internally
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   acquire_lock     — prevents duplicate runs at array start
#   Root check       — sysctl writes require root
#   validate_unraid  — notify validated before use
#   Silent on success — runs every boot, no noise when already correct
#   Only warns on changes or failures
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   INOTIFY_MAX_INSTANCES     — default 1024
#   INOTIFY_MAX_WATCHES       — default 1048576 (1M)
#   INOTIFY_MAX_QUEUED_EVENTS — default 32768
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   inotify_tuning.sh              — normal run (apply settings)
#   inotify_tuning.sh --dry-run    — show what would change
#   inotify_tuning.sh --status     — show current vs target values and top consumers
#   inotify_tuning.sh --log        — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root — sysctl writes require root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY INOTIFY STATUS ━━━━━"
    echo "$ICON_HOST Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""

    echo "━━━ Kernel Limits ━━━"
    CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?")
    CURRENT_WATCHES=$(sysctl  -n fs.inotify.max_user_watches    2>/dev/null || echo "?")
    CURRENT_EVENTS=$(sysctl   -n fs.inotify.max_queued_events   2>/dev/null || echo "?")

    for label in "max_user_instances  current=$CURRENT_INSTANCES target=$INOTIFY_MAX_INSTANCES" \
                 "max_user_watches    current=$CURRENT_WATCHES  target=$INOTIFY_MAX_WATCHES" \
                 "max_queued_events   current=$CURRENT_EVENTS   target=$INOTIFY_MAX_QUEUED_EVENTS"; do
        echo "  $label"
    done

    echo ""
    echo "━━━ Active Instances ━━━"
    USED_INSTANCES=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
    USED_INSTANCES="${USED_INSTANCES//[^0-9]/}"
    echo "  Instances in use: ${USED_INSTANCES:-0} / $CURRENT_INSTANCES"
    if [[ "$CURRENT_INSTANCES" -gt 0 ]]; then
        PCT=$(( ${USED_INSTANCES:-0} * 100 / CURRENT_INSTANCES ))
        echo "  Utilisation: ${PCT}%"
    fi

    echo ""
    echo "━━━ Top Consumers ━━━"
    find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
        awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -10 | \
        while read -r count pid; do
            cmd=$(cat /proc/"$pid"/comm 2>/dev/null || echo "?")
            cgroup=$(cat /proc/"$pid"/cgroup 2>/dev/null | \
                grep docker | grep -o '[a-f0-9]\{12\}' | head -1 || echo "")
            if [[ -n "$cgroup" ]]; then
                label="[docker:${cgroup}] $cmd"
            else
                label="[host] $cmd"
            fi
            echo "  ${count} instances — $label (PID $pid)"
        done | head -10

    echo ""
    echo "━━━ VSCode / Code-Server ━━━"
    echo "  If VSCode shows 'unable to watch for file changes':"
    echo "  1. Verify max_user_watches target is set high enough"
    echo "  2. Check total watches used: cat /proc/sys/fs/inotify/max_user_watches"
    echo "  3. After any limit change: docker restart Code-Server"
    echo "     (running containers inherit limits at start, not dynamically)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Apply Settings ━━━
# ==============================================================================================
CHANGED=0
FAILED=0

apply_sysctl() {
    local key="$1" value="$2"
    local current
    current=$(sysctl -n "$key" 2>/dev/null || echo 0)

    if [[ "$current" -eq "$value" ]]; then
        log "$key = $value (already correct)"
        return 0
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would set $key = $value (currently $current)"
        return 0
    fi

    if sysctl -w "${key}=${value}" >/dev/null 2>&1; then
        warn "Set $key = $value (was $current)"
        (( CHANGED++ ))
    else
        error "Failed to set $key = $value"
        (( FAILED++ ))
    fi
}

apply_sysctl "fs.inotify.max_user_instances" "$INOTIFY_MAX_INSTANCES"
apply_sysctl "fs.inotify.max_user_watches"   "$INOTIFY_MAX_WATCHES"
apply_sysctl "fs.inotify.max_queued_events"  "$INOTIFY_MAX_QUEUED_EVENTS"

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$FAILED" -gt 0 ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
    echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_ERROR $FAILED setting(s) failed to apply"
    notify "inotify tuning failed on $(hostname) ($MY_ID) — $FAILED setting(s) could not be applied" \
        "inotify Tuning" "warning"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 1
elif [[ "$CHANGED" -gt 0 ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
    echo "$ICON_HOST Identity:          $MY_ID ($LOCAL_SERVER_NAME)"
    echo "  max_user_instances:  $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)"
    echo "  max_user_watches:    $(sysctl -n fs.inotify.max_user_watches   2>/dev/null)"
    echo "  max_queued_events:   $(sysctl -n fs.inotify.max_queued_events  2>/dev/null)"
    echo ""
    warn "$CHANGED setting(s) updated"
    if [[ "$CHANGED" -gt 0 ]]; then
        warn "If Code-Server is running: docker restart Code-Server"
        warn "Running containers inherit limits at start — restart picks up new values"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
    # Already correct — completely silent (runs every boot)
    log "inotify limits already correct — no changes needed"
fi

exit 0