240 lines
10 KiB
Bash
Executable File
240 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= inotify Tuning ============================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Raises Linux inotify limits at array start to prevent exhaustion across the
|
||
# container stack. Run once at array start via ARRAY_START_SCRIPTS. Settings
|
||
# are lost on reboot — this script reapplies them on every array start.
|
||
#
|
||
# Must run FIRST in ARRAY_START_SCRIPTS before any containers start — containers
|
||
# inherit inotify limits at launch, not dynamically. Running this after Code-Server
|
||
# starts requires a docker restart to pick up the new values.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Three inotify Limits
|
||
# max_user_instances — max independent inotify fd objects per user; each container
|
||
# calling inotify_init() consumes one. Default 128 — exhausted
|
||
# quickly with 20+ active containers.
|
||
#
|
||
# max_user_watches — SHARED budget across ALL users and containers on the system.
|
||
# Each watched file or directory costs one watch. Default 8192 —
|
||
# VSCode alone needs 50K–200K for large workspaces. Combined
|
||
# usage of Sonarr, Radarr, Lidarr, Emby, Nextcloud, Code-Server
|
||
# easily exceeds 512K on a busy server.
|
||
#
|
||
# max_queued_events — max events buffered before the kernel drops them. Low value =
|
||
# events silently lost during high-activity bursts. Default 16384.
|
||
#
|
||
# Why 1M Watches
|
||
# Raising max_user_watches to 1048576 (1M) gives sufficient headroom for all
|
||
# containers combined — safe on 128GB RAM (~128MB kernel use for the pool).
|
||
#
|
||
# Idempotent Per-Setting
|
||
# Each sysctl value is read before writing. Only changed if different from target —
|
||
# no-op on boots where limits are already correct.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root Required
|
||
# sysctl writes require root.
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock prevents duplicate runs at array start.
|
||
#
|
||
# Silent on Success
|
||
# Runs every boot — no noise when already correct.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# INOTIFY_MAX_INSTANCES
|
||
# Max inotify fd objects per user. (default: 1024)
|
||
#
|
||
# INOTIFY_MAX_WATCHES
|
||
# Max watched files/dirs shared across all users and containers. (default: 1048576)
|
||
#
|
||
# INOTIFY_MAX_QUEUED_EVENTS
|
||
# Max events buffered before kernel drops them. (default: 32768)
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# inotify_tuning.sh
|
||
# Apply inotify limits. No-op per setting if already at target.
|
||
#
|
||
# inotify_tuning.sh --dry-run
|
||
# Show current vs target for each limit. No sysctl writes.
|
||
#
|
||
# inotify_tuning.sh --status
|
||
# Show current vs target, active instance count, and top consumers by PID.
|
||
#
|
||
# inotify_tuning.sh --log
|
||
# Verbose output — show each sysctl check and result.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root — sysctl writes require root"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
acquire_lock
|
||
|
||
if ! command -v docker &>/dev/null; then
|
||
error "Docker command not found"
|
||
exit 1
|
||
fi
|
||
|
||
detect_hosts
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY INOTIFY STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
|
||
echo "━━━ Kernel Limits ━━━"
|
||
CURRENT_INSTANCES=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo "?")
|
||
CURRENT_WATCHES=$(sysctl -n fs.inotify.max_user_watches 2>/dev/null || echo "?")
|
||
CURRENT_EVENTS=$(sysctl -n fs.inotify.max_queued_events 2>/dev/null || echo "?")
|
||
|
||
for label in "max_user_instances current=$CURRENT_INSTANCES target=$INOTIFY_MAX_INSTANCES" \
|
||
"max_user_watches current=$CURRENT_WATCHES target=$INOTIFY_MAX_WATCHES" \
|
||
"max_queued_events current=$CURRENT_EVENTS target=$INOTIFY_MAX_QUEUED_EVENTS"; do
|
||
echo " $label"
|
||
done
|
||
|
||
echo ""
|
||
echo "━━━ Active Instances ━━━"
|
||
USED_INSTANCES=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
USED_INSTANCES="${USED_INSTANCES//[^0-9]/}"
|
||
echo " Instances in use: ${USED_INSTANCES:-0} / $CURRENT_INSTANCES"
|
||
if [[ "$CURRENT_INSTANCES" -gt 0 ]]; then
|
||
PCT=$(( ${USED_INSTANCES:-0} * 100 / CURRENT_INSTANCES ))
|
||
echo " Utilisation: ${PCT}%"
|
||
fi
|
||
|
||
echo ""
|
||
echo "━━━ Top Consumers ━━━"
|
||
find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
|
||
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -10 | \
|
||
while read -r count pid; do
|
||
cmd=$(cat /proc/"$pid"/comm 2>/dev/null || echo "?")
|
||
cgroup=$(cat /proc/"$pid"/cgroup 2>/dev/null | \
|
||
grep docker | grep -o '[a-f0-9]\{12\}' | head -1 || echo "")
|
||
if [[ -n "$cgroup" ]]; then
|
||
label="[docker:${cgroup}] $cmd"
|
||
else
|
||
label="[host] $cmd"
|
||
fi
|
||
echo " ${count} instances — $label (PID $pid)"
|
||
done | head -10
|
||
|
||
echo ""
|
||
echo "━━━ VSCode / Code-Server ━━━"
|
||
echo " If VSCode shows 'unable to watch for file changes':"
|
||
echo " 1. Verify max_user_watches target is set high enough"
|
||
echo " 2. Check total watches used: cat /proc/sys/fs/inotify/max_user_watches"
|
||
echo " 3. After any limit change: docker restart Code-Server"
|
||
echo " (running containers inherit limits at start, not dynamically)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Apply Settings ━━━
|
||
# ==============================================================================================
|
||
CHANGED=0
|
||
FAILED=0
|
||
|
||
apply_sysctl() {
|
||
local key="$1" value="$2"
|
||
local current
|
||
current=$(sysctl -n "$key" 2>/dev/null || echo 0)
|
||
|
||
if [[ "$current" -eq "$value" ]]; then
|
||
log "$key = $value (already correct)"
|
||
return 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would set $key = $value (currently $current)"
|
||
return 0
|
||
fi
|
||
|
||
if sysctl -w "${key}=${value}" >/dev/null 2>&1; then
|
||
warn "Set $key = $value (was $current)"
|
||
(( CHANGED++ ))
|
||
else
|
||
error "Failed to set $key = $value"
|
||
(( FAILED++ ))
|
||
fi
|
||
}
|
||
|
||
apply_sysctl "fs.inotify.max_user_instances" "$INOTIFY_MAX_INSTANCES"
|
||
apply_sysctl "fs.inotify.max_user_watches" "$INOTIFY_MAX_WATCHES"
|
||
apply_sysctl "fs.inotify.max_queued_events" "$INOTIFY_MAX_QUEUED_EVENTS"
|
||
apply_sysctl "vm.overcommit_memory" "1" # Redis: prevents background save failures under low memory
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary ━━━
|
||
# ==============================================================================================
|
||
if [[ "$FAILED" -gt 0 ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_ERROR $FAILED setting(s) failed to apply"
|
||
notify "inotify tuning failed on $(hostname) ($MY_ID) — $FAILED setting(s) could not be applied" \
|
||
"inotify Tuning" "warning"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 1
|
||
elif [[ "$CHANGED" -gt 0 ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY INOTIFY TUNING SUMMARY ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " max_user_instances: $(sysctl -n fs.inotify.max_user_instances 2>/dev/null)"
|
||
echo " max_user_watches: $(sysctl -n fs.inotify.max_user_watches 2>/dev/null)"
|
||
echo " max_queued_events: $(sysctl -n fs.inotify.max_queued_events 2>/dev/null)"
|
||
echo ""
|
||
warn "$CHANGED setting(s) updated"
|
||
if [[ "$CHANGED" -gt 0 ]]; then
|
||
warn "If Code-Server is running: docker restart Code-Server"
|
||
warn "Running containers inherit limits at start — restart picks up new values"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
else
|
||
echo "inotify limits already correct ✅"
|
||
fi
|
||
|
||
exit 0 |