205 lines
10 KiB
Bash
205 lines
10 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= System Tuning Monitor ==========================================
|
||
# ==============================================================================================
|
||
# Tracks inotify and php-fpm usage over time.
|
||
# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary.
|
||
# Schedule: 0 */6 * * * (every 6 hours via User Scripts)
|
||
#
|
||
# ── WHAT IT TRACKS ────────────────────────────────────────────────────────────────────────────
|
||
# inotify instances:
|
||
# Current in use vs kernel limit
|
||
# % utilization — warns above INOTIFY_WARN_PCT (default 80%)
|
||
# Top 5 consumers by instance count
|
||
# Symptom of exhaustion: containers miss file events, downloads not detected,
|
||
# Live TV stutter, library not updated
|
||
#
|
||
# php-fpm workers:
|
||
# Active workers vs PHP_MAX_CHILDREN limit
|
||
# % utilization — warns above PHP_FPM_WARN_PCT (default 80%)
|
||
# Symptom: unRAID WebGUI slowdowns or timeouts under load
|
||
#
|
||
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
|
||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||
# Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size.
|
||
#
|
||
# ── WHAT THE WEEKLY REPORT SHOWS ──────────────────────────────────────────────────────────────
|
||
# inotify: peak, average, warning count over the week
|
||
# php-fpm: peak workers, average workers, warning count over the week
|
||
#
|
||
# ── SILENT BY DEFAULT ─────────────────────────────────────────────────────────────────────────
|
||
# Background snapshot script — no output when healthy.
|
||
# Warns to stderr when thresholds exceeded — visible in User Scripts output log.
|
||
# Does NOT notify on every snapshot — only when threshold exceeded.
|
||
#
|
||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||
# Each server writes to its own DATA_DIR — no collision between servers.
|
||
# MY_ID included in warning output for clarity in shared notification channels.
|
||
#
|
||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||
# acquire_lock — prevents overlapping 6-hour snapshots
|
||
# root check — /proc/*/fd requires root access
|
||
# atomic log write — tmp file + mv prevents partial writes on trim
|
||
# validate_unraid — notify script validated before use
|
||
#
|
||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||
# INOTIFY_WARN_PCT — warn threshold % (default 80)
|
||
# PHP_FPM_WARN_PCT — warn threshold % (default 80)
|
||
# PHP_MAX_CHILDREN — max php-fpm workers (set by php_fpm_max_children.sh)
|
||
# TUNING_MONITOR_LOG — log file path
|
||
# TUNING_LOG_RETENTION — days before old entries purged (default 30)
|
||
#
|
||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||
# system_tuning_monitor.sh — normal snapshot run
|
||
# system_tuning_monitor.sh --dry-run — measure and show, no log write
|
||
# system_tuning_monitor.sh --log — verbose output
|
||
# system_tuning_monitor.sh --status — show config and exit
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root — /proc/*/fd requires root access"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
acquire_lock
|
||
|
||
# detect_hosts() sets MY_ID — used in warning output
|
||
detect_hosts
|
||
|
||
DATE=$(date '+%Y-%m-%d')
|
||
TIME=$(date '+%H:%M')
|
||
INOTIFY_WARN=0
|
||
PHPFPM_WARN=0
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_GEAR inotify warn: ${INOTIFY_WARN_PCT:-80}%"
|
||
echo "$ICON_GEAR php-fpm warn: ${PHP_FPM_WARN_PCT:-80}%"
|
||
echo "$ICON_GEAR php-fpm max: ${PHP_MAX_CHILDREN:-250}"
|
||
echo "$ICON_GEAR Log file: ${TUNING_MONITOR_LOG:-not set}"
|
||
echo "$ICON_GEAR Retention: ${TUNING_LOG_RETENTION:-30} days"
|
||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||
|
||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||
ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
|
||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
|
||
NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
|
||
echo "$ICON_MONITOR Log entries: $ENTRY_COUNT ($OLDEST → $NEWEST)"
|
||
else
|
||
echo "$ICON_MONITOR Log entries: none yet"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ inotify ━━━
|
||
# ==============================================================================================
|
||
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
|
||
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
|
||
INOTIFY_USED="${INOTIFY_USED:-0}"
|
||
|
||
if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
|
||
INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
|
||
else
|
||
INOTIFY_PCT=0
|
||
fi
|
||
|
||
[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1
|
||
|
||
# Top 5 inotify consumers
|
||
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
|
||
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
|
||
while read -r count pid; do
|
||
comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
|
||
echo "${count}×${comm}"
|
||
done | tr '\n' ',' | sed 's/,$//')
|
||
|
||
if [[ "$INOTIFY_WARN" -eq 1 ]]; then
|
||
warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
|
||
warn "Top consumers: ${INOTIFY_TOP:-unknown}"
|
||
warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
|
||
notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
|
||
"System Tuning" "warning"
|
||
else
|
||
log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
|
||
log "inotify top consumers: ${INOTIFY_TOP:-none}"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ php-fpm ━━━
|
||
# ==============================================================================================
|
||
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"
|
||
|
||
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
|
||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"
|
||
|
||
if [[ "$PHPFPM_MAX" -gt 0 ]]; then
|
||
PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
|
||
else
|
||
PHPFPM_PCT=0
|
||
fi
|
||
|
||
[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1
|
||
|
||
if [[ "$PHPFPM_WARN" -eq 1 ]]; then
|
||
warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
|
||
warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
|
||
notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
|
||
"System Tuning" "warning"
|
||
else
|
||
log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Write Snapshot ━━━
|
||
# ==============================================================================================
|
||
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
|
||
warn "TUNING_MONITOR_LOG not set — snapshot not written"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — snapshot not written"
|
||
warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
|
||
exit 0
|
||
fi
|
||
|
||
mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"
|
||
|
||
# Trim old entries — atomic write via temp file
|
||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||
CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
|
||
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
|
||
"$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
|
||
mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
|
||
fi
|
||
|
||
# Append snapshot
|
||
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
|
||
>> "$TUNING_MONITOR_LOG"
|
||
|
||
log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%" |