massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= System Tuning Monitor ==========================================
|
||||
# ==============================================================================================
|
||||
# Tracks inotify and php-fpm usage over time.
|
||||
# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary.
|
||||
# Schedule: 0 */6 * * * (every 6 hours via User Scripts)
|
||||
#
|
||||
# ── WHAT IT TRACKS ────────────────────────────────────────────────────────────────────────────
|
||||
# inotify instances:
|
||||
# Current in use vs kernel limit
|
||||
# % utilization — warns above INOTIFY_WARN_PCT (default 80%)
|
||||
# Top 5 consumers by instance count
|
||||
# Symptom of exhaustion: containers miss file events, downloads not detected,
|
||||
# Live TV stutter, library not updated
|
||||
#
|
||||
# php-fpm workers:
|
||||
# Active workers vs PHP_MAX_CHILDREN limit
|
||||
# % utilization — warns above PHP_FPM_WARN_PCT (default 80%)
|
||||
# Symptom: unRAID WebGUI slowdowns or timeouts under load
|
||||
#
|
||||
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
|
||||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||||
# Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size.
|
||||
#
|
||||
# ── WHAT THE WEEKLY REPORT SHOWS ──────────────────────────────────────────────────────────────
|
||||
# inotify: peak, average, warning count over the week
|
||||
# php-fpm: peak workers, average workers, warning count over the week
|
||||
#
|
||||
# ── SILENT BY DEFAULT ─────────────────────────────────────────────────────────────────────────
|
||||
# Background snapshot script — no output when healthy.
|
||||
# Warns to stderr when thresholds exceeded — visible in User Scripts output log.
|
||||
# Does NOT notify on every snapshot — only when threshold exceeded.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Each server writes to its own DATA_DIR — no collision between servers.
|
||||
# MY_ID included in warning output for clarity in shared notification channels.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents overlapping 6-hour snapshots
|
||||
# root check — /proc/*/fd requires root access
|
||||
# atomic log write — tmp file + mv prevents partial writes on trim
|
||||
# validate_unraid — notify script validated before use
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# INOTIFY_WARN_PCT — warn threshold % (default 80)
|
||||
# PHP_FPM_WARN_PCT — warn threshold % (default 80)
|
||||
# PHP_MAX_CHILDREN — max php-fpm workers (set by php_fpm_max_children.sh)
|
||||
# TUNING_MONITOR_LOG — log file path
|
||||
# TUNING_LOG_RETENTION — days before old entries purged (default 30)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# system_tuning_monitor.sh — normal snapshot run
|
||||
# system_tuning_monitor.sh --dry-run — measure and show, no log write
|
||||
# system_tuning_monitor.sh --log — verbose output
|
||||
# system_tuning_monitor.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — /proc/*/fd requires root access"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in warning output
|
||||
detect_hosts
|
||||
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
TIME=$(date '+%H:%M')
|
||||
INOTIFY_WARN=0
|
||||
PHPFPM_WARN=0
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR inotify warn: ${INOTIFY_WARN_PCT:-80}%"
|
||||
echo "$ICON_GEAR php-fpm warn: ${PHP_FPM_WARN_PCT:-80}%"
|
||||
echo "$ICON_GEAR php-fpm max: ${PHP_MAX_CHILDREN:-250}"
|
||||
echo "$ICON_GEAR Log file: ${TUNING_MONITOR_LOG:-not set}"
|
||||
echo "$ICON_GEAR Retention: ${TUNING_LOG_RETENTION:-30} days"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
|
||||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||||
ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
|
||||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
|
||||
NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
|
||||
echo "$ICON_MONITOR Log entries: $ENTRY_COUNT ($OLDEST → $NEWEST)"
|
||||
else
|
||||
echo "$ICON_MONITOR Log entries: none yet"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ inotify ━━━
|
||||
# ==============================================================================================
|
||||
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
|
||||
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||||
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
|
||||
INOTIFY_USED="${INOTIFY_USED:-0}"
|
||||
|
||||
if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
|
||||
INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
|
||||
else
|
||||
INOTIFY_PCT=0
|
||||
fi
|
||||
|
||||
[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1
|
||||
|
||||
# Top 5 inotify consumers
|
||||
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
|
||||
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
|
||||
while read -r count pid; do
|
||||
comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
|
||||
echo "${count}×${comm}"
|
||||
done | tr '\n' ',' | sed 's/,$//')
|
||||
|
||||
if [[ "$INOTIFY_WARN" -eq 1 ]]; then
|
||||
warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
|
||||
warn "Top consumers: ${INOTIFY_TOP:-unknown}"
|
||||
warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
|
||||
notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
|
||||
"System Tuning" "warning"
|
||||
else
|
||||
log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
|
||||
log "inotify top consumers: ${INOTIFY_TOP:-none}"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ php-fpm ━━━
|
||||
# ==============================================================================================
|
||||
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"
|
||||
|
||||
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
|
||||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"
|
||||
|
||||
if [[ "$PHPFPM_MAX" -gt 0 ]]; then
|
||||
PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
|
||||
else
|
||||
PHPFPM_PCT=0
|
||||
fi
|
||||
|
||||
[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1
|
||||
|
||||
if [[ "$PHPFPM_WARN" -eq 1 ]]; then
|
||||
warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
|
||||
warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
|
||||
notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
|
||||
"System Tuning" "warning"
|
||||
else
|
||||
log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Write Snapshot ━━━
|
||||
# ==============================================================================================
|
||||
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
|
||||
warn "TUNING_MONITOR_LOG not set — snapshot not written"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — snapshot not written"
|
||||
warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"
|
||||
|
||||
# Trim old entries — atomic write via temp file
|
||||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||||
CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
|
||||
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
|
||||
"$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
|
||||
mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
|
||||
fi
|
||||
|
||||
# Append snapshot
|
||||
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
|
||||
>> "$TUNING_MONITOR_LOG"
|
||||
|
||||
log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
|
||||
Reference in New Issue
Block a user