Gmer4Lfe
e13f2fa14f
- downloaders_reset: connection check block before slskd API sections; triggers PUT /api/v0/server reconnect if disconnected, polls 60s, gates Stuck Searches and Dead Transfer Records on SLSKD_CONNECTED - Sync all modified/new/deleted files from v2 refactor across Docker_Essentials, Media, Monitors, Partnership, Rsync, Tools, Transcodes, unRAID_Essentials, common.sh, master confs, and new Manual/README docs
248 lines
10 KiB
Bash
248 lines
10 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= System Tuning Monitor ==========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# inotify and PHP-FPM utilisation tracking via time-series snapshots. Scheduled
|
||
# every 6 hours (0 */6 * * *). Writes one bounded log entry per run to
|
||
# TUNING_MONITOR_LOG. weekly_health_digest.sh reads this log to report peak,
|
||
# average, and warning counts over the week.
|
||
#
|
||
# Background snapshot script — no output when healthy. Warns (and notifies) only
|
||
# when INOTIFY_WARN_PCT or PHP_FPM_WARN_PCT thresholds are exceeded.
|
||
#
|
||
# inotify exhaustion symptoms: downloads complete but arrs don't detect them,
|
||
# live TV stutter, library updates stop. PHP-FPM exhaustion symptoms: unRAID
|
||
# WebGUI slowdowns or timeouts under load.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Each run snapshots:
|
||
# inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit.
|
||
# Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT.
|
||
# php-fpm: active workers vs PHP_MAX_CHILDREN limit.
|
||
# Warns above PHP_FPM_WARN_PCT.
|
||
#
|
||
# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days):
|
||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|
|
||
# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||
# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts
|
||
# warnings over the week to show trend severity.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Single Instance Lock
|
||
# acquire_lock prevents overlapping 6-hour snapshot runs.
|
||
#
|
||
# Root Enforcement
|
||
# /proc/*/fd enumeration for inotify consumer counting requires root access.
|
||
#
|
||
# Atomic Log Write
|
||
# Trim uses tmp file + mv — partial writes during log rotation cannot corrupt
|
||
# the accumulated history.
|
||
#
|
||
# Notification Validated
|
||
# validate_unraid_cmd confirms the notify script is present before use.
|
||
#
|
||
# ==============================================================================================
|
||
# STATE FILES
|
||
# ==============================================================================================
|
||
#
|
||
# TUNING_MONITOR_LOG — DATA_DIR/tuning_monitor.db
|
||
# One line per 6-hour snapshot. Trimmed to TUNING_LOG_RETENTION days on
|
||
# each write — bounded size. Read by weekly_health_digest.sh for trend
|
||
# reporting. Resets if DATA_DIR is cleared.
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# INOTIFY_WARN_PCT
|
||
# Warn if inotify instances exceed this percentage of the kernel limit. (default: 80)
|
||
#
|
||
# PHP_FPM_WARN_PCT
|
||
# Warn if php-fpm active workers exceed this percentage of PHP_MAX_CHILDREN. (default: 80)
|
||
#
|
||
# PHP_MAX_CHILDREN
|
||
# Maximum php-fpm workers — set by php_fpm_max_children.sh in unRAID_Essentials.
|
||
#
|
||
# TUNING_MONITOR_LOG
|
||
# Log file path. (default: DATA_DIR/tuning_monitor.db)
|
||
#
|
||
# TUNING_LOG_RETENTION
|
||
# Days before old entries are purged. (default: 30)
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# system_tuning_monitor.sh
|
||
# Take snapshot. Write to log. Warn (and notify) if thresholds exceeded.
|
||
# Silent when healthy.
|
||
#
|
||
# system_tuning_monitor.sh --dry-run
|
||
# Measure inotify and PHP-FPM utilisation and display results. No log write.
|
||
#
|
||
# system_tuning_monitor.sh --status
|
||
# Show thresholds, log path, and retention. Then exit.
|
||
#
|
||
# system_tuning_monitor.sh --log
|
||
# Verbose output during the snapshot including top inotify consumers.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root — /proc/*/fd requires root access"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
acquire_lock
|
||
|
||
# detect_hosts() sets MY_ID — used in warning output
|
||
detect_hosts
|
||
|
||
DATE=$(date '+%Y-%m-%d')
|
||
TIME=$(date '+%H:%M')
|
||
INOTIFY_WARN=0
|
||
PHPFPM_WARN=0
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo "$ICON_GEAR inotify warn: ${INOTIFY_WARN_PCT:-80}%"
|
||
echo "$ICON_GEAR php-fpm warn: ${PHP_FPM_WARN_PCT:-80}%"
|
||
echo "$ICON_GEAR php-fpm max: ${PHP_MAX_CHILDREN:-250}"
|
||
echo "$ICON_GEAR Log file: ${TUNING_MONITOR_LOG:-not set}"
|
||
echo "$ICON_GEAR Retention: ${TUNING_LOG_RETENTION:-30} days"
|
||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||
|
||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||
ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
|
||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
|
||
NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
|
||
echo "$ICON_MONITOR Log entries: $ENTRY_COUNT ($OLDEST → $NEWEST)"
|
||
else
|
||
echo "$ICON_MONITOR Log entries: none yet"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ inotify ━━━
|
||
# ==============================================================================================
|
||
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
|
||
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
|
||
INOTIFY_USED="${INOTIFY_USED:-0}"
|
||
|
||
if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
|
||
INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
|
||
else
|
||
INOTIFY_PCT=0
|
||
fi
|
||
|
||
[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1
|
||
|
||
# Top 5 inotify consumers
|
||
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
|
||
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
|
||
while read -r count pid; do
|
||
comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
|
||
echo "${count}×${comm}"
|
||
done | tr '\n' ',' | sed 's/,$//')
|
||
|
||
if [[ "$INOTIFY_WARN" -eq 1 ]]; then
|
||
warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
|
||
warn "Top consumers: ${INOTIFY_TOP:-unknown}"
|
||
warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
|
||
notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
|
||
"System Tuning" "warning"
|
||
else
|
||
log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
|
||
log "inotify top consumers: ${INOTIFY_TOP:-none}"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ php-fpm ━━━
|
||
# ==============================================================================================
|
||
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"
|
||
|
||
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
|
||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"
|
||
|
||
if [[ "$PHPFPM_MAX" -gt 0 ]]; then
|
||
PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
|
||
else
|
||
PHPFPM_PCT=0
|
||
fi
|
||
|
||
[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1
|
||
|
||
if [[ "$PHPFPM_WARN" -eq 1 ]]; then
|
||
warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
|
||
warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
|
||
notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
|
||
"System Tuning" "warning"
|
||
else
|
||
log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Write Snapshot ━━━
|
||
# ==============================================================================================
|
||
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
|
||
warn "TUNING_MONITOR_LOG not set — snapshot not written"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — snapshot not written"
|
||
warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
|
||
exit 0
|
||
fi
|
||
|
||
mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"
|
||
|
||
# Trim old entries — atomic write via temp file
|
||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||
CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
|
||
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
|
||
"$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
|
||
mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
|
||
fi
|
||
|
||
# Append snapshot
|
||
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
|
||
>> "$TUNING_MONITOR_LOG"
|
||
|
||
log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%" |