diff --git a/Master.conf b/Master.conf index 4fd2acb..226a14e 100644 --- a/Master.conf +++ b/Master.conf @@ -167,7 +167,7 @@ ARRAY_START_SCRIPTS=( "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop -# "Failover/failover.sh" # mutual failover — continuous loop + "Failover/failover.sh" # mutual failover — continuous loop ) # ━━━ Daily Sync Maintenance ━━━ @@ -219,22 +219,6 @@ HOST2_PERSONAL_SHARES=( # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset ) -# ━━━ Media Management ━━━ -# Job list run directly by daily_sync_maintenance.sh after the media share sync. -# Runs sequentially — permissions first, then cleaners, then arr cleanup. -# Comment out any job to disable without removing it. -# Each individual script can still be run manually for one-off maintenance. - -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions — runs first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares -# "Media/lidarr_cleanup.sh" # remove orphaned music files -# "Media/sonarr_cleanup.sh" # remove orphaned TV files -# "Media/radarr_cleanup.sh" # remove orphaned movie files - "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history -) - # ━━━ Weekly Sync Maintenance ━━━ # weekly_sync_maintenance.sh handles the critical sync built into the script first: # stop containers both sides → pull updates → sync Emby + Critical-Data → restart @@ -261,6 +245,22 @@ WEEKLY_SYNC_JOBS=( CRITICAL_SYNC_UPDATES=true # pull container updates locally CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH +# ━━━ Media Management ━━━ +# Job list run directly by daily_sync_maintenance.sh after the media share sync. +# Runs sequentially — permissions first, then cleaners, then arr cleanup. +# Comment out any job to disable without removing it. +# Each individual script can still be run manually for one-off maintenance. + +MEDIA_MANAGEMENT_JOBS=( + "Media/media_shares_permissions.sh" # apply permissions — runs first + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + "Media/lidarr_cleanup.sh" # remove orphaned music files + "Media/sonarr_cleanup.sh" # remove orphaned TV files + "Media/radarr_cleanup.sh" # remove orphaned movie files + "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history +) + # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -823,10 +823,19 @@ NETWORK_CONNECT_NETWORKS=( # (Sonarr, Radarr, Lidarr, NextCloud etc.) you can silently exhaust the limit. # Symptoms: containers miss file events, downloads not detected, library not updated. # These settings are lost on reboot — reapplied automatically at array start. - INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user + INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping +# ━━━ System Tuning Monitor ━━━ +# Tracks inotify and php-fpm usage over time — read by sunday_morning_coffee_report.sh +# Snapshot written every 6 hours by system_tuning_monitor.sh +# Log bounded to TUNING_LOG_RETENTION days — auto-purges on each write + INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit + PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children + TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" + TUNING_LOG_RETENTION=30 # days — enough for monthly trend visibility + # ━━━ Reboot ━━━ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. # Gives users time to save work — 300s = 5 minutes diff --git a/Monitors/system_tuning_monitor.sh b/Monitors/system_tuning_monitor.sh new file mode 100644 index 0000000..e69de29 diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 9030253..e883f49 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -690,6 +690,59 @@ fi # ----------------------------------------------------------------------------------------------- section "⚙️ SYSTEM HEALTH" +# inotify + php-fpm weekly stats +if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then + # inotify weekly stats + INOTIFY_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {if ($3+0 > max) max=$3+0} END {print max+0}' "$TUNING_MONITOR_LOG") + INOTIFY_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$3; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \ + "$TUNING_MONITOR_LOG") + INOTIFY_LOW=$(awk -F'|' -v cutoff="$WEEK_START" \ + 'BEGIN{min=99999} $1 >= cutoff {if($3+0 < min) min=$3+0} END {print min+0}' \ + "$TUNING_MONITOR_LOG") + INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) + INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) + INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}" + INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW )) + INOTIFY_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff && $6 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG") + + if [[ "${INOTIFY_WARN_COUNT:-0}" -gt 0 ]]; then + issue "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW | ⚠️ warnings: $INOTIFY_WARN_COUNT" + else + line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW ✅" + fi + + # php-fpm weekly stats + PHPFPM_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {if ($7+0 > max) max=$7+0} END {print max+0}' "$TUNING_MONITOR_LOG") + PHPFPM_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$7; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \ + "$TUNING_MONITOR_LOG") + PHPFPM_MAX_NOW="${PHP_MAX_CHILDREN:-250}" + PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) + PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}" + PHPFPM_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff && $10 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG") + + if [[ "${PHPFPM_WARN_COUNT:-0}" -gt 0 ]]; then + issue "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG | ⚠️ warnings: $PHPFPM_WARN_COUNT" + else + line "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG ✅" + fi +else + # No log yet — just show live values + INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) + INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) + INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}" + INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW )) + PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) + PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}" + line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} (${INOTIFY_NOW_PCT}%) — no weekly data yet" + line "php-fpm: ${PHPFPM_NOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet" +fi + # SMART summary if command -v smartctl >/dev/null 2>&1; then SMART_ISSUES=0 diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 91e5c29..e9ff7e2 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -65,6 +65,7 @@ # │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check # │ ├── emby_session_report.sh # Weekly Emby usage statistics via API # │ ├── smart_health.sh # Drive SMART attribute monitoring +# │ ├── system_tuning_monitor.sh # inotify + php-fpm usage tracking — every 6hr # │ ├── weekly_health_digest.sh # Aggregated system health — always/smart/weekly # │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report # │ └── README-Monitors.md @@ -163,6 +164,8 @@ # ━━━ Monitors ━━━ #/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh # ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover +#/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh +# ^^ run manually for live snapshot — scheduled every 6hr via cron #/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh #/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report #/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh @@ -201,12 +204,17 @@ # blocklist + re-search failed imports and stalled downloads # across Sonarr, Radarr, and Lidarr automatically # +# system_tuning_monitor.sh — 0 */6 * * * (every 6 hours) +# snapshot inotify + php-fpm usage — feeds sunday coffee report +# weekly peak, average, low, warning count +# #/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh #/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh +#/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh # # ━━━ Rsync — Appdata Profiles ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack @@ -240,8 +248,10 @@ #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh +# ^^ ensures custom networks exist + connects containers — run at array start #/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --dry-run --log #/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh +# ^^ called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS — run manually for testing # # ━━━ Media ━━━ # media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS. @@ -289,7 +299,6 @@ # ━━━ unRAID Essentials ━━━ #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh -#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh #/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh