updated weekly health and sunday coffee report, all needed monitor now in both
This commit is contained in:
+26
-17
@@ -167,7 +167,7 @@ ARRAY_START_SCRIPTS=(
|
||||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||||
"unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop
|
||||
"Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop
|
||||
# "Failover/failover.sh" # mutual failover — continuous loop
|
||||
"Failover/failover.sh" # mutual failover — continuous loop
|
||||
)
|
||||
|
||||
# ━━━ Daily Sync Maintenance ━━━
|
||||
@@ -219,22 +219,6 @@ HOST2_PERSONAL_SHARES=(
|
||||
# /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset
|
||||
)
|
||||
|
||||
# ━━━ Media Management ━━━
|
||||
# Job list run directly by daily_sync_maintenance.sh after the media share sync.
|
||||
# Runs sequentially — permissions first, then cleaners, then arr cleanup.
|
||||
# Comment out any job to disable without removing it.
|
||||
# Each individual script can still be run manually for one-off maintenance.
|
||||
|
||||
MEDIA_MANAGEMENT_JOBS=(
|
||||
"Media/media_shares_permissions.sh" # apply permissions — runs first
|
||||
"Media/media_cleaner.sh anime" # remove junk from anime shares
|
||||
"Media/media_cleaner.sh media" # remove junk from media shares
|
||||
# "Media/lidarr_cleanup.sh" # remove orphaned music files
|
||||
# "Media/sonarr_cleanup.sh" # remove orphaned TV files
|
||||
# "Media/radarr_cleanup.sh" # remove orphaned movie files
|
||||
"Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history
|
||||
)
|
||||
|
||||
# ━━━ Weekly Sync Maintenance ━━━
|
||||
# weekly_sync_maintenance.sh handles the critical sync built into the script first:
|
||||
# stop containers both sides → pull updates → sync Emby + Critical-Data → restart
|
||||
@@ -261,6 +245,22 @@ WEEKLY_SYNC_JOBS=(
|
||||
CRITICAL_SYNC_UPDATES=true # pull container updates locally
|
||||
CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||||
|
||||
# ━━━ Media Management ━━━
|
||||
# Job list run directly by daily_sync_maintenance.sh after the media share sync.
|
||||
# Runs sequentially — permissions first, then cleaners, then arr cleanup.
|
||||
# Comment out any job to disable without removing it.
|
||||
# Each individual script can still be run manually for one-off maintenance.
|
||||
|
||||
MEDIA_MANAGEMENT_JOBS=(
|
||||
"Media/media_shares_permissions.sh" # apply permissions — runs first
|
||||
"Media/media_cleaner.sh anime" # remove junk from anime shares
|
||||
"Media/media_cleaner.sh media" # remove junk from media shares
|
||||
"Media/lidarr_cleanup.sh" # remove orphaned music files
|
||||
"Media/sonarr_cleanup.sh" # remove orphaned TV files
|
||||
"Media/radarr_cleanup.sh" # remove orphaned movie files
|
||||
"Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -827,6 +827,15 @@ NETWORK_CONNECT_NETWORKS=(
|
||||
INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance
|
||||
INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping
|
||||
|
||||
# ━━━ System Tuning Monitor ━━━
|
||||
# Tracks inotify and php-fpm usage over time — read by sunday_morning_coffee_report.sh
|
||||
# Snapshot written every 6 hours by system_tuning_monitor.sh
|
||||
# Log bounded to TUNING_LOG_RETENTION days — auto-purges on each write
|
||||
INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit
|
||||
PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children
|
||||
TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
|
||||
TUNING_LOG_RETENTION=30 # days — enough for monthly trend visibility
|
||||
|
||||
# ━━━ Reboot ━━━
|
||||
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
|
||||
# Gives users time to save work — 300s = 5 minutes
|
||||
|
||||
@@ -690,6 +690,59 @@ fi
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "⚙️ SYSTEM HEALTH"
|
||||
|
||||
# inotify + php-fpm weekly stats
|
||||
if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then
|
||||
# inotify weekly stats
|
||||
INOTIFY_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {if ($3+0 > max) max=$3+0} END {print max+0}' "$TUNING_MONITOR_LOG")
|
||||
INOTIFY_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$3; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \
|
||||
"$TUNING_MONITOR_LOG")
|
||||
INOTIFY_LOW=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'BEGIN{min=99999} $1 >= cutoff {if($3+0 < min) min=$3+0} END {print min+0}' \
|
||||
"$TUNING_MONITOR_LOG")
|
||||
INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024)
|
||||
INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||||
INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}"
|
||||
INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW ))
|
||||
INOTIFY_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff && $6 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG")
|
||||
|
||||
if [[ "${INOTIFY_WARN_COUNT:-0}" -gt 0 ]]; then
|
||||
issue "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW | ⚠️ warnings: $INOTIFY_WARN_COUNT"
|
||||
else
|
||||
line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW ✅"
|
||||
fi
|
||||
|
||||
# php-fpm weekly stats
|
||||
PHPFPM_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {if ($7+0 > max) max=$7+0} END {print max+0}' "$TUNING_MONITOR_LOG")
|
||||
PHPFPM_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$7; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \
|
||||
"$TUNING_MONITOR_LOG")
|
||||
PHPFPM_MAX_NOW="${PHP_MAX_CHILDREN:-250}"
|
||||
PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||||
PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}"
|
||||
PHPFPM_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff && $10 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG")
|
||||
|
||||
if [[ "${PHPFPM_WARN_COUNT:-0}" -gt 0 ]]; then
|
||||
issue "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG | ⚠️ warnings: $PHPFPM_WARN_COUNT"
|
||||
else
|
||||
line "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG ✅"
|
||||
fi
|
||||
else
|
||||
# No log yet — just show live values
|
||||
INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024)
|
||||
INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||||
INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}"
|
||||
INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW ))
|
||||
PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||||
PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}"
|
||||
line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} (${INOTIFY_NOW_PCT}%) — no weekly data yet"
|
||||
line "php-fpm: ${PHPFPM_NOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet"
|
||||
fi
|
||||
|
||||
# SMART summary
|
||||
if command -v smartctl >/dev/null 2>&1; then
|
||||
SMART_ISSUES=0
|
||||
|
||||
+10
-1
@@ -65,6 +65,7 @@
|
||||
# │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check
|
||||
# │ ├── emby_session_report.sh # Weekly Emby usage statistics via API
|
||||
# │ ├── smart_health.sh # Drive SMART attribute monitoring
|
||||
# │ ├── system_tuning_monitor.sh # inotify + php-fpm usage tracking — every 6hr
|
||||
# │ ├── weekly_health_digest.sh # Aggregated system health — always/smart/weekly
|
||||
# │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
|
||||
# │ └── README-Monitors.md
|
||||
@@ -163,6 +164,8 @@
|
||||
# ━━━ Monitors ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh
|
||||
# ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh
|
||||
# ^^ run manually for live snapshot — scheduled every 6hr via cron
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh
|
||||
@@ -201,12 +204,17 @@
|
||||
# blocklist + re-search failed imports and stalled downloads
|
||||
# across Sonarr, Radarr, and Lidarr automatically
|
||||
#
|
||||
# system_tuning_monitor.sh — 0 */6 * * * (every 6 hours)
|
||||
# snapshot inotify + php-fpm usage — feeds sunday coffee report
|
||||
# weekly peak, average, low, warning count
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/system_tuning_monitor.sh
|
||||
#
|
||||
# ━━━ Rsync — Appdata Profiles ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
|
||||
@@ -240,8 +248,10 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_daily_restart.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh
|
||||
# ^^ ensures custom networks exist + connects containers — run at array start
|
||||
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh --dry-run --log
|
||||
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/downloaders_reset.sh
|
||||
# ^^ called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS — run manually for testing
|
||||
#
|
||||
# ━━━ Media ━━━
|
||||
# media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS.
|
||||
@@ -289,7 +299,6 @@
|
||||
# ━━━ unRAID Essentials ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/clear_logs.sh
|
||||
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/docker_syslog_filter.sh
|
||||
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/inotify_tuning.sh
|
||||
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/mover_stop.sh
|
||||
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/php_fpm_max_children.sh
|
||||
#/mnt/user/appdata/unraid_scripts/unRAID_Essentials/rsync_stop.sh
|
||||
|
||||
Reference in New Issue
Block a user