Added system watchdog. and way to many other changes

This commit is contained in:
2026-04-10 18:11:16 -04:00
parent 6cc26c8fb9
commit b7706f4ab4
8 changed files with 1532 additions and 229 deletions
+302 -104
View File
@@ -1,63 +1,96 @@
#!/bin/bash
# ----------------------------------------------------------------------------------------------
# ---------------------------- Master Variables for unRAID scripts -----------------------------
# ----------------------------------------------------------------------------------------------
# ==============================================================================================
# ================================= MASTER CONFIGURATION =======================================
# ==============================================================================================
# All user-facing variables for the unRAID script ecosystem.
# Scripts source this file — edit here, changes apply everywhere on next git pull.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
# Section Description
# ───────────────────────────────────────────────────────────────────────────────────────────
# HOST CONFIGURATION Server hostnames and SSH key paths
# LOGGING Enable or disable verbose logging
# NOTIFICATIONS unRAID native and Discord webhook settings
# GIT / REPO Gitea repository and SSH settings
#
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
# RSYNC DEFAULTS Global fallback rsync settings
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
# DAILY SYNC SHARES Media shares synced by daily_sync.sh
# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles)
#
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
# DOCKER DAILY RESTART Containers restarted daily
# DOCKER WEEKLY RESTART Containers restarted weekly
# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP
#
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
# REBOOT User warning delay before scheduled reboot
# MOVER Mover stop timeout
# SYSLOG FILTER Docker veth noise filter file path
# PHP-FPM PHP-FPM max children config
# CLEAR LOGS System log file paths
#
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
# MEDIA CLEANER Anime and media folder lists and file patterns
#
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
# SYSTEM WATCHDOG System health monitoring — last line of defense
#
# ==============================================================================================
# ━━━ Host Configuration ━━━
# List the Hostnames of both servers
# Hostnames must match Tailscale machine names exactly — case sensitive
HOST1="unRAID-Gmer4Lfe"
HOST2="unRAID-Jayred365"
# Assign SSH keys for each host pair (adjust paths as needed) HOST1 from above
# Must have its key added to HOST1_SSH_Key same applies for HOST2
# SSH keys for server-to-server rsync — each server needs the other's key authorised
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
# ━━━ Logging ━━━
ENABLE_LOGGING=true # false = only echo user-facing messages
# true = verbose [LOG] output in scripts / false = user-facing output only
ENABLE_LOGGING=true
# ━━━ Notifications ━━━
# unRAID native notification system — set to true to enable
# Configure unRAID to send errors only: Settings → Notification Settings
# unRAID native — configure Settings → Notification Settings for errors/warnings only
NOTIFY_UNRAID=true
# Discord webhook URL — leave blank to disable
DISCORD_WEBHOOK=""
# ━━━ Git, Pull & Execute Script ━━━
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync"
SSH_PORT=221
# ━━━ Git / Repo ━━━
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync"
SSH_PORT=221
# ━━━ Rsync Script Defaults ━━━
# These are the fallback values used when no matching profile is found.
# Any share whose directory basename does not match a profile key below
# will use these globals for all rsync behaviour.
BW_LIMIT=12500
# Retry logic
RETRY_COUNT=3
# Sleep between retries (seconds)
SLEEP=300
# Container start/stop/restart before and after rsync
CRITICAL_CONTAINER_NAMES=()
# Containers that need delayed before starting
DELAYED_CONTAINERS=()
# Delay in seconds between starting containers, useful for things like Authelia
CONTAINER_DELAY=5
# Directories to exclude during transfer
EXCLUDE_DIRS=()
# Default global rsync options
# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Rsync Defaults ━━━
# Global fallback values — used when no profile match is found for a share.
# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined).
BW_LIMIT=12500 # network speed limit KB/s
RETRY_COUNT=3 # number of retry attempts on failure
SLEEP=300 # seconds between retries
CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync
DELAYED_CONTAINERS=() # containers needing delay before start
CONTAINER_DELAY=5 # seconds delay before starting delayed containers
EXCLUDE_DIRS=() # directories to exclude from transfer
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
# ━━━ Remote Health Checks ━━━
# Abort if remote rootfs usage is at or above this percentage.
# Protects against rsync writing to rootfs when the remote array is down or drives are missing.
# Recommended: 75 — gives headroom before the server becomes unstable
# Abort rsync if remote rootfs exceeds this percentage.
# Protects against rsync filling rootfs when remote array is down or drives are missing.
ROOTFS_WARN=75
# ━━━ Daily Sync Shares ━━━
# Shares synced once daily by Orchestrators/daily_sync.sh
# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above.
# Add or remove paths here to manage what gets synced.
# These shares have no profile entry and fall through to DEFAULT_RSYNC_OPTS above.
DAILY_SYNC_SHARES=(
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows-Old
@@ -73,81 +106,22 @@ DAILY_SYNC_SHARES=(
/mnt/user/Tv_Shows
)
# ━━━ unRAID Essential Scripts ━━━
# unRAID reboot script user warning time (seconds)
REBOOT_SLEEP=300
# unRAID mover stop script timeout (seconds)
MOVER_STOP_TIMEOUT=300
# docker_syslog_filter.sh filter file path
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
# php_fpm_max_children.sh php-fpm config file path
PHP_CONF="/etc/php-fpm.d/www.conf"
# php_fpm_max_children.sh max children value
PHP_MAX_CHILDREN=250
# clear_logs.sh system log file paths
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
# ━━━ Docker Daily Restart ━━━
# Containers to restart daily — space-separated, case-sensitive
# These are the same containers as critical-data and other rsync profiles
DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Dispatcharr-Iptv-Users"
"Dispatcharr"
"Dispatcharr-Basic"
"Code-Server"
)
# ━━━ Docker Watchdog ━━━
# Containers to monitor with their memory hard limits in MB
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 8GB=8192 6GB=6144 4GB=4096 1GB=1024
declare -A WATCHDOG_CONTAINERS=(
["Emby"]=16384
["jellyfin_with_request"]=12288
["LidaTube"]=6144
["Tdarr"]=6144
["Code-Server"]=1024
)
# Containers to check for HTTP responsiveness — omit a container to skip its check
declare -A WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
["Jellyfin-Gmer4Lfe"]="http://localhost:8095"
)
# State file for tracking CPU and responsiveness strikes between runs
# Lives in /tmp — resets on reboot which is correct behaviour for strike tracking
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
# CPU thresholds — normalised against total core count automatically at runtime
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
# Memory threshold
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
# Responsiveness check settings
RESP_FAIL_LIMIT=2 # consecutive failures before restart
CURL_TIMEOUT=5 # seconds before curl gives up
# ━━━ Profile System ━━━
# ━━━ Rsync Profile System ━━━
# Profiles are matched by directory basename (lowercased).
# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
#
# How fallthrough works:
# - If a key exists in a profile array, that value is used
# - If a key is missing, the global default above is used instead
# - If a key exists in a profile array that value is used
# - If a key is missing the global default above is used instead
# - Shares in DAILY_SYNC_SHARES have no profile and always use globals
#
# To add a new profile:
# 1. Add a key to each array below with your chosen profile name
# 2. Call rsync.sh with a directory whose basename matches that key
# 3. Any array you omit will fall back to its global default
# 3. Any array you omit falls back to its global default
#
# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
# if you define it for a profile you must list all desired options explicitly
# list all desired options explicitly if you define a profile entry
# SPACE-SEPARATED STRINGS
declare -A PROFILE_RSYNC_OPTS=(
@@ -191,7 +165,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[emby]=""
)
# SPACE-SEPARATED STRINGS — containers that need a delay before starting
# SPACE-SEPARATED STRINGS — containers needing delay before starting
declare -A PROFILE_DELAYED_CONTAINERS=(
[arrs_stack]=""
[critical-data]="Authelia"
@@ -216,6 +190,230 @@ declare -A PROFILE_EXCLUDE_DIRS=(
[important-data]="logs *.tmp"
[emby]="logs *.tmp"
)
# ----------------------------------------------------------------------------------------------
# ---------------------- End Of User Variables, Please adjust above as needed ------------------
# ----------------------------------------------------------------------------------------------
# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Docker Daily Restart ━━━
# Containers restarted every day — case-sensitive names
DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Dispatcharr-Iptv-Users"
"Dispatcharr"
"Dispatcharr-Basic"
"Code-Server"
)
# ━━━ Docker Weekly Restart ━━━
# Containers restarted once per week — case-sensitive names
WEEKLY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Dispatcharr-Iptv-Users"
"Dispatcharr"
"Dispatcharr-Basic"
"Code-Server"
)
# ━━━ Docker Watchdog ━━━
# First line of defense — monitors and restarts unhealthy containers.
# Runs on a cron schedule (recommended every 15 minutes).
# Strike system prevents restarts on brief spikes.
# Containers to monitor with memory hard limits in MB
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
declare -A WATCHDOG_CONTAINERS=(
["Emby"]=16384
["LidaTube"]=6144
["Tdarr"]=6144
["Code-Server"]=1024
)
# Containers to check HTTP responsiveness — omit to skip
declare -A WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
)
# Containers that should always be running — monitored for unexpected stops
# Strike system used — persistent skip list prevents reboot loops
WATCHDOG_REQUIRED_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Emby"
)
# Strike state file — /tmp resets on reboot, correct for strike tracking
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
# CPU thresholds — normalised against total core count at runtime
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
HARD_CPU_THRESHOLD=90 # strike at this % of total system CPU
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
# Memory threshold
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
# Responsiveness check
RESP_FAIL_LIMIT=2 # consecutive failures before restart
CURL_TIMEOUT=5 # seconds before curl gives up
# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Reboot ━━━
# User warning delay before scheduled reboot (seconds)
REBOOT_SLEEP=300
# ━━━ Mover ━━━
# Timeout before stopping the mover (seconds)
MOVER_STOP_TIMEOUT=300
# ━━━ Syslog Filter ━━━
# Path for the rsyslog Docker noise filter file
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
# ━━━ PHP-FPM ━━━
# PHP-FPM config file path and max children value
PHP_CONF="/etc/php-fpm.d/www.conf"
PHP_MAX_CHILDREN=250
# ━━━ Clear Logs ━━━
# System log files to clear on each run
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Media Permissions ━━━
# Mode and owner applied recursively to all listed shares
PERMISSIONS_MODE="777"
PERMISSIONS_OWNER="nobody:users"
MEDIA_PERMISSION_SHARES=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows
/mnt/user/Anime_Shows-Old
/mnt/user/appcache
/mnt/user/Books
/mnt/user/Downloads
/mnt/user/Games
/mnt/user/Intros
/mnt/user/Kids_Movies
/mnt/user/Kids_Tv_Shows
/mnt/user/Movie_Recordings
/mnt/user/Movies
/mnt/user/Music
/mnt/user/Music_Videos
/mnt/user/Photo
/mnt/user/Sports
/mnt/user/stand-up_comedy
/mnt/user/Temp_Storage
/mnt/user/Tv_Recordings
/mnt/user/Tv_Shows
/mnt/user/YouTube
)
# ━━━ Media Cleaner ━━━
# Two profiles: anime and media — passed as argument to media_cleaner.sh
# Usage: media_cleaner.sh anime or media_cleaner.sh media
ANIME_CLEAN_FOLDERS=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows
/mnt/user/Anime_Shows-Old
)
MEDIA_CLEAN_FOLDERS=(
/mnt/user/Kids_Movies
/mnt/user/Kids_Tv_Shows
/mnt/user/Movies
/mnt/user/Music
/mnt/user/Sports
/mnt/user/stand-up_comedy
/mnt/user/Tv_Shows
)
# Anime file patterns — junk files common in anime downloads
ANIME_FILE_PATTERNS=(
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
'*.log' '*.json'
)
# Media file patterns — includes *.iso and *.lrc not needed in anime
MEDIA_FILE_PATTERNS=(
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
'*.log' '*.json' '*.iso' '*.lrc'
)
# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Last line of defense — reboots the system cleanly if it is about to become unstable.
# Designed to run on a cron schedule (recommended every 15-30 minutes).
# Works alongside docker_watchdog.sh — containers first, system second.
#
# Strike system — sustained threshold hits trigger reboot, not single spikes.
# Reboot loop protection — shuts down instead of rebooting if limit hit in window.
# Container skip list — persistent, auto-clears when container recovers.
# ━━━ System Watchdog State Files ━━━
# Strike counts — /tmp resets on reboot, correct for strike tracking
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
# Persistent container skip list — survives reboots, auto-clears on recovery
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
# Reboot timestamp log — survives reboots for loop detection
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
# ━━━ Strike and Reboot Loop Settings ━━━
# Consecutive threshold hits before triggering reboot
SYS_WATCHDOG_STRIKE_LIMIT=2
# Maximum reboots allowed within the window before shutdown instead
SYS_WATCHDOG_REBOOT_LIMIT=3
# Window in hours — controls both reboot count window AND rolling log purge
# 12 = entries older than 12hrs purge automatically / 24 = entries older than 24hrs purge
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
# ━━━ Thresholds ━━━
# Set at "about to fall over" levels — not just high usage
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs usage % before strike
SYS_WATCHDOG_LOG_PCT=95 # /var/log usage % before strike
SYS_WATCHDOG_MEM_GB=4 # free RAM in GB below which strikes
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers
SYS_WATCHDOG_LOAD_MULTIPLIER=32 # strike if load avg > cores x this value
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie process count exceeds this
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — tjmax varies by CPU
# ━━━ Check Toggles ━━━
# true = run this check / false = skip entirely
SYS_WATCHDOG_CHECK_ROOTFS=true
SYS_WATCHDOG_CHECK_LOG=true
SYS_WATCHDOG_CHECK_RAM=true
SYS_WATCHDOG_CHECK_ARC=true
SYS_WATCHDOG_CHECK_CPU_TEMP=true
SYS_WATCHDOG_CHECK_LOAD=true
SYS_WATCHDOG_CHECK_ZOMBIES=true
SYS_WATCHDOG_CHECK_CONTAINERS=true
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
# ━━━ Abort Toggles ━━━
# true = abort reboot if condition is active / false = reboot anyway
# Default true = conservative / set false to reboot regardless
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
SYS_WATCHDOG_ABORT_ON_PARITY=false
SYS_WATCHDOG_ABORT_ON_MOVER=false
# ==============================================================================================
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
# ==============================================================================================