added a toggle to check disk for pool only shares

This commit is contained in:
2026-04-12 00:10:04 -04:00
parent 8bd2426337
commit 1dd11894ce
3 changed files with 491 additions and 103 deletions
+286 -69
View File
@@ -51,19 +51,28 @@
# ==============================================================================================
# ━━━ Host Configuration ━━━
# Hostnames must match Tailscale machine names exactly — case sensitive
HOST1="unRAID-Gmer4Lfe"
HOST2="unRAID-Jayred365"
# SSH keys for server-to-server rsync and failover operations
# Each server authenticates with its own key — both must be authorised on the remote
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
# ━━━ Logging ━━━
# true = verbose [LOG] lines in all script output / false = user-facing output only
ENABLE_LOGGING=true
# ━━━ Notifications ━━━
# unRAID native — fires through Settings → Notification Settings
# Recommended: configure unRAID to send errors/warnings only so normal completions stay quiet
NOTIFY_UNRAID=true
# Discord webhook — paste full webhook URL to enable, leave blank to disable
DISCORD_WEBHOOK=""
# ━━━ Git / Repo ━━━
# Gitea self-hosted repository for the script ecosystem
# git_pull_execute.sh uses these to clone or pull the latest version on both servers
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
@@ -74,19 +83,42 @@
# ==============================================================================================
# ━━━ Rsync Defaults ━━━
# Global fallback values used when no profile match is found for a directory.
# Shares in DAILY_SYNC_SHARES always use these globals — no profile is defined for them.
# Appdata shares (Arrs_Stack, Critical-Data etc.) match profiles by directory basename.
# Network transfer speed cap in KB/s — 12500 ≈ 100Mbit
BW_LIMIT=12500
# Number of retry attempts if rsync fails before giving up
RETRY_COUNT=3
# Seconds to wait between retry attempts
SLEEP=300
# Containers to stop on remote before rsync and restart after — empty by default
# Profiles below override this per appdata share
CRITICAL_CONTAINER_NAMES=()
# Containers that need a delay before starting — e.g. Authelia needs its DB ready first
DELAYED_CONTAINERS=()
# Seconds to wait before starting delayed containers
CONTAINER_DELAY=5
# Directories to exclude from transfer — empty by default, profiles override per share
EXCLUDE_DIRS=()
# Default rsync options used when no profile match is found
# --delete removes files on remote that no longer exist on source
# --inplace writes directly to destination file rather than temp file — better for large files
# --no-whole-file forces delta transfer even on local-like connections
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
# ━━━ Remote Health Checks ━━━
# Abort rsync if remote rootfs (/) usage is at or above this percentage.
# When the remote array is down or drives are missing, rsync writes land on rootfs instead
# of /mnt/user — this fills the filesystem rapidly and can crash the remote server.
# 75% gives enough headroom to detect the problem before it becomes critical.
ROOTFS_WARN=75
# ━━━ Daily Sync Shares ━━━
# Media shares synced once daily by Orchestrators/daily_sync.sh
# These shares have no profile — all use DEFAULT_RSYNC_OPTS above
# Add or remove paths to control what gets synced each night
DAILY_SYNC_SHARES=(
/mnt/user/Anime_Movies-Old
/mnt/user/Anime_Shows-Old
@@ -103,6 +135,37 @@ DAILY_SYNC_SHARES=(
)
# ━━━ Rsync Profile System ━━━
# Profiles allow per-share rsync behaviour without touching script logic.
# Each profile is matched automatically by the basename of the directory passed to rsync.sh
# (lowercased). Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
#
# How fallthrough works:
# If a key exists in a profile array → that value is used for this run
# If a key is missing → the global default above is used instead
# Shares in DAILY_SYNC_SHARES → always use globals, no profile defined
#
# To add a new profile:
# 1. Add a key to each array below with your chosen profile name
# 2. Call rsync.sh with a directory whose basename matches that key
# 3. Any array you omit falls back to its global default automatically
#
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
# if you define a profile entry you must list all desired options explicitly
#
# Profile descriptions:
# arrs_stack — Sonarr, Radarr, Lidarr, Readarr, Prowlarr, Bazarr, Pinchflat appdata
# Lower bandwidth — runs alongside media syncs, containers stopped during sync
# critical-data — Auth stack: NPM, Authelia, Mariadb, Redis, LLDAP
# High bandwidth — small data, synced frequently, Authelia needs delayed start
# gmer4lfe — Server-specific appdata: Organizr, UptimeKuma, VaultWarden
# Medium bandwidth — personal services, no container stop needed
# important-data — NextCloud + Postgres database
# High bandwidth — NextCloud needs graceful stop before sync
# emby — Emby media server appdata and metadata
# Medium bandwidth — large appdata, no containers stopped (metadata only)
# SPACE-SEPARATED STRINGS — rsync options per profile
# If defined for a profile, these replace DEFAULT_RSYNC_OPTS entirely for that run
declare -A PROFILE_RSYNC_OPTS=(
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
@@ -111,14 +174,16 @@ declare -A PROFILE_RSYNC_OPTS=(
[emby]="-av --human-readable --bwlimit=$BW_LIMIT"
)
# Bandwidth limit in KB/s per profile — overrides global BW_LIMIT for this profile
declare -A PROFILE_BW_LIMIT=(
[arrs_stack]=5000
[critical-data]=9500
[gmer4lfe]=8000
[important-data]=9500
[emby]=8000
[arrs_stack]=5000 # lower — runs alongside other jobs
[critical-data]=9500 # high — small data, sync fast
[gmer4lfe]=8000 # medium
[important-data]=9500 # high — database sync, prioritise speed
[emby]=8000 # medium — large files, steady transfer
)
# Retry attempts per profile — overrides global RETRY_COUNT
declare -A PROFILE_RETRY_COUNT=(
[arrs_stack]=3
[critical-data]=3
@@ -127,6 +192,7 @@ declare -A PROFILE_RETRY_COUNT=(
[emby]=3
)
# Sleep between retries in seconds — overrides global SLEEP
declare -A PROFILE_SLEEP=(
[arrs_stack]=300
[critical-data]=300
@@ -135,6 +201,9 @@ declare -A PROFILE_SLEEP=(
[emby]=300
)
# Containers stopped on REMOTE before rsync and restarted after — SPACE-SEPARATED STRINGS
# Only containers that need to be stopped for data consistency — running containers are fine
# for most media but databases and auth stacks need clean state during sync
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
[critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia"
@@ -143,6 +212,9 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[emby]=""
)
# Containers that need a delay before starting after rsync — SPACE-SEPARATED STRINGS
# Authelia needs its database containers (Mariadb, Redis) to be ready before it starts
# NextCloud needs Postgres to be accepting connections before it starts
declare -A PROFILE_DELAYED_CONTAINERS=(
[arrs_stack]=""
[critical-data]="Authelia"
@@ -151,14 +223,17 @@ declare -A PROFILE_DELAYED_CONTAINERS=(
[emby]=""
)
# Seconds to wait before starting delayed containers — per profile
declare -A PROFILE_CONTAINER_DELAY=(
[arrs_stack]=5
[critical-data]=10
[critical-data]=10 # Authelia needs DB ready — 10s gives Mariadb/Redis time to start
[gmer4lfe]=5
[important-data]=10
[important-data]=10 # NextCloud needs Postgres ready
[emby]=5
)
# Directories excluded from transfer per profile — SPACE-SEPARATED STRINGS
# logs and *.tmp are excluded universally — they are ephemeral and regenerated on start
declare -A PROFILE_EXCLUDE_DIRS=(
[arrs_stack]="logs *.tmp"
[critical-data]="logs *.tmp"
@@ -167,52 +242,102 @@ declare -A PROFILE_EXCLUDE_DIRS=(
[emby]="logs *.tmp"
)
# ━━━ Profile Disk Check Toggle ━━━
# true = skip per-disk check for this profile — use when share lives on a ZFS pool
# false = run per-disk check — use for traditional unRAID array with individual disks
# Per-disk check looks for /mnt/disk*/sharename — ZFS pools don't have this structure
# The share existence and content checks still run regardless of this setting
declare -A PROFILE_SKIP_DISK_CHECK=(
[arrs_stack]=true
[critical-data]=true
[gmer4lfe]=true
[important-data]=true
[emby]=true
)
# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Mutual container failover between two unRAID servers 50 miles apart.
# Each server runs failover.sh independently — no coordination between servers.
# Decisions based solely on two ping checks: remote reachable + internet reachable.
# All decisions are based solely on two ping checks: remote reachable + internet reachable.
#
# States:
# NORMAL — remote up, internet up — own containers only, silent operation
# FAILOVER — remote down, internet up — start remote's containers locally (additive)
# NO_INTERNET — internet down — stop public-facing containers, wait
# DARK — remote down + internet down — same as NO_INTERNET
#
# Handback sequence when remote returns after FAILOVER:
# Strike confirmation (FAILOVER_HANDBACK_STRIKES consecutive remote-up checks)
# → pre-flight checks (remote array, docker daemon, rootfs)
# → rsync data back via rsync.sh (uses profile system)
# → start failover containers on remote via SSH
# → stop failover containers locally
# → return to NORMAL
#
# Comment out any container or rsync job to disable without removing the entry.
# Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically.
# External IP to ping for internet connectivity check — Google DNS, reliable and fast
EXTERNAL_IP="8.8.8.8"
# Seconds between state checks — 120s = 2 minute polling interval
# With 1 minute DNS TTL this means failover is visible to users within ~3 minutes
FAILOVER_CHECK_INTERVAL=120
# Consecutive remote-up confirmations required before initiating handback
# Prevents handing back during a brief network blip — 2 strikes = 4 minutes confirmation
FAILOVER_HANDBACK_STRIKES=2
# State file path — on /boot/ so it survives reboots
# Script re-evaluates from scratch on restart using live pings — state file is reference only
FAILOVER_STATE_FILE="/boot/config/failover_state.db"
# ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━
# Containers HOST1 starts locally when HOST2 goes down
# These run on top of HOST1's normal containers — additive, not replacement
# HOST2's DDNS containers must start here so DNS stays pointing at HOST1 during HOST2 outage
FAILOVER_HOST1_STARTS_FOR_HOST2=(
"Vaultwarden-Jayred365"
"Nextcloud-Jayred365"
"Cloudflare-DDNS-Jayred365"
"Gmer4Lfe.com"
"Gmer4Lfe.us"
)
# Containers HOST1 stops when it loses internet
# No point serving DDNS or media if HOST1 itself has no internet — stops unnecessary churn
# HOST2 will independently detect HOST1 is gone and start its own failover
FAILOVER_HOST1_STOP_ON_NO_NET=(
"Emby"
"Cloudflare-DDNS-Gmer4Lfe"
"Gmer4Lfe.com"
"Gmer4Lfe.us"
)
# Rsync jobs HOST1 runs before handing containers back to HOST2
# Uses rsync.sh with existing profile system — basename matched to profile keys
# Comment out jobs that don't need syncing back or aren't ready yet
FAILOVER_HOST1_RSYNC_JOBS=(
"/mnt/user/appdata-Failover/Jayred365"
"/mnt/user/Media_Server/Emby-Jayred"
# "/mnt/user/appdata-Failover/Jayred365"
# "/mnt/user/Media_Server/Emby-Jayred"
)
# ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━
# Containers HOST2 starts locally when HOST1 goes down
# Emby starts on HOST2 so media keeps working during HOST1 outage
# HOST1's DDNS containers start here so DNS updates to point at HOST2
FAILOVER_HOST2_STARTS_FOR_HOST1=(
"Emby"
"Cloudflare-DDNS-Gmer4Lfe"
"Gmer4Lfe.com"
"Gmer4Lfe.us"
)
# Containers HOST2 stops when it loses internet
# HOST2's DDNS containers serve no purpose without internet connectivity
FAILOVER_HOST2_STOP_ON_NO_NET=(
"Cloudflare-DDNS-Jayred365"
"Gmer4Lfe.com"
"Gmer4Lfe.us"
)
# Rsync jobs HOST2 runs before handing containers back to HOST1
# Comment out jobs that don't need syncing back or aren't ready yet
FAILOVER_HOST2_RSYNC_JOBS=(
"/mnt/user/appdata-Failover/Gmer4Lfe"
# "/mnt/user/appdata-Failover/Gmer4Lfe"
)
# ==============================================================================================
@@ -220,58 +345,80 @@ FAILOVER_HOST2_RSYNC_JOBS=(
# ==============================================================================================
# ━━━ Docker Daily Restart ━━━
# Containers restarted every day — keeps services fresh, clears memory leaks
# Case-sensitive — must match exact Docker container names
DAILY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Dispatcharr-Iptv-Users"
"Dispatcharr"
"Dispatcharr-Basic"
"Code-Server"
"ErsatzTV-Emby"
)
# ━━━ Docker Weekly Restart ━━━
# Containers restarted once per week — less critical services that benefit from periodic restart
WEEKLY_RESTART_CONTAINERS=(
"NginxProxyManager"
"Authelia"
"Dispatcharr-Iptv-Users"
"Dispatcharr"
"Dispatcharr-Basic"
"Code-Server"
"NextCloud"
"Organizrv2-Gmer4Lfe"
"AdGuard-Home"
"Immich-Gmer4Lfe"
)
# ━━━ Docker Watchdog ━━━
# First line of defense — monitors containers for memory, CPU and HTTP responsiveness.
# Restarts containers that exceed thresholds using a strike system to avoid false positives.
# Works alongside system_watchdog.sh — containers first, system reboot second.
# Containers to monitor with their memory hard limits in MB
# Strike system used for CPU and responsiveness — immediate restart for memory hard limit
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
declare -A WATCHDOG_CONTAINERS=(
["Emby"]=16384
["jellyfin_with_request"]=12288
["LidaTube"]=6144
["Tdarr"]=6144
["Code-Server"]=1024
["Emby"]=16384 # 16GB — large media server, transcoding can spike
["LidaTube"]=6144 # 6GB — YouTube downloader
["Tdarr"]=6144 # 6GB — transcoding node
["Code-Server"]=1024 # 1GB — VS Code server
)
# Containers to check HTTP responsiveness — curl check per interval
# Omit a container entirely to skip its HTTP check
declare -A WATCHDOG_CONTAINER_URLS=(
["Emby"]="http://localhost:8096"
["Jellyfin-Gmer4Lfe"]="http://localhost:8095"
)
# Containers that should always be running — monitored for unexpected stops
# Strike system used — persistent skip list on /boot/ prevents reboot loops
# Auto-clears from skip list when container recovers after reboot or manual fix
WATCHDOG_REQUIRED_CONTAINERS=(
"NginxProxyManager"
"Lldap-Gmer4Lfe"
"Authelia"
"Emby"
"Mariadb-Authelia"
"Redis-Authelia"
"Authelia-Secondary"
"Redis-Authelia-Secondary"
)
# Strike state file — /tmp resets on reboot which is correct for strike tracking
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
SOFT_CPU_THRESHOLD=80
HARD_CPU_THRESHOLD=85
CPU_FAIL_LIMIT=2
# CPU thresholds — normalised against total core count at runtime
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
CPU_FAIL_LIMIT=2 # consecutive strikes before container restart
# Memory — warn at this % of per-container hard limit defined in WATCHDOG_CONTAINERS
SOFT_MEM_THRESHOLD=80
RESP_FAIL_LIMIT=2
CURL_TIMEOUT=5
# HTTP responsiveness check
RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart
CURL_TIMEOUT=5 # seconds before curl gives up per check
# ━━━ Docker Network Connect ━━━
# Containers connected to extra networks on array start — many-to-many
# Every container connects to every network listed
# Connects containers to extra Docker networks on array start — many-to-many
# Every container in the list connects to every network in the list
# Useful when containers need to communicate across networks they weren't configured with
# Comment out entries to disable without removing them
NETWORK_CONNECT_CONTAINERS=(
"memcached"
@@ -287,41 +434,49 @@ NETWORK_CONNECT_NETWORKS=(
# ==============================================================================================
# ━━━ Reboot ━━━
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots
REBOOT_SLEEP=300
# ━━━ Mover ━━━
# Seconds to wait after warning users before mover_stop.sh kills the mover process
MOVER_STOP_TIMEOUT=300
# ━━━ Syslog Filter ━━━
# Path for the rsyslog filter file that suppresses Docker veth/docker0 noise
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
# ━━━ PHP-FPM ━━━
# Config file path and max children value for php_fpm_max_children.sh
PHP_CONF="/etc/php-fpm.d/www.conf"
PHP_MAX_CHILDREN=250
# ━━━ Clear Logs ━━━
# System log files cleared by clear_logs.sh — Docker logs cleared automatically too
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
# ━━━ WebGUI Watchdog ━━━
# Monitors unRAID WebGUI — escalates from nginx restart to emhttp restart if needed
WEBGUI_URL="http://localhost" # adjust port if running non-standard e.g. http://localhost:8080
WEBGUI_TIMEOUT=5 # seconds before curl gives up
WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck
WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck
# Monitors unRAID WebGUI and restarts services if unresponsive
# Escalation: nginx restart → recheck → emhttp restart → recheck → notify warning
WEBGUI_URL="http://localhost" # adjust port if non-standard e.g. http://localhost:8080
WEBGUI_TIMEOUT=5 # seconds before curl gives up
WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck
WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck
# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS health and memory diagnostic report
# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken
# system_watchdog.sh handles threshold-based intervention
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this %
ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB
ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB
ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show
ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show in report
# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# ━━━ Media Permissions ━━━
# Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES
PERMISSIONS_MODE="777"
PERMISSIONS_OWNER="nobody:users"
@@ -351,6 +506,10 @@ MEDIA_PERMISSION_SHARES=(
)
# ━━━ Media Cleaner ━━━
# Two profiles: anime and media — passed as argument to media_cleaner.sh
# Run via Orchestrators/media_management.sh for permissions + both cleaners in one job
# Usage: media_cleaner.sh anime or media_cleaner.sh media
ANIME_CLEAN_FOLDERS=(
/mnt/user/Anime_Movies
/mnt/user/Anime_Movies-Old
@@ -368,6 +527,7 @@ MEDIA_CLEAN_FOLDERS=(
/mnt/user/Tv_Shows
)
# Junk file patterns common in anime downloads
ANIME_FILE_PATTERNS=(
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
@@ -375,6 +535,7 @@ ANIME_FILE_PATTERNS=(
'*.log' '*.json'
)
# Junk file patterns for general media — includes *.iso and *.lrc not needed in anime
MEDIA_FILE_PATTERNS=(
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
@@ -382,53 +543,109 @@ MEDIA_FILE_PATTERNS=(
'*.log' '*.json' '*.iso' '*.lrc'
)
# ━━━ Media Management Orchestrator ━━━
# Scripts run sequentially by Orchestrators/media_management.sh
# Order matters — permissions runs first so cleaner operates on correctly owned files
# Format: "script_path [optional_argument]"
# Add a new line to run another script — comment out to disable without removing
MEDIA_MAINTENANCE_JOBS=(
"Media/media_shares_permissions.sh"
"Media/media_cleaner.sh anime"
"Media/media_cleaner.sh media"
)
# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem indirection.
# ffmpeg resolves the symlink path once at session start — existing sessions are never affected.
# New sessions land wherever TRANSCODE_LINK points at the moment they start.
#
# Flow:
# ramdisk_setup.sh — run once at array start, creates ramdisk and sets symlink
# transcode_manager.sh — every 3 min, monitors usage and flips symlink if needed
# transcode_cleanup.sh — every 5 min, removes old inactive files from both locations
#
# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop
# when usage hovers near the threshold — gap should be at least 0.5-1GB
RAMDISK_PATH="/mnt/ramdisk_transcodes"
RAMDISK_SIZE="8G"
TRANSCODE_LINK="/mnt/ram-transcode"
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
RAMDISK_WARN_GB=6.8
RAMDISK_LOW_GB=5.5
RAMDISK_SSD_MIN_GB=20
TRANSCODE_MAX_AGE=20
TRANSCODE_ORPHAN_AGE=30
TRANSCODE_FLIP_WARN=3
RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point — created at array start
RAMDISK_SIZE="8G" # ceiling — tmpfs is dynamic, only uses what's needed
TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — never changes location
TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback for edge cases
# Thresholds in GB — flip symlink to SSD at WARN, flip back to ramdisk at LOW
RAMDISK_WARN_GB=6.8 # flip to SSD at or above this — "getting full"
RAMDISK_LOW_GB=5.5 # flip back to ramdisk when cleanup brings usage here
RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — safety net
# Cleanup — files must be older than MAX_AGE and not open by any process to be deleted
TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup
TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution
# Flip frequency monitoring — notify if symlink flips too often (indicates sizing issue)
TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour
# Permissions — must match your Emby container user
TRANSCODE_OWNER="nobody:users"
TRANSCODE_MODE="755"
# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Last line of defense — reboots cleanly if the system is about to become unstable.
# Works alongside docker_watchdog.sh — containers first, system reboot second.
# Thresholds set at "about to fall over" levels — not just high usage.
# Strike system prevents rebooting on single spikes — sustained threshold hits trigger action.
# ━━━ State Files ━━━
# Strike counts reset on reboot — /tmp is correct for this
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
# Persistent container skip list — survives reboots, auto-clears when container recovers
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
# Reboot timestamp log — survives reboots for loop detection
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
# ━━━ Strike and Reboot Loop Settings ━━━
# Consecutive threshold hits before triggering reboot
SYS_WATCHDOG_STRIKE_LIMIT=2
# Maximum reboots allowed in window before shutdown instead — prevents reboot loops
SYS_WATCHDOG_REBOOT_LIMIT=3
# Window in hours — controls both reboot count window AND rolling log purge
# 12 = entries older than 12hrs purge automatically, fresh window starts
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
SYS_WATCHDOG_ROOTFS_PCT=95
SYS_WATCHDOG_LOG_PCT=95
SYS_WATCHDOG_MEM_GB=4
SYS_WATCHDOG_ARC_PINNED_PCT=98
SYS_WATCHDOG_ARC_RELEASE_PCT=95
SYS_WATCHDOG_LOAD_MULTIPLIER=3
SYS_WATCHDOG_ZOMBIE_LIMIT=50
SYS_WATCHDOG_CPU_TEMP_MAX=95
# ━━━ Thresholds ━━━
# Set these at "I am about to become unstable" levels — not just "things are a bit tight"
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % — at 95% something is seriously wrong
SYS_WATCHDOG_LOG_PCT=95 # /var/log % — log spam filling disk
SYS_WATCHDOG_MEM_GB=4 # free RAM GB — 4GB on 128GB system is critical
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before attempting reclaim
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers reboot
SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # zombie process count before strike
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax
# ━━━ Check Toggles ━━━
# true = run this check / false = skip entirely
# Disable checks that are not relevant to your hardware or cause false positives
SYS_WATCHDOG_CHECK_ROOTFS=true
SYS_WATCHDOG_CHECK_LOG=true
SYS_WATCHDOG_CHECK_RAM=true
SYS_WATCHDOG_CHECK_ARC=true
SYS_WATCHDOG_CHECK_CPU_TEMP=true
SYS_WATCHDOG_CHECK_LOAD=false
SYS_WATCHDOG_CHECK_LOAD=false
SYS_WATCHDOG_CHECK_ZOMBIES=true
SYS_WATCHDOG_CHECK_CONTAINERS=true
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
SYS_WATCHDOG_ABORT_ON_PARITY=true
SYS_WATCHDOG_ABORT_ON_MOVER=true
# ━━━ Abort Toggles ━━━
# true = abort reboot if condition is active / false = reboot anyway
# Default true = conservative — set false only when "reboot no matter what" is wanted
# Goal: graceful reboot before crash is always better than hard crash mid-operation
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot = potential data loss
SYS_WATCHDOG_ABORT_ON_PARITY=true # aborting parity better than crashing mid-check
SYS_WATCHDOG_ABORT_ON_MOVER=true # aborting move better than crashing mid-move
# ==============================================================================================
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
+175
View File
@@ -0,0 +1,175 @@
#!/bin/bash
# -----------------------------------------------------------------------------------------------
# --------------------------------- Media Management Orchestrator ------------------------------
# -----------------------------------------------------------------------------------------------
# Runs all media maintenance scripts sequentially in the order defined in Master.conf.
# Each job in MEDIA_MAINTENANCE_JOBS is a script path with an optional argument.
# Scripts are resolved relative to the ecosystem root directory.
#
# To add a new job — edit MEDIA_MAINTENANCE_JOBS in Master.conf:
# "Media/my_new_script.sh" — script with no argument
# "Media/media_cleaner.sh profile" — script with argument
#
# Order matters — permissions runs before cleaners so files are correctly owned first.
# Each job runs independently — a failure in one does not stop the others.
# Supports --dry-run — passes through to all child scripts.
# -----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$ECOSYSTEM_ROOT/Master.conf"
source "$ECOSYSTEM_ROOT/common.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all child scripts"
# Validate job list
if [[ ${#MEDIA_MAINTENANCE_JOBS[@]} -eq 0 ]]; then
warn "MEDIA_MAINTENANCE_JOBS is empty in Master.conf — nothing to run"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Status ━━━
# -----------------------------------------------------------------------------------------------
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
echo "$ICON_CLEAN Jobs to run: ${#MEDIA_MAINTENANCE_JOBS[@]}"
local_idx=1
for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do
echo " $local_idx. $job"
((local_idx++))
done
echo "$ICON_GEAR Dry Run: $DRY_RUN"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# -----------------------------------------------------------------------------------------------
# Tracking
# -----------------------------------------------------------------------------------------------
PASS=()
FAIL=()
JOB_TIMES=()
TOTAL_START=$(date +%s)
# -----------------------------------------------------------------------------------------------
# JOB RUNNER
# Splits each MEDIA_MAINTENANCE_JOBS entry into script path + optional argument.
# Resolves script relative to ecosystem root. Passes --dry-run if active.
# Records pass/fail and duration for summary.
# -----------------------------------------------------------------------------------------------
run_job() {
local entry="$1"
# Split entry into script path and optional argument
local script_rel arg=""
read -r script_rel arg <<< "$entry"
local script="$ECOSYSTEM_ROOT/$script_rel"
local label
label="$(basename "$script_rel" .sh)${arg:+ $arg}"
local job_start
job_start=$(date +%s)
echo ""
echo "━━━ $ICON_CLEAN $label ━━━"
if [[ ! -f "$script" ]]; then
error "$script not found — skipping"
FAIL+=("$label")
JOB_TIMES+=("$label:0")
return
fi
if [[ ! -x "$script" ]]; then
warn "$script is not executable — attempting to fix"
chmod +x "$script"
fi
local dry_flag=""
[[ "$DRY_RUN" == true ]] && dry_flag="--dry-run"
# Run script with optional argument and optional dry-run flag
if bash "$script" $arg $dry_flag; then
local job_end
job_end=$(date +%s)
PASS+=("$label")
JOB_TIMES+=("$label:$((job_end - job_start))")
success "$label complete"
else
local job_end
job_end=$(date +%s)
FAIL+=("$label")
JOB_TIMES+=("$label:$((job_end - job_start))")
error "$label failed — continuing to next job"
fi
}
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_CLEAN Media Management ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_CLEAN Media Management — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_SUMMARY Jobs: ${#MEDIA_MAINTENANCE_JOBS[@]}"
echo ""
JOB_INDEX=1
for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do
info "Job $JOB_INDEX of ${#MEDIA_MAINTENANCE_JOBS[@]}: $job"
run_job "$job"
((JOB_INDEX++))
done
TOTAL_END=$(date +%s)
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
JOB_COUNT=$(( ${#PASS[@]} + ${#FAIL[@]} ))
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY MEDIA MANAGEMENT SUMMARY ━━━━━"
echo "$ICON_TIME Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')"
echo "$ICON_TIME Finished: $(date -d @$TOTAL_END '+%Y-%m-%d %H:%M:%S')"
echo ""
for entry in "${JOB_TIMES[@]}"; do
label="${entry%%:*}"
duration="${entry##*:}"
if printf '%s\n' "${FAIL[@]}" | grep -qx "$label"; then
echo " $ICON_ERROR $label$(format_duration $duration)"
else
echo " $ICON_DONE $label$(format_duration $duration)"
fi
done
echo ""
echo " $ICON_SUCCESS Passed: ${#PASS[@]}/$JOB_COUNT"
echo " $ICON_ERROR Failed: ${#FAIL[@]}/$JOB_COUNT"
echo " $ICON_TIME Duration: $(format_duration $TOTAL_DURATION)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
if [[ ${#FAIL[@]} -gt 0 ]]; then
notify "Media management completed with failures on $(hostname) — failed: ${FAIL[*]}" "Media Management" "warning"
exit 1
else
notify "Media management complete on $(hostname)${#PASS[@]}/$JOB_COUNT jobs in $(format_duration $TOTAL_DURATION)" "Media Management" "normal"
exit 0
fi
+30 -34
View File
@@ -24,10 +24,13 @@
# v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added
# Directory tree updated to reflect full ecosystem
# v1.3 — Failover script added, directory tree updated with Failover folder
# v1.4 — WebGUI watchdog added
# ZFS memory snapshot added
# Docker network connect added
# Directory tree updated to reflect full ecosystem
# v1.4 — WebGUI watchdog, ZFS memory snapshot, Docker network connect added
# Recommended schedules section added
# v1.5 — media_management.sh orchestrator added to Orchestrators folder
# Corrected filenames: media_shares_permissions.sh, user_scripts_stop.sh
# Git repo section added
# Individual media cleaner entries replaced by media_management.sh
# Directory tree updated
# ==============================================================================================
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
@@ -45,7 +48,8 @@
# │ └── failover.sh # Mutual container failover — runs continuously
# │
# ├── Orchestrators/
# │ ── daily_sync.sh # Runs all daily media share syncs sequentially
# │ ── daily_sync.sh # Runs all daily media share syncs sequentially
# │ └── media_management.sh # Runs permissions + anime + media cleaner sequentially
# │
# ├── Rsync/
# │ ├── rsync.sh # Core rsync script — called per share or profile
@@ -58,7 +62,7 @@
# │ └── docker_network_connect.sh # Connects containers to extra networks on boot
# │
# ├── Media/
# │ ├── media_permissions.sh # Applies permissions to all media shares
# │ ├── media_shares_permissions.sh # Applies permissions to all media shares
# │ └── media_cleaner.sh # Removes junk files — profiles: anime, media
# │
# ├── Transcodes/
@@ -77,7 +81,7 @@
# ├── rsync_stop.sh # Stops all rsync processes on both servers
# ├── server_reboot.sh # Graceful server reboot with user warning
# ├── system_watchdog.sh # System health monitor — last line of defense
# ├── user_script_stop.sh # Stops running User Scripts plugin jobs
# ├── user_scripts_stop.sh # Stops running User Scripts plugin jobs
# ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp escalating restart
# └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
#
@@ -96,6 +100,7 @@
#
# ━━━ Orchestrators ━━━
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh
#/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh
#
# ━━━ Rsync — Appdata Profiles (scheduled individually) ━━━
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
@@ -125,11 +130,6 @@
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh
#/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh
#
# ━━━ Media ━━━
#/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime
#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media
#
# ━━━ Transcodes ━━━
#/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh
#/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh
@@ -153,6 +153,7 @@
# ━━━ Git repo (Gitea) ━━━
#/mnt/user/appdata/unraid_scripts/git_pull_execute.sh
#
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
# ━━━ ⚙️ Arguments — Add after the script path ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
@@ -164,10 +165,6 @@
# --help Show usage information
# KEY=VALUE Override any Master.conf variable for this run only
#
# ━━━ Media Cleaner profile (required first argument) ━━━
# media_cleaner.sh anime --dry-run
# media_cleaner.sh media --dry-run
#
# ━━━ Failover specific ━━━
# failover.sh --status — check current state
# failover.sh --dry-run --log — test logic without touching containers
@@ -176,27 +173,26 @@
# ━━━ 📋 Recommended Schedules ━━━
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
#
# failover.sh — At Startup of Array (background task)
# ramdisk_setup.sh — At Startup of Array
# failover.sh — At Startup of Array (background task)
# ramdisk_setup.sh — At Startup of Array
# docker_network_connect.sh — At Startup of Array
# docker_syslog_filter.sh — At Startup of Array
# php_fpm_max_children.sh — At Startup of Array
# docker_syslog_filter.sh — At Startup of Array
# php_fpm_max_children.sh — At Startup of Array
#
# transcode_manager.sh — */3 * * * * (every 3 minutes)
# transcode_cleanup.sh — */5 * * * * (every 5 minutes)
# docker_watchdog.sh — */15 * * * * (every 15 minutes)
# webgui_restart.sh — */10 * * * * (every 10 minutes)
# system_watchdog.sh — */15 * * * * (every 15 minutes)
# transcode_manager.sh — */3 * * * * (every 3 minutes)
# transcode_cleanup.sh — */5 * * * * (every 5 minutes)
# docker_watchdog.sh — */15 * * * * (every 15 minutes)
# webgui_restart.sh — */10 * * * * (every 10 minutes)
# system_watchdog.sh — */15 * * * * (every 15 minutes)
#
# daily_sync.sh — 0 1 * * * (daily at 1am)
# docker_daily_restart.sh — 0 3 * * * (daily at 3am)
# media_cleaner.sh anime — 0 2 * * * (daily at 2am)
# media_cleaner.sh media — 0 2 * * * (daily at 2am)
# media_permissions.sh — 0 4 * * 0 (weekly Sunday 4am)
# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am)
# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am)
# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am)
# daily_sync.sh — 0 1 * * * (daily at 1am)
# docker_daily_restart.sh — 0 3 * * * (daily at 3am)
# media_management.sh — 0 2 * * * (daily at 2am)
#
# rsync.sh profiles — schedule individually as needed
# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am)
# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am)
# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am)
#
# rsync.sh profiles — schedule individually as needed
#
# ==============================================================================================