diff --git a/Master.conf b/Master.conf index 7354b8f..86df17e 100644 --- a/Master.conf +++ b/Master.conf @@ -51,19 +51,28 @@ # ============================================================================================== # ━━━ Host Configuration ━━━ +# Hostnames must match Tailscale machine names exactly — case sensitive HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" +# SSH keys for server-to-server rsync and failover operations +# Each server authenticates with its own key — both must be authorised on the remote HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # ━━━ Logging ━━━ +# true = verbose [LOG] lines in all script output / false = user-facing output only ENABLE_LOGGING=true # ━━━ Notifications ━━━ +# unRAID native — fires through Settings → Notification Settings +# Recommended: configure unRAID to send errors/warnings only so normal completions stay quiet NOTIFY_UNRAID=true +# Discord webhook — paste full webhook URL to enable, leave blank to disable DISCORD_WEBHOOK="" # ━━━ Git / Repo ━━━ +# Gitea self-hosted repository for the script ecosystem +# git_pull_execute.sh uses these to clone or pull the latest version on both servers REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" TARGET_DIR="/mnt/user/appdata/unraid_scripts" GITEA_SSH_KEY="/root/.ssh/unraid_gitea" @@ -74,19 +83,42 @@ # ============================================================================================== # ━━━ Rsync Defaults ━━━ +# Global fallback values used when no profile match is found for a directory. +# Shares in DAILY_SYNC_SHARES always use these globals — no profile is defined for them. +# Appdata shares (Arrs_Stack, Critical-Data etc.) match profiles by directory basename. + +# Network transfer speed cap in KB/s — 12500 ≈ 100Mbit BW_LIMIT=12500 +# Number of retry attempts if rsync fails before giving up RETRY_COUNT=3 +# Seconds to wait between retry attempts SLEEP=300 +# Containers to stop on remote before rsync and restart after — empty by default +# Profiles below override this per appdata share CRITICAL_CONTAINER_NAMES=() +# Containers that need a delay before starting — e.g. Authelia needs its DB ready first DELAYED_CONTAINERS=() +# Seconds to wait before starting delayed containers CONTAINER_DELAY=5 +# Directories to exclude from transfer — empty by default, profiles override per share EXCLUDE_DIRS=() +# Default rsync options used when no profile match is found +# --delete removes files on remote that no longer exist on source +# --inplace writes directly to destination file rather than temp file — better for large files +# --no-whole-file forces delta transfer even on local-like connections DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ +# Abort rsync if remote rootfs (/) usage is at or above this percentage. +# When the remote array is down or drives are missing, rsync writes land on rootfs instead +# of /mnt/user — this fills the filesystem rapidly and can crash the remote server. +# 75% gives enough headroom to detect the problem before it becomes critical. ROOTFS_WARN=75 # ━━━ Daily Sync Shares ━━━ +# Media shares synced once daily by Orchestrators/daily_sync.sh +# These shares have no profile — all use DEFAULT_RSYNC_OPTS above +# Add or remove paths to control what gets synced each night DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows-Old @@ -103,6 +135,37 @@ DAILY_SYNC_SHARES=( ) # ━━━ Rsync Profile System ━━━ +# Profiles allow per-share rsync behaviour without touching script logic. +# Each profile is matched automatically by the basename of the directory passed to rsync.sh +# (lowercased). Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack +# +# How fallthrough works: +# If a key exists in a profile array → that value is used for this run +# If a key is missing → the global default above is used instead +# Shares in DAILY_SYNC_SHARES → always use globals, no profile defined +# +# To add a new profile: +# 1. Add a key to each array below with your chosen profile name +# 2. Call rsync.sh with a directory whose basename matches that key +# 3. Any array you omit falls back to its global default automatically +# +# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS — +# if you define a profile entry you must list all desired options explicitly +# +# Profile descriptions: +# arrs_stack — Sonarr, Radarr, Lidarr, Readarr, Prowlarr, Bazarr, Pinchflat appdata +# Lower bandwidth — runs alongside media syncs, containers stopped during sync +# critical-data — Auth stack: NPM, Authelia, Mariadb, Redis, LLDAP +# High bandwidth — small data, synced frequently, Authelia needs delayed start +# gmer4lfe — Server-specific appdata: Organizr, UptimeKuma, VaultWarden +# Medium bandwidth — personal services, no container stop needed +# important-data — NextCloud + Postgres database +# High bandwidth — NextCloud needs graceful stop before sync +# emby — Emby media server appdata and metadata +# Medium bandwidth — large appdata, no containers stopped (metadata only) + +# SPACE-SEPARATED STRINGS — rsync options per profile +# If defined for a profile, these replace DEFAULT_RSYNC_OPTS entirely for that run declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" @@ -111,14 +174,16 @@ declare -A PROFILE_RSYNC_OPTS=( [emby]="-av --human-readable --bwlimit=$BW_LIMIT" ) +# Bandwidth limit in KB/s per profile — overrides global BW_LIMIT for this profile declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 - [critical-data]=9500 - [gmer4lfe]=8000 - [important-data]=9500 - [emby]=8000 + [arrs_stack]=5000 # lower — runs alongside other jobs + [critical-data]=9500 # high — small data, sync fast + [gmer4lfe]=8000 # medium + [important-data]=9500 # high — database sync, prioritise speed + [emby]=8000 # medium — large files, steady transfer ) +# Retry attempts per profile — overrides global RETRY_COUNT declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 @@ -127,6 +192,7 @@ declare -A PROFILE_RETRY_COUNT=( [emby]=3 ) +# Sleep between retries in seconds — overrides global SLEEP declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 @@ -135,6 +201,9 @@ declare -A PROFILE_SLEEP=( [emby]=300 ) +# Containers stopped on REMOTE before rsync and restarted after — SPACE-SEPARATED STRINGS +# Only containers that need to be stopped for data consistency — running containers are fine +# for most media but databases and auth stacks need clean state during sync declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Redis-Authelia Lldap-Gmer4Lfe NginxProxyManager Authelia" @@ -143,6 +212,9 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [emby]="" ) +# Containers that need a delay before starting after rsync — SPACE-SEPARATED STRINGS +# Authelia needs its database containers (Mariadb, Redis) to be ready before it starts +# NextCloud needs Postgres to be accepting connections before it starts declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia" @@ -151,14 +223,17 @@ declare -A PROFILE_DELAYED_CONTAINERS=( [emby]="" ) +# Seconds to wait before starting delayed containers — per profile declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 - [critical-data]=10 + [critical-data]=10 # Authelia needs DB ready — 10s gives Mariadb/Redis time to start [gmer4lfe]=5 - [important-data]=10 + [important-data]=10 # NextCloud needs Postgres ready [emby]=5 ) +# Directories excluded from transfer per profile — SPACE-SEPARATED STRINGS +# logs and *.tmp are excluded universally — they are ephemeral and regenerated on start declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp" @@ -167,52 +242,102 @@ declare -A PROFILE_EXCLUDE_DIRS=( [emby]="logs *.tmp" ) +# ━━━ Profile Disk Check Toggle ━━━ +# true = skip per-disk check for this profile — use when share lives on a ZFS pool +# false = run per-disk check — use for traditional unRAID array with individual disks +# Per-disk check looks for /mnt/disk*/sharename — ZFS pools don't have this structure +# The share existence and content checks still run regardless of this setting +declare -A PROFILE_SKIP_DISK_CHECK=( + [arrs_stack]=true + [critical-data]=true + [gmer4lfe]=true + [important-data]=true + [emby]=true +) # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== -# Mutual container failover between two unRAID servers. +# Mutual container failover between two unRAID servers 50 miles apart. # Each server runs failover.sh independently — no coordination between servers. -# Decisions based solely on two ping checks: remote reachable + internet reachable. +# All decisions are based solely on two ping checks: remote reachable + internet reachable. +# +# States: +# NORMAL — remote up, internet up — own containers only, silent operation +# FAILOVER — remote down, internet up — start remote's containers locally (additive) +# NO_INTERNET — internet down — stop public-facing containers, wait +# DARK — remote down + internet down — same as NO_INTERNET +# +# Handback sequence when remote returns after FAILOVER: +# Strike confirmation (FAILOVER_HANDBACK_STRIKES consecutive remote-up checks) +# → pre-flight checks (remote array, docker daemon, rootfs) +# → rsync data back via rsync.sh (uses profile system) +# → start failover containers on remote via SSH +# → stop failover containers locally +# → return to NORMAL +# # Comment out any container or rsync job to disable without removing the entry. +# Script runs on BOTH servers — detect_hosts() selects the correct arrays automatically. +# External IP to ping for internet connectivity check — Google DNS, reliable and fast EXTERNAL_IP="8.8.8.8" +# Seconds between state checks — 120s = 2 minute polling interval +# With 1 minute DNS TTL this means failover is visible to users within ~3 minutes FAILOVER_CHECK_INTERVAL=120 +# Consecutive remote-up confirmations required before initiating handback +# Prevents handing back during a brief network blip — 2 strikes = 4 minutes confirmation FAILOVER_HANDBACK_STRIKES=2 +# State file path — on /boot/ so it survives reboots +# Script re-evaluates from scratch on restart using live pings — state file is reference only FAILOVER_STATE_FILE="/boot/config/failover_state.db" +# ━━━ HOST1 Failover Config (unRAID-Gmer4Lfe — Primary) ━━━ + # Containers HOST1 starts locally when HOST2 goes down +# These run on top of HOST1's normal containers — additive, not replacement +# HOST2's DDNS containers must start here so DNS stays pointing at HOST1 during HOST2 outage FAILOVER_HOST1_STARTS_FOR_HOST2=( - "Vaultwarden-Jayred365" - "Nextcloud-Jayred365" - "Cloudflare-DDNS-Jayred365" + "Gmer4Lfe.com" + "Gmer4Lfe.us" ) # Containers HOST1 stops when it loses internet +# No point serving DDNS or media if HOST1 itself has no internet — stops unnecessary churn +# HOST2 will independently detect HOST1 is gone and start its own failover FAILOVER_HOST1_STOP_ON_NO_NET=( - "Emby" - "Cloudflare-DDNS-Gmer4Lfe" + "Gmer4Lfe.com" + "Gmer4Lfe.us" ) # Rsync jobs HOST1 runs before handing containers back to HOST2 +# Uses rsync.sh with existing profile system — basename matched to profile keys +# Comment out jobs that don't need syncing back or aren't ready yet FAILOVER_HOST1_RSYNC_JOBS=( - "/mnt/user/appdata-Failover/Jayred365" - "/mnt/user/Media_Server/Emby-Jayred" +# "/mnt/user/appdata-Failover/Jayred365" +# "/mnt/user/Media_Server/Emby-Jayred" ) +# ━━━ HOST2 Failover Config (unRAID-Jayred365 — Secondary) ━━━ + # Containers HOST2 starts locally when HOST1 goes down +# Emby starts on HOST2 so media keeps working during HOST1 outage +# HOST1's DDNS containers start here so DNS updates to point at HOST2 FAILOVER_HOST2_STARTS_FOR_HOST1=( "Emby" - "Cloudflare-DDNS-Gmer4Lfe" + "Gmer4Lfe.com" + "Gmer4Lfe.us" ) # Containers HOST2 stops when it loses internet +# HOST2's DDNS containers serve no purpose without internet connectivity FAILOVER_HOST2_STOP_ON_NO_NET=( - "Cloudflare-DDNS-Jayred365" + "Gmer4Lfe.com" + "Gmer4Lfe.us" ) # Rsync jobs HOST2 runs before handing containers back to HOST1 +# Comment out jobs that don't need syncing back or aren't ready yet FAILOVER_HOST2_RSYNC_JOBS=( - "/mnt/user/appdata-Failover/Gmer4Lfe" +# "/mnt/user/appdata-Failover/Gmer4Lfe" ) # ============================================================================================== @@ -220,58 +345,80 @@ FAILOVER_HOST2_RSYNC_JOBS=( # ============================================================================================== # ━━━ Docker Daily Restart ━━━ +# Containers restarted every day — keeps services fresh, clears memory leaks +# Case-sensitive — must match exact Docker container names DAILY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" "Dispatcharr-Iptv-Users" "Dispatcharr" "Dispatcharr-Basic" - "Code-Server" + "ErsatzTV-Emby" ) # ━━━ Docker Weekly Restart ━━━ +# Containers restarted once per week — less critical services that benefit from periodic restart WEEKLY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Dispatcharr-Iptv-Users" - "Dispatcharr" - "Dispatcharr-Basic" - "Code-Server" + "NextCloud" + "Organizrv2-Gmer4Lfe" + "AdGuard-Home" + "Immich-Gmer4Lfe" ) # ━━━ Docker Watchdog ━━━ +# First line of defense — monitors containers for memory, CPU and HTTP responsiveness. +# Restarts containers that exceed thresholds using a strike system to avoid false positives. +# Works alongside system_watchdog.sh — containers first, system reboot second. + +# Containers to monitor with their memory hard limits in MB +# Strike system used for CPU and responsiveness — immediate restart for memory hard limit # 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 - ["jellyfin_with_request"]=12288 - ["LidaTube"]=6144 - ["Tdarr"]=6144 - ["Code-Server"]=1024 + ["Emby"]=16384 # 16GB — large media server, transcoding can spike + ["LidaTube"]=6144 # 6GB — YouTube downloader + ["Tdarr"]=6144 # 6GB — transcoding node + ["Code-Server"]=1024 # 1GB — VS Code server ) +# Containers to check HTTP responsiveness — curl check per interval +# Omit a container entirely to skip its HTTP check declare -A WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" - ["Jellyfin-Gmer4Lfe"]="http://localhost:8095" ) +# Containers that should always be running — monitored for unexpected stops +# Strike system used — persistent skip list on /boot/ prevents reboot loops +# Auto-clears from skip list when container recovers after reboot or manual fix WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" + "Lldap-Gmer4Lfe" "Authelia" - "Emby" + "Mariadb-Authelia" + "Redis-Authelia" + "Authelia-Secondary" + "Redis-Authelia-Secondary" ) +# Strike state file — /tmp resets on reboot which is correct for strike tracking WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - SOFT_CPU_THRESHOLD=80 - HARD_CPU_THRESHOLD=85 - CPU_FAIL_LIMIT=2 + +# CPU thresholds — normalised against total core count at runtime + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive strikes before container restart + +# Memory — warn at this % of per-container hard limit defined in WATCHDOG_CONTAINERS SOFT_MEM_THRESHOLD=80 - RESP_FAIL_LIMIT=2 - CURL_TIMEOUT=5 + +# HTTP responsiveness check + RESP_FAIL_LIMIT=2 # consecutive failed curl checks before restart + CURL_TIMEOUT=5 # seconds before curl gives up per check # ━━━ Docker Network Connect ━━━ -# Containers connected to extra networks on array start — many-to-many -# Every container connects to every network listed +# Connects containers to extra Docker networks on array start — many-to-many +# Every container in the list connects to every network in the list +# Useful when containers need to communicate across networks they weren't configured with # Comment out entries to disable without removing them NETWORK_CONNECT_CONTAINERS=( "memcached" @@ -287,41 +434,49 @@ NETWORK_CONNECT_NETWORKS=( # ============================================================================================== # ━━━ Reboot ━━━ +# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots REBOOT_SLEEP=300 # ━━━ Mover ━━━ +# Seconds to wait after warning users before mover_stop.sh kills the mover process MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ +# Path for the rsyslog filter file that suppresses Docker veth/docker0 noise FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ +# Config file path and max children value for php_fpm_max_children.sh PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ +# System log files cleared by clear_logs.sh — Docker logs cleared automatically too LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # ━━━ WebGUI Watchdog ━━━ -# Monitors unRAID WebGUI — escalates from nginx restart to emhttp restart if needed - WEBGUI_URL="http://localhost" # adjust port if running non-standard e.g. http://localhost:8080 - WEBGUI_TIMEOUT=5 # seconds before curl gives up - WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck - WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck +# Monitors unRAID WebGUI and restarts services if unresponsive +# Escalation: nginx restart → recheck → emhttp restart → recheck → notify warning + WEBGUI_URL="http://localhost" # adjust port if non-standard e.g. http://localhost:8080 + WEBGUI_TIMEOUT=5 # seconds before curl gives up + WEBGUI_NGINX_WAIT=15 # seconds to wait after nginx restart before recheck + WEBGUI_EMHTTP_WAIT=30 # seconds to wait after emhttp restart before recheck # ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS health and memory diagnostic report +# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken +# system_watchdog.sh handles threshold-based intervention ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC utilization above this % ZFS_REPORT_FREE_WARN_GB=10 # warn if free RAM below this GB ZFS_REPORT_AVAIL_WARN_GB=20 # warn if available RAM below this GB - ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show + ZFS_REPORT_DOCKER_TOP=10 # number of top Docker memory users to show in report # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Media Permissions ━━━ +# Mode and owner applied recursively to all shares in MEDIA_PERMISSION_SHARES PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" @@ -351,6 +506,10 @@ MEDIA_PERMISSION_SHARES=( ) # ━━━ Media Cleaner ━━━ +# Two profiles: anime and media — passed as argument to media_cleaner.sh +# Run via Orchestrators/media_management.sh for permissions + both cleaners in one job +# Usage: media_cleaner.sh anime or media_cleaner.sh media + ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old @@ -368,6 +527,7 @@ MEDIA_CLEAN_FOLDERS=( /mnt/user/Tv_Shows ) +# Junk file patterns common in anime downloads ANIME_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -375,6 +535,7 @@ ANIME_FILE_PATTERNS=( '*.log' '*.json' ) +# Junk file patterns for general media — includes *.iso and *.lrc not needed in anime MEDIA_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' @@ -382,53 +543,109 @@ MEDIA_FILE_PATTERNS=( '*.log' '*.json' '*.iso' '*.lrc' ) +# ━━━ Media Management Orchestrator ━━━ +# Scripts run sequentially by Orchestrators/media_management.sh +# Order matters — permissions runs first so cleaner operates on correctly owned files +# Format: "script_path [optional_argument]" +# Add a new line to run another script — comment out to disable without removing +MEDIA_MAINTENANCE_JOBS=( + "Media/media_shares_permissions.sh" + "Media/media_cleaner.sh anime" + "Media/media_cleaner.sh media" +) + # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# Session-based storage allocator using filesystem indirection. +# ffmpeg resolves the symlink path once at session start — existing sessions are never affected. +# New sessions land wherever TRANSCODE_LINK points at the moment they start. +# +# Flow: +# ramdisk_setup.sh — run once at array start, creates ramdisk and sets symlink +# transcode_manager.sh — every 3 min, monitors usage and flips symlink if needed +# transcode_cleanup.sh — every 5 min, removes old inactive files from both locations +# +# Hysteresis gap between RAMDISK_WARN_GB and RAMDISK_LOW_GB prevents flip-flop +# when usage hovers near the threshold — gap should be at least 0.5-1GB - RAMDISK_PATH="/mnt/ramdisk_transcodes" - RAMDISK_SIZE="8G" - TRANSCODE_LINK="/mnt/ram-transcode" - TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" - RAMDISK_WARN_GB=6.8 - RAMDISK_LOW_GB=5.5 - RAMDISK_SSD_MIN_GB=20 - TRANSCODE_MAX_AGE=20 - TRANSCODE_ORPHAN_AGE=30 - TRANSCODE_FLIP_WARN=3 + RAMDISK_PATH="/mnt/ramdisk_transcodes" # tmpfs mount point — created at array start + RAMDISK_SIZE="8G" # ceiling — tmpfs is dynamic, only uses what's needed + TRANSCODE_LINK="/mnt/ram-transcode" # symlink Emby points at — never changes location + TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # SSD fallback for edge cases + +# Thresholds in GB — flip symlink to SSD at WARN, flip back to ramdisk at LOW + RAMDISK_WARN_GB=6.8 # flip to SSD at or above this — "getting full" + RAMDISK_LOW_GB=5.5 # flip back to ramdisk when cleanup brings usage here + RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip — safety net + +# Cleanup — files must be older than MAX_AGE and not open by any process to be deleted + TRANSCODE_MAX_AGE=20 # minutes before a file is eligible for cleanup + TRANSCODE_ORPHAN_AGE=30 # minutes before an orphaned file is eligible — extra caution + +# Flip frequency monitoring — notify if symlink flips too often (indicates sizing issue) + TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour + +# Permissions — must match your Emby container user TRANSCODE_OWNER="nobody:users" TRANSCODE_MODE="755" # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# Last line of defense — reboots cleanly if the system is about to become unstable. +# Works alongside docker_watchdog.sh — containers first, system reboot second. +# Thresholds set at "about to fall over" levels — not just high usage. +# Strike system prevents rebooting on single spikes — sustained threshold hits trigger action. +# ━━━ State Files ━━━ +# Strike counts reset on reboot — /tmp is correct for this SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" +# Persistent container skip list — survives reboots, auto-clears when container recovers SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" +# Reboot timestamp log — survives reboots for loop detection SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" + +# ━━━ Strike and Reboot Loop Settings ━━━ +# Consecutive threshold hits before triggering reboot SYS_WATCHDOG_STRIKE_LIMIT=2 +# Maximum reboots allowed in window before shutdown instead — prevents reboot loops SYS_WATCHDOG_REBOOT_LIMIT=3 +# Window in hours — controls both reboot count window AND rolling log purge +# 12 = entries older than 12hrs purge automatically, fresh window starts SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 - SYS_WATCHDOG_ROOTFS_PCT=95 - SYS_WATCHDOG_LOG_PCT=95 - SYS_WATCHDOG_MEM_GB=4 - SYS_WATCHDOG_ARC_PINNED_PCT=98 - SYS_WATCHDOG_ARC_RELEASE_PCT=95 - SYS_WATCHDOG_LOAD_MULTIPLIER=3 - SYS_WATCHDOG_ZOMBIE_LIMIT=50 - SYS_WATCHDOG_CPU_TEMP_MAX=95 + +# ━━━ Thresholds ━━━ +# Set these at "I am about to become unstable" levels — not just "things are a bit tight" + SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs % — at 95% something is seriously wrong + SYS_WATCHDOG_LOG_PCT=95 # /var/log % — log spam filling disk + SYS_WATCHDOG_MEM_GB=4 # free RAM GB — 4GB on 128GB system is critical + SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before attempting reclaim + SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers reboot + SYS_WATCHDOG_LOAD_MULTIPLIER=3 # strike if load avg > cores x this value + SYS_WATCHDOG_ZOMBIE_LIMIT=50 # zombie process count before strike + SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — adjust for your CPU tjmax + +# ━━━ Check Toggles ━━━ +# true = run this check / false = skip entirely +# Disable checks that are not relevant to your hardware or cause false positives SYS_WATCHDOG_CHECK_ROOTFS=true SYS_WATCHDOG_CHECK_LOG=true SYS_WATCHDOG_CHECK_RAM=true SYS_WATCHDOG_CHECK_ARC=true SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=false + SYS_WATCHDOG_CHECK_LOAD=false SYS_WATCHDOG_CHECK_ZOMBIES=true SYS_WATCHDOG_CHECK_CONTAINERS=true SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true - SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true - SYS_WATCHDOG_ABORT_ON_PARITY=true - SYS_WATCHDOG_ABORT_ON_MOVER=true + +# ━━━ Abort Toggles ━━━ +# true = abort reboot if condition is active / false = reboot anyway +# Default true = conservative — set false only when "reboot no matter what" is wanted +# Goal: graceful reboot before crash is always better than hard crash mid-operation + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot = potential data loss + SYS_WATCHDOG_ABORT_ON_PARITY=true # aborting parity better than crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=true # aborting move better than crashing mid-move # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── diff --git a/Orchestrators/media_management.sh b/Orchestrators/media_management.sh new file mode 100644 index 0000000..3f0159a --- /dev/null +++ b/Orchestrators/media_management.sh @@ -0,0 +1,175 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Media Management Orchestrator ------------------------------ +# ----------------------------------------------------------------------------------------------- +# Runs all media maintenance scripts sequentially in the order defined in Master.conf. +# Each job in MEDIA_MAINTENANCE_JOBS is a script path with an optional argument. +# Scripts are resolved relative to the ecosystem root directory. +# +# To add a new job — edit MEDIA_MAINTENANCE_JOBS in Master.conf: +# "Media/my_new_script.sh" — script with no argument +# "Media/media_cleaner.sh profile" — script with argument +# +# Order matters — permissions runs before cleaners so files are correctly owned first. +# Each job runs independently — a failure in one does not stop the others. +# Supports --dry-run — passes through to all child scripts. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +source "$ECOSYSTEM_ROOT/Master.conf" +source "$ECOSYSTEM_ROOT/common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all child scripts" + +# Validate job list +if [[ ${#MEDIA_MAINTENANCE_JOBS[@]} -eq 0 ]]; then + warn "MEDIA_MAINTENANCE_JOBS is empty in Master.conf — nothing to run" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_CLEAN Jobs to run: ${#MEDIA_MAINTENANCE_JOBS[@]}" + local_idx=1 + for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do + echo " $local_idx. $job" + ((local_idx++)) + done + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# Tracking +# ----------------------------------------------------------------------------------------------- +PASS=() +FAIL=() +JOB_TIMES=() +TOTAL_START=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# JOB RUNNER +# Splits each MEDIA_MAINTENANCE_JOBS entry into script path + optional argument. +# Resolves script relative to ecosystem root. Passes --dry-run if active. +# Records pass/fail and duration for summary. +# ----------------------------------------------------------------------------------------------- +run_job() { + local entry="$1" + + # Split entry into script path and optional argument + local script_rel arg="" + read -r script_rel arg <<< "$entry" + + local script="$ECOSYSTEM_ROOT/$script_rel" + local label + label="$(basename "$script_rel" .sh)${arg:+ $arg}" + + local job_start + job_start=$(date +%s) + + echo "" + echo "━━━ $ICON_CLEAN $label ━━━" + + if [[ ! -f "$script" ]]; then + error "$script not found — skipping" + FAIL+=("$label") + JOB_TIMES+=("$label:0") + return + fi + + if [[ ! -x "$script" ]]; then + warn "$script is not executable — attempting to fix" + chmod +x "$script" + fi + + local dry_flag="" + [[ "$DRY_RUN" == true ]] && dry_flag="--dry-run" + + # Run script with optional argument and optional dry-run flag + if bash "$script" $arg $dry_flag; then + local job_end + job_end=$(date +%s) + PASS+=("$label") + JOB_TIMES+=("$label:$((job_end - job_start))") + success "$label complete" + else + local job_end + job_end=$(date +%s) + FAIL+=("$label") + JOB_TIMES+=("$label:$((job_end - job_start))") + error "$label failed — continuing to next job" + fi +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Media Management ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Media Management — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_SUMMARY Jobs: ${#MEDIA_MAINTENANCE_JOBS[@]}" +echo "" + +JOB_INDEX=1 +for job in "${MEDIA_MAINTENANCE_JOBS[@]}"; do + info "Job $JOB_INDEX of ${#MEDIA_MAINTENANCE_JOBS[@]}: $job" + run_job "$job" + ((JOB_INDEX++)) +done + +TOTAL_END=$(date +%s) +TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) +JOB_COUNT=$(( ${#PASS[@]} + ${#FAIL[@]} )) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY MEDIA MANAGEMENT SUMMARY ━━━━━" +echo "$ICON_TIME Started: $(date -d @$TOTAL_START '+%Y-%m-%d %H:%M:%S')" +echo "$ICON_TIME Finished: $(date -d @$TOTAL_END '+%Y-%m-%d %H:%M:%S')" +echo "" + +for entry in "${JOB_TIMES[@]}"; do + label="${entry%%:*}" + duration="${entry##*:}" + if printf '%s\n' "${FAIL[@]}" | grep -qx "$label"; then + echo " $ICON_ERROR $label — $(format_duration $duration)" + else + echo " $ICON_DONE $label — $(format_duration $duration)" + fi +done + +echo "" +echo " $ICON_SUCCESS Passed: ${#PASS[@]}/$JOB_COUNT" +echo " $ICON_ERROR Failed: ${#FAIL[@]}/$JOB_COUNT" +echo " $ICON_TIME Duration: $(format_duration $TOTAL_DURATION)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +if [[ ${#FAIL[@]} -gt 0 ]]; then + notify "Media management completed with failures on $(hostname) — failed: ${FAIL[*]}" "Media Management" "warning" + exit 1 +else + notify "Media management complete on $(hostname) — ${#PASS[@]}/$JOB_COUNT jobs in $(format_duration $TOTAL_DURATION)" "Media Management" "normal" + exit 0 +fi \ No newline at end of file diff --git a/user_script_pluin.sh b/user_script_pluin.sh index bae1474..0c562f6 100644 --- a/user_script_pluin.sh +++ b/user_script_pluin.sh @@ -24,10 +24,13 @@ # v1.2 — Docker Essentials, Media, Transcodes, System Watchdog added # Directory tree updated to reflect full ecosystem # v1.3 — Failover script added, directory tree updated with Failover folder -# v1.4 — WebGUI watchdog added -# ZFS memory snapshot added -# Docker network connect added -# Directory tree updated to reflect full ecosystem +# v1.4 — WebGUI watchdog, ZFS memory snapshot, Docker network connect added +# Recommended schedules section added +# v1.5 — media_management.sh orchestrator added to Orchestrators folder +# Corrected filenames: media_shares_permissions.sh, user_scripts_stop.sh +# Git repo section added +# Individual media cleaner entries replaced by media_management.sh +# Directory tree updated # ============================================================================================== # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -45,7 +48,8 @@ # │ └── failover.sh # Mutual container failover — runs continuously # │ # ├── Orchestrators/ -# │ └── daily_sync.sh # Runs all daily media share syncs sequentially +# │ ├── daily_sync.sh # Runs all daily media share syncs sequentially +# │ └── media_management.sh # Runs permissions + anime + media cleaner sequentially # │ # ├── Rsync/ # │ ├── rsync.sh # Core rsync script — called per share or profile @@ -58,7 +62,7 @@ # │ └── docker_network_connect.sh # Connects containers to extra networks on boot # │ # ├── Media/ -# │ ├── media_permissions.sh # Applies permissions to all media shares +# │ ├── media_shares_permissions.sh # Applies permissions to all media shares # │ └── media_cleaner.sh # Removes junk files — profiles: anime, media # │ # ├── Transcodes/ @@ -77,7 +81,7 @@ # ├── rsync_stop.sh # Stops all rsync processes on both servers # ├── server_reboot.sh # Graceful server reboot with user warning # ├── system_watchdog.sh # System health monitor — last line of defense -# ├── user_script_stop.sh # Stops running User Scripts plugin jobs +# ├── user_scripts_stop.sh # Stops running User Scripts plugin jobs # ├── webgui_restart.sh # WebGUI watchdog — nginx + emhttp escalating restart # └── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report # @@ -96,6 +100,7 @@ # # ━━━ Orchestrators ━━━ #/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/media_management.sh # # ━━━ Rsync — Appdata Profiles (scheduled individually) ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack @@ -125,11 +130,6 @@ #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_weekly_restart.sh #/mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_network_connect.sh # -# ━━━ Media ━━━ -#/mnt/user/appdata/unraid_scripts/Media/media_shares_permissions.sh -#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh anime -#/mnt/user/appdata/unraid_scripts/Media/media_cleaner.sh media -# # ━━━ Transcodes ━━━ #/mnt/user/appdata/unraid_scripts/Transcodes/ramdisk_setup.sh #/mnt/user/appdata/unraid_scripts/Transcodes/transcode_manager.sh @@ -153,6 +153,7 @@ # ━━━ Git repo (Gitea) ━━━ #/mnt/user/appdata/unraid_scripts/git_pull_execute.sh # + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ━━━ ⚙️ Arguments — Add after the script path ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -164,10 +165,6 @@ # --help Show usage information # KEY=VALUE Override any Master.conf variable for this run only # -# ━━━ Media Cleaner profile (required first argument) ━━━ -# media_cleaner.sh anime --dry-run -# media_cleaner.sh media --dry-run -# # ━━━ Failover specific ━━━ # failover.sh --status — check current state # failover.sh --dry-run --log — test logic without touching containers @@ -176,27 +173,26 @@ # ━━━ 📋 Recommended Schedules ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # -# failover.sh — At Startup of Array (background task) -# ramdisk_setup.sh — At Startup of Array +# failover.sh — At Startup of Array (background task) +# ramdisk_setup.sh — At Startup of Array # docker_network_connect.sh — At Startup of Array -# docker_syslog_filter.sh — At Startup of Array -# php_fpm_max_children.sh — At Startup of Array +# docker_syslog_filter.sh — At Startup of Array +# php_fpm_max_children.sh — At Startup of Array # -# transcode_manager.sh — */3 * * * * (every 3 minutes) -# transcode_cleanup.sh — */5 * * * * (every 5 minutes) -# docker_watchdog.sh — */15 * * * * (every 15 minutes) -# webgui_restart.sh — */10 * * * * (every 10 minutes) -# system_watchdog.sh — */15 * * * * (every 15 minutes) +# transcode_manager.sh — */3 * * * * (every 3 minutes) +# transcode_cleanup.sh — */5 * * * * (every 5 minutes) +# docker_watchdog.sh — */15 * * * * (every 15 minutes) +# webgui_restart.sh — */10 * * * * (every 10 minutes) +# system_watchdog.sh — */15 * * * * (every 15 minutes) # -# daily_sync.sh — 0 1 * * * (daily at 1am) -# docker_daily_restart.sh — 0 3 * * * (daily at 3am) -# media_cleaner.sh anime — 0 2 * * * (daily at 2am) -# media_cleaner.sh media — 0 2 * * * (daily at 2am) -# media_permissions.sh — 0 4 * * 0 (weekly Sunday 4am) -# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am) -# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am) -# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am) +# daily_sync.sh — 0 1 * * * (daily at 1am) +# docker_daily_restart.sh — 0 3 * * * (daily at 3am) +# media_management.sh — 0 2 * * * (daily at 2am) # -# rsync.sh profiles — schedule individually as needed +# clear_logs.sh — 0 5 * * 0 (weekly Sunday 5am) +# zfs_memory_snapshot.sh — 0 6 * * 0 (weekly Sunday 6am) +# docker_weekly_restart.sh — 0 3 * * 0 (weekly Sunday 3am) +# +# rsync.sh profiles — schedule individually as needed # # ============================================================================================== \ No newline at end of file