#!/bin/bash # ============================================================================================== # ================================= MASTER CONFIGURATION ======================================= # ============================================================================================== # All user-facing variables for the unRAID script ecosystem. # Scripts source this file — edit here, changes apply everywhere on next git pull. # # ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── # Every script sources Master.conf and common.sh at startup. # Change a value here and it affects all scripts that use it — no hunting through files. # To disable something: comment it out with # rather than deleting it. # To add a new rsync profile: add a key to each PROFILE_* array. # To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. # # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── # # Section Description # ─────────────────────────────────────────────────────────────────────────────────────────── # HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR # LOGGING Enable or disable verbose logging # NOTIFICATIONS unRAID native and Discord webhook settings # GIT / REPO Gitea repository and SSH settings # # ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── # ARRAY START Scripts launched at array start (array_start.sh) # DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) # WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) # MEDIA MANAGEMENT Job list for media_management.sh # # ── RSYNC ────────────────────────────────────────────────────────────────────────────────── # RSYNC DEFAULTS Global fallback rsync options and limits # REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort # RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs # # ── FAILOVER ─────────────────────────────────────────────────────────────────────────────── # FAILOVER Mutual container failover between two servers # FAILOVER TEST Simulated outage settings for failover_test.sh # DDNS Script-controlled DDNS — absolute rules # INTERNET LOSS Containers to stop when internet is lost # TIERED CONTAINER LISTS What each server runs for the other per tier # TIER DELAY SETTINGS How long before each tier activates (minutes) # RSYNC WRITEBACK JOBS Appdata synced back to primary on handback # # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly # DOCKER WATCHDOG Continuous two-tier self-healing container monitoring # DOCKER NETWORK CONNECT Connect containers to extra networks on array start # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── # REBOOT User warning delay before scheduled reboot # MOVER Mover stop timeout # SYSLOG FILTER Docker veth noise filter file path # PHP-FPM PHP-FPM max children config # CLEAR LOGS System log file paths # WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart # # ── MEDIA ────────────────────────────────────────────────────────────────────────────────── # MEDIA PERMISSIONS Share list, mode and owner for permissions script # MEDIA CLEANER Anime and media folder lists and file patterns # ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup # ARR FAILED/STALLED RECOVERY Auto blocklist + re-search failed imports and stalled downloads # # ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── # TRANSCODE MANAGER Ramdisk and SSD fallback transcode management # TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) # # ── MONITORS ─────────────────────────────────────────────────────────────────────────────── # CERTIFICATE MONITOR SSL certificate expiry monitoring # BACKUP VERIFY Random sample checksum verification against remote # SMART HEALTH Drive SMART attribute monitoring # ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report # BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary # HEALTH DIGEST Aggregated system health digest — always/smart/weekly # EMBY SESSION REPORT Weekly Emby usage statistics via API # # ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── # SYSTEM WATCHDOG Continuous system health monitoring — last line of defense # # ============================================================================================== # ============================================================================================== # ── HOST CONFIGURATION ──────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Hosts ━━━ # Hostnames must match exact Docker/unRAID hostnames — case sensitive. # Used by detect_hosts() in common.sh to determine which server is local and which is remote. # Both servers run identical scripts — host detection makes them bidirectional. HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" # Data directory — persistent script state and statistics files. # Array share — survives reboots, no flash drive wear. # Created automatically if it doesn't exist. # Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. DATA_DIR="/mnt/user/appdata/unraid_scripts/data" # SSH keys for server-to-server rsync and failover container operations. # Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # ━━━ Emby ━━━ # Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, # emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array. # API key: Emby Dashboard → API Keys → + New Key HOST1_EMBY_CONTAINER="Emby" HOST1_EMBY_URL="http://localhost:8096" HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" HOST2_EMBY_CONTAINER="Emby-Jayred365" HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key HOST2_EMBY_API_KEY="your-host2-emby-api-key" # ============================================================================================== # ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Controls verbose [LOG] output across all scripts. # true = show detailed [LOG] lines — useful for debugging or first-time setup # false = show only user-facing output — cleaner for scheduled runs ENABLE_LOGGING=true # ============================================================================================== # ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== # unRAID native notification system — integrates with the bell icon in the WebGUI. # normal = job completed successfully / warning = something failed or needs attention NOTIFY_UNRAID=true # Discord webhook URL — leave blank to disable DISCORD_WEBHOOK="" # ============================================================================================== # ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Gitea self-hosted repository — used by git_pull_execute.sh. # Detects Gitea container location at runtime — works through failover automatically. # Falls back to GITEA_DOMAIN if local and Tailscale both fail. GITEA_CONTAINER="Gitea" # exact Docker container name GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" # repo path on Gitea server GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to GITEA_SSH_KEY="/root/.ssh/unraid_gitea" # SSH key for authenticating to Gitea SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) # ============================================================================================== # ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # All orchestrator job lists live here — edit arrays to add/remove scripts. # No changes to orchestrator scripts needed when adding or removing jobs. # ━━━ Array Start ━━━ # Scripts launched by array_start.sh when the array comes online. # Launched in order — each as a background process. # One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. # Continuous scripts (watchdogs, failover) run until array stops. ARRAY_START_SCRIPTS=( "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop # "Failover/failover.sh" # mutual failover — continuous loop ) # ━━━ Daily Sync Maintenance ━━━ # daily_sync_maintenance.sh runs the media share sync built into the script first, # then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. # Schedule: 0 1 * * * (1am daily) DAILY_MAINTENANCE_SCRIPTS=( "git_pull_execute.sh" # pull latest scripts — always runs first "Docker_Essentials/docker_daily_restart.sh" # daily container restarts ) # Media shares synced daily by daily_sync_maintenance.sh. # Each server syncs only the shares it owns (source of truth) — direction is automatic. # HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. # Never both pushing the same share — one server is always the truth holder. # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. # For shares needing custom options or container stops — create a profile in the RSYNC section. HOST1_DAILY_SYNC_SHARES=( /mnt/user/Books /mnt/user/Intros /mnt/user/Kids_Movies /mnt/user/Kids_Tv_Shows /mnt/user/Movies /mnt/user/Music /mnt/user/Music_Videos /mnt/user/Nextcloud /mnt/user/stand-up_comedy /mnt/user/Sports /mnt/user/Tv_Shows /mnt/user/Anime_Shows-Old /mnt/user/Anime_Movies-Old ) HOST2_DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Shows ) # Personal encrypted shares — synced for offsite backup, independent of media shares. # ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. # See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. HOST1_PERSONAL_SHARES=( # /mnt/user/Gmer4Lfe-Personal # uncomment after creating encrypted dataset ) HOST2_PERSONAL_SHARES=( # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset ) # ━━━ Weekly Sync Maintenance ━━━ # weekly_sync_maintenance.sh handles the critical sync built into the script first: # stop containers both sides → pull updates → sync Emby + Critical-Data → restart # Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. # Schedule: 30 2 * * 0 (Sunday 2:30am) WEEKLY_MAINTENANCE_SCRIPTS=( "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync ) # Shares synced by weekly_sync_maintenance.sh during the maintenance window. # Containers are stopped both sides before these sync — full clean state guaranteed. # Profiles drive container stops, excludes, and options — configure in RSYNC section. # Order matters — Emby first, then auth stack. WEEKLY_SYNC_JOBS=( "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack ) # Container update toggles for the weekly sync window. # Containers are already stopped for the sync — updates pull at no extra downtime. # Both false → sync only, no updates. # Toggle false temporarily to skip updates without changing the schedule. CRITICAL_SYNC_UPDATES=true # pull container updates locally CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH # ━━━ Media Management ━━━ # Job list run directly by daily_sync_maintenance.sh after the media share sync. # Runs sequentially — permissions first, then cleaners, then arr cleanup. # Comment out any job to disable without removing it. # Each individual script can still be run manually for one-off maintenance. MEDIA_MANAGEMENT_JOBS=( "Media/media_shares_permissions.sh" # apply permissions — runs first "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares # "Media/lidarr_cleanup.sh" # remove orphaned music files # "Media/sonarr_cleanup.sh" # remove orphaned TV files # "Media/radarr_cleanup.sh" # remove orphaned movie files ) # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. # Appdata shares match profiles by directory basename (lowercased). # If a profile key exists it overrides the global. If missing the global is used. BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap RETRY_COUNT=3 # retry attempts if rsync fails before giving up SLEEP=300 # seconds between retry attempts CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override CONTAINER_DELAY=5 # seconds to wait before starting delayed containers EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override # --delete removes files on remote that no longer exist on source (mirror behaviour) # --inplace writes directly to destination — better for large files, avoids temp copies # --no-whole-file forces delta transfer even on fast connections — sends only changed blocks DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ # Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. # When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. ROOTFS_WARN=75 # ━━━ Rsync Profile System ━━━ # Profiles allow per-share rsync behaviour without touching script logic. # Profile key matched by basename of directory passed to rsync.sh (lowercased). # Override with --profile=name flag. # # IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. # List ALL desired options explicitly when defining a profile. # # Current profiles: # arrs_stack — arr databases — lower bandwidth, containers stopped for consistency # critical-data — auth stack — containers stopped both sides, Authelia delayed start # gmer4lfe — server-specific appdata — no container stops needed # important-data — NextCloud + Postgres — NextCloud delayed start after Postgres # emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes # called by weekly_sync_maintenance.sh only — do NOT schedule separately # emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only # also used for failover writeback on handback declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" ) # Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only # Lower for shares running alongside other jobs, higher for time-sensitive critical data declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link [critical-data]=9500 # high — small dataset, get it synced fast and clean [gmer4lfe]=8000 [important-data]=9500 # high — database sync needs to be fast [emby]=8000 # medium — large full mirror, steady transfer [emby-failover]=9500 # high — small critical dataset, sync as fast as possible ) # Retry attempts per profile — how many times to retry before giving up on a failed sync declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 [gmer4lfe]=3 [important-data]=3 [emby]=3 [emby-failover]=3 ) # Seconds to wait between retry attempts # emby-failover shorter — frequent sync, faster retry on transient failures declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 [gmer4lfe]=300 [important-data]=300 [emby]=300 [emby-failover]=120 # shorter — frequent dirty sync, retry faster ) # Containers stopped on BOTH LOCAL and REMOTE servers before rsync. # Local stops first — flushes databases cleanly before pushing data out. # Remote stops next — prevents writes to destination while receiving. # Only containers that were running get restarted — stopped containers stay stopped. # Same container names on both servers — consistent naming is required by this ecosystem. # If a container is not found on a server it is skipped gracefully, not errored. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" [important-data]="Postgres-NextCloud NextCloud" [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync ) # Containers that need a delay before starting after rsync completes. # Database containers must be accepting connections before dependent apps start. # Authelia waits for Mariadb + Redis. NextCloud waits for Postgres. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready [gmer4lfe]="" [important-data]="NextCloud" # wait for Postgres to accept connections [emby]="" [emby-failover]="" ) # Seconds to wait before starting delayed containers # 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=15 # Mariadb + Redis need time to accept connections [gmer4lfe]=5 [important-data]=10 # Postgres needs time before NextCloud [emby]=5 [emby-failover]=5 ) # Directories excluded from rsync transfer per profile # emby-failover excludes WAL files — safe to sync while Emby is running # emby clean sync only excludes logs, transcodes, cache — full metadata mirror # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [gmer4lfe]="logs *.tmp" [important-data]="logs *.tmp" [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" [emby]="logs transcodes cache crash*" # emby-failover: Emby running, WAL excluded — only safe critical data synced # users.db, library.db, authentication.db, config/ — everything else excluded [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" ) # Skip per-disk space check for these profiles — appdata syncs go to cache/appdata # not to array disks, so disk space check is irrelevant and just slows things down declare -A PROFILE_SKIP_DISK_CHECK=( [arrs_stack]=true [critical-data]=true [gmer4lfe]=true [important-data]=true [emby]=true [emby-failover]=true ) # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Mutual container failover between two unRAID servers. # Each server runs Failover/failover.sh independently via array_start.sh. # All decisions based on two pings: remote reachable + internet reachable. # # States: NORMAL | FAILOVER | NO_INTERNET | DARK # # DDNS rules — absolute: # Internet loss → stop own DDNS immediately # Failover → start remote DDNS first (Tier 1) # Handback → stop remote DDNS → rsync → start containers → start local DDNS last # # Tiers: # Tier 1 — Immediate — vital services + Live TV # Tier 2 — configurable delay — productivity services # Tier 3 — configurable delay — secondary services # Tier 4 — configurable delay — arrs + downloaders EXTERNAL_IP="8.8.8.8" FAILOVER_CHECK_INTERVAL=120 FAILOVER_HANDBACK_STRIKES=2 FAILOVER_STATE_FILE="/boot/config/failover_state.db" # ━━━ Failover Test ━━━ FAILOVER_TEST_BLOCK_WAIT=150 FAILOVER_TEST_HANDBACK_WAIT=360 # ━━━ DDNS ━━━ HOST1_DDNS_CONTAINERS=( "Gmer4Lfe.com" ) HOST2_DDNS_CONTAINERS=( "Gmer4Lfe.us" ) # ━━━ Internet Loss ━━━ FAILOVER_HOST1_STOP_ON_NO_NET=( "Gmer4Lfe.com" ) FAILOVER_HOST2_STOP_ON_NO_NET=( "Gmer4Lfe.us" ) # ━━━ Tiered Container Lists ━━━ # HOST1 runs for HOST2 FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( "Gmer4Lfe.us" "VaultWarden-Jayred365" # "container-placeholder" ) FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( # "container-placeholder" ) FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( # "container-placeholder" ) FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( # "container-placeholder" ) # HOST2 runs for HOST1 FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( "Gmer4Lfe.com" "Emby" "VaultWarden-Gmer4Lfe" "Dispatcharr" "Dispatcharr-Basic" "Dispatcharr-Iptv-Users" "ErsatzTV-Emby" ) FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( "Postgres-NextCloud" "NextCloud" "PostgreSQL_Immich" "Immich-Gmer4Lfe" # "container-placeholder" ) FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( "Gitea" # "container-placeholder" ) FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( "Sonarr" "Radarr" "Lidarr" "Readarr" "Prowlarr" "Bazarr" "SABnzbd-Gmer4Lfe" "Qbittorrent-Gmer4Lfe" "LidaTube" "Pinchflat" "ChannelTube" # "container-placeholder" ) # ━━━ Tier Delay Settings ━━━ # How long the primary server must be down before each tier activates — in minutes. # Tier 1 is always immediate — Live TV and media can't wait. # Set independently per host — adjust based on hardware and what's worth starting. # Longer delays = less resource usage on covering server but slower recovery. # # HOST1's containers running on HOST2 (HOST1 is down): HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc. HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders # HOST2's containers running on HOST1 (HOST2 is down): HOST2_TIER2_DELAY=240 HOST2_TIER3_DELAY=720 HOST2_TIER4_DELAY=1440 # ━━━ Rsync Writeback Jobs ━━━ # Syncs critical appdata BACK to primary server during handback after failover. # Containers are stopped before writeback runs — clean source, no competing writes. # Purpose: primary comes back online with the state that built up during its outage # (watch states, auth changes, library updates that happened on HOST2) # # HOST*_TIER1_WRITEBACK_DELAY: # Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data # Only writeback if outage lasted longer than this many minutes # 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby # # Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here # Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr HOST2_TIER1_WRITEBACK_DELAY=60 # HOST1 writeback — run by HOST2 during HOST1 handback FAILOVER_HOST1_WRITEBACK_TIER1=( "/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage ) FAILOVER_HOST1_WRITEBACK_TIER2=( "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage ) FAILOVER_HOST1_WRITEBACK_TIER3=( # "location-placeholder" ) FAILOVER_HOST1_WRITEBACK_TIER4=( # Edge cases outside HOST1_DAILY_SYNC_SHARES "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage ) # HOST2 writeback — run by HOST1 during HOST2 handback FAILOVER_HOST2_WRITEBACK_TIER1=( # "/mnt/user/appdata-Failover/Jayred365-Emby" ) FAILOVER_HOST2_WRITEBACK_TIER2=( # "/mnt/user/appdata-Failover/Jayred365-Important" ) FAILOVER_HOST2_WRITEBACK_TIER3=( # "location-placeholder" ) FAILOVER_HOST2_WRITEBACK_TIER4=( # Edge cases outside HOST2_DAILY_SYNC_SHARES "/mnt/user/appdata-Failover/Arrs_Stack" ) # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Docker Daily Restart ━━━ # Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. # These containers run better with a daily restart — not just "keeping things fresh". # Dispatcharr specifically degrades over time without restart — daily is intentional. # Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window. # Case-sensitive — must match exact Docker container names. DAILY_RESTART_CONTAINERS=( "NginxProxyManager" "Authelia" "Dispatcharr-Iptv-Users" "Dispatcharr" # Live TV scheduler — degrades without daily restart "Dispatcharr-Basic" "ErsatzTV-Emby" ) # ━━━ Docker Weekly Restart ━━━ # Less critical services restarted weekly by docker_weekly_restart.sh. # Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped # for the weekly sync window so restart adds zero extra downtime. # Weekly restarts also catch any pending image updates not applied during weekly sync. WEEKLY_RESTART_CONTAINERS=( "NextCloud" "Organizrv2-Gmer4Lfe" "AdGuard-Home" "Immich-Gmer4Lfe" ) # ━━━ Docker Watchdog ━━━ # Continuous two-tier self-healing container monitoring. # Started by array_start.sh — runs until array stops. # Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. # Silent when all healthy — only logs when something needs attention. # Heartbeat fires periodically as proof of life even when everything is healthy. # # Tier 1 — strict monitoring of explicitly configured containers: # Memory hard limits — immediate restart if container exceeds limit # CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes # HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks # Required containers — must always be running, strike + skip list with auto-clear # # Tier 2 — global health scan of ALL running containers: # Unhealthy status — Docker HEALTHCHECK unhealthy → restart # OOM killed — kernel killed container → restart + notify # Crash loop detection — RestartCount climbing → notify, critical above limit # Dead containers — remove and restart # Unexpected exits — non-zero exit code → restart # Memory hard limits in MB — immediate restart if exceeded # Container restarted the moment it crosses this line — no strike system # 20GB=20480 16GB=16384 12GB=12288 10GB=10240 # 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 declare -A WATCHDOG_CONTAINERS=( ["Emby"]=16384 ["LidaTube"]=6144 ["Tdarr"]=6144 ["Code-Server"]=1024 ) # HTTP health check URLs — checked every cycle, strike system before restart # Container must respond with HTTP 200 within CURL_TIMEOUT seconds # Per-host — HOST1 and HOST2 may run different containers on different ports declare -A HOST1_WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ) declare -A HOST2_WATCHDOG_CONTAINER_URLS=( ["Emby"]="http://localhost:8096" ) # Required containers — must always be running # Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt # Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window # Skip list auto-clears when container recovers — no manual intervention needed # Per-host — each server has different critical containers HOST1_WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" "Lldap-Gmer4Lfe" "Authelia" "Mariadb-Authelia" "Redis-Authelia" "Authelia-Secondary" "Redis-Authelia-Secondary" ) HOST2_WATCHDOG_REQUIRED_CONTAINERS=( "NginxProxyManager" # add HOST2 required containers here ) # Strike state file — /tmp resets on reboot which is correct # Fresh start after reboot means no stale strikes carrying over WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" # CPU thresholds — normalised against total core count automatically at runtime # SOFT = warn only, HARD = strike toward restart # CPU_FAIL_LIMIT = consecutive HARD strikes before restart SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart # Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit # Does not trigger restart — informational only SOFT_MEM_THRESHOLD=80 # HTTP responsiveness — consecutive failed checks before restart # CURL_TIMEOUT = seconds before curl gives up on a single check RESP_FAIL_LIMIT=2 # consecutive failed checks before restart CURL_TIMEOUT=5 # seconds per check before timeout # How often the watchdog runs its checks # 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly # Containers have this long to recover before next check DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles # Heartbeat — proof of life logged periodically even when everything is healthy # Useful to confirm the watchdog is still running without flooding logs DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) # Tier 2 master toggle — set false to disable global container scanning entirely # When false only WATCHDOG_CONTAINERS and required containers are monitored WATCHDOG_SCAN_ALL=true # Containers to skip in Tier 2 scan entirely # Useful for containers that legitimately exit/restart frequently WATCHDOG_SCAN_IGNORE=( "DashGate" "PIA-WG-Config-Generator" "Aperture" "Aperture-Kids" "pgvector-18-Apeture-Kids" "Pgvector18-Aperture" ) # Individual Tier 2 check toggles — disable specific checks without disabling Tier 2 WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy WATCHDOG_RESTART_DEAD=true # restart containers in dead state WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing # Crash loop threshold — notify critical if Docker has restarted this many times total # Above this number the notification escalates to critical — manual intervention needed WATCHDOG_CRASH_LIMIT=5 # Startup grace period — skip restarts while system is still booting after array start # Prevents watchdog from restarting containers that are legitimately still initializing WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures # Restart loop protection — stops hammering a broken container # If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list # Skip list auto-clears when container recovers healthy WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" # rolling restart history for loop detection # Notification batching — one clean summary per cycle instead of one ping per event # true = batch all events into one notification at end of cycle # false = send one notification per event (noisy on busy systems) WATCHDOG_BATCH_NOTIFY=true # Dependency ordering — skip restarting a container if its dependency is also down # Prevents restarting Authelia before its database is ready # Space-separated list of dependencies per container declare -A WATCHDOG_DEPENDENCIES=( ["Authelia"]="Mariadb-Authelia Redis-Authelia" ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" ["NextCloud"]="Postgres-NextCloud" ) # ━━━ Docker Network Connect ━━━ # Ensures custom Docker networks exist then connects specified containers to them. # Two operations in one pass — no separate ensure script needed. # # Step 1 — network ensure: # Checks each network in NETWORK_CONNECT_NETWORKS # Missing → creates it (bridge driver, Docker assigns subnet automatically) # Exists → skips creation # unRAID updates occasionally wipe custom networks — this self-heals on next array start # # Step 2 — container connect: # Connects each container in NETWORK_CONNECT_CONTAINERS to each network # Container not found → warn and skip (not an error — may not be running on this host) # Already connected → skip cleanly # # Use case: # high-availability is the main custom bridge — most containers run here # Containers with their own networks (NextCloud AIO, etc.) can be listed in # NETWORK_CONNECT_CONTAINERS to gain access to high-availability # without touching their primary network — they simply join both NETWORK_CONNECT_CONTAINERS=( # "memcached" # "Npm-CrowdSec" ) NETWORK_CONNECT_NETWORKS=( "high-availability" # must exist before array start — create in Docker settings ) # ============================================================================================== # ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Reboot ━━━ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. # Gives users time to save work — 300s = 5 minutes REBOOT_SLEEP=300 # ━━━ Mover ━━━ # Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. # Gives mover time to finish current file transfer before being interrupted. MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ # Path for the rsyslog filter file that suppresses Docker veth interface noise. # Docker creates a new veth interface for each container — generates hundreds of # log lines per hour that have no diagnostic value. Filter removes them at source. FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ # Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. # Default is very low — increasing it prevents WebGUI slowdowns under load. # 250 is safe for servers with 32GB+ RAM. PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ # System log files cleared weekly to prevent rootfs fill over time. # These grow continuously — without clearing they eventually consume all rootfs space. LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # ━━━ WebGUI Watchdog ━━━ # Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. # Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. # WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking # WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking WEBGUI_URL="http://localhost" WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Media Permissions ━━━ # Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh. # Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership. # 777 mode = read/write/execute for all users — standard for unRAID media shares # nobody:users = standard unRAID media share ownership PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" MEDIA_PERMISSION_SHARES=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows /mnt/user/Anime_Shows-Old /mnt/user/appcache /mnt/user/Books /mnt/user/Downloads /mnt/user/Games /mnt/user/Intros /mnt/user/Kids_Movies /mnt/user/Kids_Tv_Shows /mnt/user/Movie_Recordings /mnt/user/Movies /mnt/user/Music /mnt/user/Music_Videos /mnt/user/Photo /mnt/user/Sports /mnt/user/stand-up_comedy /mnt/user/Temp_Storage /mnt/user/Tv_Recordings /mnt/user/Tv_Shows /mnt/user/YouTube ) # ━━━ Media Cleaner ━━━ # Removes junk files from media shares — two profiles: anime and media. # Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media ANIME_CLEAN_FOLDERS=( /mnt/user/Anime_Movies /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows /mnt/user/Anime_Shows-Old ) MEDIA_CLEAN_FOLDERS=( /mnt/user/Kids_Movies /mnt/user/Kids_Tv_Shows /mnt/user/Movies /mnt/user/Music /mnt/user/Sports /mnt/user/stand-up_comedy /mnt/user/Tv_Shows ) ANIME_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' '*.log' '*.json' ) MEDIA_FILE_PATTERNS=( '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' '*.log' '*.json' '*.iso' '*.lrc' ) # ━━━ Arr Cleanup ━━━ # Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. # Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. # detect_hosts() selects correct URL, API key, and root path at runtime. # # Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr # are not included in the tracked file API response but must not be deleted. # # API versions and endpoint patterns: # Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series # Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie # Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist # All require per-ID loops — bulk endpoints removed in newer versions # # Version checking — scripts verify the arr major version matches before running # If the arr updates and breaks the API the script exits safely before touching files # Update the MAJOR version here when the script is updated to support a new version # MINOR = 0 means any minor version within that major is accepted SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs # ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── HOST1_LIDARR_URL="http://192.168.50.2:8686" HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck # Container path → host path translation # Lidarr stores file paths using container paths — script scans host paths # Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders declare -A HOST1_LIDARR_PATH_MAP=( ["/ext-music"]="/mnt/user/Music-New" ) declare -A HOST2_LIDARR_PATH_MAP=( # HOST2 does not run Lidarr — fill in if that changes # ["/ext-music"]="/mnt/user/Music-New" ) LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion # protects files that may still be mid-import or recently downloaded LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=( # Metadata "*.nfo" "*.tbn" # Images — album art, artist images, Emby artwork "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" "banner.*" "thumb.*" "landscape.*" "folder.*" "cover.*" "album.*" "artist.*" "disc.*" # Lyrics "*.lrc" ) # NEVER deleted — cover art, metadata, lyrics # Lidarr generates these but doesn't include them in trackFile API # Without this protection cleanup would delete all your artwork LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" # persists last known tracked count for percentage comparison # ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── HOST1_SONARR_URL="http://192.168.50.2:8989" HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" # Container path → host path translation # Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders # Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder declare -A HOST1_SONARR_PATH_MAP=( ["/tv"]="/mnt/user/Tv_Shows" ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" ["/kids tv"]="/mnt/user/Kids_Tv_Shows" ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" ) HOST2_SONARR_URL="http://localhost:8989" HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" declare -A HOST2_SONARR_PATH_MAP=( # Fill in when HOST2 is back online # ["/tv"]="/mnt/user/Anime_Shows" ) SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" # Metadata "*.nfo" "*.tbn" # Images — cover art, posters, fanart, Emby artwork "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" "banner.*" "thumb.*" "landscape.*" # Kodi/Emby extras — not tracked by Sonarr API "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" "*-clip.*" "*-other.*" # Theme songs — stored in show folder, not tracked "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) # ── Radarr ──────────────────────────────────────────────────────────────────────────────────── HOST1_RADARR_URL="http://192.168.50.2:7878" HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" # Container path → host path translation # Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders # Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder declare -A HOST1_RADARR_PATH_MAP=( ["/movies"]="/mnt/user/Movies" ["/kids movies"]="/mnt/user/Kids_Movies" ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" ["/anime-movies"]="/mnt/user/Anime_Movies-Old" ) HOST2_RADARR_URL="http://localhost:7878" HOST2_RADARR_API_KEY="your-host2-radarr-api-key" HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" declare -A HOST2_RADARR_PATH_MAP=( # Fill in when HOST2 is back online # ["/anime-movies"]="/mnt/user/Anime_Movies" ) RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" # Metadata "*.nfo" "*.tbn" # Images — cover art, posters, fanart, Emby artwork "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" "banner.*" "thumb.*" "landscape.*" # Kodi/Emby extras — not tracked by Radarr API "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" "*-clip.*" "*-other.*" # Theme songs — stored in movie folder, not tracked "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) # ━━━ Arr Failed/Stalled Recovery ━━━ # Auto blocklist + re-search failed imports and stalled downloads. # Runs every 6 hours — schedule: 0 */6 * * * # # Targets four problem types: # importFailed — downloaded but arr couldn't import # importPending — downloaded, stuck waiting to import (won't self-resolve) # error status — serious failure not covered above # stalled — download stuck with no connections or progress # # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. # API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ # Lidarr runs on HOST1 only — exits cleanly on HOST2. ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this # gives the arr time to retry on its own before we intervene # matches cron interval — items are eligible after one missed cycle # Per-arr enable/disable toggles — set false to temporarily disable without removing from cron # Useful if an arr is having issues and you want to skip it for a few runs HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery HOST1_RADARR_RECOVERY=true # Movies import recovery HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2 HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Session-based storage allocator using filesystem symlink indirection. # ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. # # How it works: # ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh # transcode_management.sh — every 3min, runs cleanup then manager in correct order # transcode_cleanup.sh — removes old inactive files # transcode_manager.sh — manages symlink direction based on usage thresholds # # ⚠️ Docker mount — must use shared propagation: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared # Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. # ━━━ Transcode Manager ━━━ # tmpfs mount point — created at array start by ramdisk_setup.sh # Must exist before Emby starts so the symlink resolves correctly RAMDISK_PATH="/mnt/ramdisk_transcodes" # Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront # Set this to a comfortable limit based on your typical concurrent stream count # Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom RAMDISK_SIZE="8G" # Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state # Emby resolves the symlink once per session at start — symlink flips are transparent # Must match the container path configured in Emby's Extra Parameters TRANSCODE_LINK="/mnt/ram-transcode" # SSD fallback location — where transcodes land when ramdisk is too full # Must have enough free space to handle peak session load TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop # RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this # RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this # Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back # Without hysteresis a session right at the threshold causes rapid flipping RAMDISK_WARN_GB=6.8 RAMDISK_LOW_GB=5.5 # Minimum free GB on SSD before allowing a flip to SSD # Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk RAMDISK_SSD_MIN_GB=20 # File age thresholds in minutes before cleanup eligibility # TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up # TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up TRANSCODE_MAX_AGE=20 TRANSCODE_ORPHAN_AGE=30 # Notify if symlink flips this many times in one hour # Frequent flips indicate the ramdisk is too small or thresholds need adjustment TRANSCODE_FLIP_WARN=3 # Permissions applied to ramdisk and SSD transcode directories TRANSCODE_OWNER="nobody:users" TRANSCODE_CHMOD="755" # Operating mode — controls symlink direction behaviour # smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB # hysteresis gap prevents flip-flop — default for production # ramdisk — always uses ramdisk, never flips to SSD # warns if RAMDISK_WARN_GB exceeded but holds position # use during SSD maintenance or when SSD space is low # ssd — always uses SSD, never flips to ramdisk # use during ramdisk maintenance or after a ramdisk issue TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd # Daily statistics log — read by weekly_health_digest.sh for transcode summary # Tracks peak usage, flip count, session ratio, files cleaned per day # Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" TRANSCODE_LOG_RETENTION=90 # days before old entries are purged # ━━━ Transcode Server Array ━━━ # All media servers sharing the ramdisk transcode space. # Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex # Entries with placeholder API keys are skipped automatically. # ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. TRANSCODE_SERVERS=( "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" # "Plex|http://localhost:32400|plex-token|plex" ) TRANSCODE_CHECK_EMBY=true # ============================================================================================== # ── MONITORS ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Certificate Monitor ━━━ # Checks SSL certificate expiry via direct openssl connection — no NPM dependency. # Checks the actual certificate served by each domain, not what NPM thinks it has. # CERT_WARN_DAYS = notify this many days before expiry # CERT_CRIT_DAYS = escalate to critical this many days before expiry # CERT_TIMEOUT = seconds before giving up on the openssl connection CERT_MONITOR_DOMAINS=( "Gmer4Lfe.com" "Gmer4Lfe.us" ) CERT_WARN_DAYS=30 # warn when cert expires within this many days CERT_CRIT_DAYS=7 # critical alert within this many days CERT_TIMEOUT=10 # seconds per domain check # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. # Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. # BACKUP_VERIFY_SAMPLE = number of random files to checksum per share # BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted) BACKUP_VERIFY_SHARES=( # leave empty to use HOST*_DAILY_SYNC_SHARES automatically ) BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample # ━━━ SMART Health ━━━ # Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*. # Reads live SMART data — no persistent writes. # SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data) SMART_TEMP_WARN=45 # Celsius — warn above this temperature SMART_TEMP_CRIT=55 # Celsius — critical above this temperature SMART_IGNORE_DRIVES=( "sda" # boot USB — SMART not meaningful on flash drives ) # ━━━ ZFS Memory Snapshot ━━━ # Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. # ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max # ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM # ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool # ZFS_REPORT_DOCKER_TOP = how many top Docker containers to show by memory usage # ZFS_REPORT_IGNORE_POOLS = individual disk pools to skip (unRAID array disks as ZFS) ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log" ZFS_REPORT_ARC_WARN_PCT=90 ZFS_REPORT_FREE_WARN_GB=10 ZFS_REPORT_AVAIL_WARN_GB=20 ZFS_REPORT_DOCKER_TOP=10 ZFS_REPORT_IGNORE_POOLS=( "disk10" "disk9" "disk8" "disk6" "disk5" ) # ━━━ Bandwidth Monitor ━━━ # Called automatically by rsync.sh after each sync — one bounded write per run. # Tracks transfer size, duration and profile per sync for weekly summary reporting. # BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records # BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report # Stats files — written by cleanup and recovery scripts, read by coffee report # All in DATA_DIR — array always running when these are written ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats # ━━━ Health Digest ━━━ # Aggregated system health summary — reads existing state files, no new writes. # Three profiles control when the digest email is sent: # always — sends every run regardless of findings # smart — sends only when DIGEST_SMART_ON_* conditions are found # weekly — sends once per week on DIGEST_DAY only # Smart profile triggers — set true to send digest when finding is detected: DIGEST_PROFILE="weekly" # always | smart | weekly DIGEST_DAY="Sunday" # day of week for weekly profile DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB # ━━━ Emby Session Report ━━━ # Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. # Shows top content, most active users, session counts over the report period. # URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY # defined in Host Configuration at the top of this file — no duplication needed. EMBY_REPORT_DAYS=7 # days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Continuous system health monitoring — last line of defense before a crash. # Started by array_start.sh — runs until array stops. # Re-sources Master.conf each cycle — config changes take effect on next cycle. # Strike system: sustained threshold hits trigger reboot — single spikes ignored. # Reboot loop protection: shuts down instead if reboot limit hit in rolling window. # Silent when healthy — logs only when a threshold is triggered. # ━━━ State Files ━━━ SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection # ━━━ Strike and Reboot Loop Settings ━━━ # Strike system: a check must fail this many consecutive cycles before action is taken # Single spikes (one bad reading) are ignored — sustained problems trigger reboot SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger # How often checks run — 300s = 5 minutes # At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles # Reboot loop protection — if system keeps rebooting something is seriously wrong # After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot # Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours # Heartbeat — proof of life logged periodically even when everything is healthy SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) # ━━━ Thresholds ━━━ # Set at "about to become unstable" levels — not "things are a bit high" # These should be high enough that normal operation never triggers them # rootfs (/) usage percentage — when array is down rsync writes land on rootfs # fills rapidly and can crash the server — 95% is almost too late, act fast SYS_WATCHDOG_ROOTFS_PCT=95 # /var/log usage percentage — log spam can fill rootfs, indicates something broken SYS_WATCHDOG_LOG_PCT=95 # Free RAM in GB — below this is critically low, OOM or swap imminent # Your server has 128GB — 4GB free means something is consuming everything SYS_WATCHDOG_MEM_GB=4 # ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck # SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger SYS_WATCHDOG_ARC_PINNED_PCT=98 SYS_WATCHDOG_ARC_RELEASE_PCT=95 # Load average multiplier — threshold = MULTIPLIER × CPU core count # MULTIPLIER=3 on 16-core = load average of 48 before triggering # Set high — transcoding causes legitimate high load spikes SYS_WATCHDOG_LOAD_MULTIPLIER=3 # Zombie process count — large numbers indicate serious process management failure # A few zombies are normal — 50 means something is very wrong SYS_WATCHDOG_ZOMBIE_LIMIT=50 # CPU temperature in Celsius — sustained high temp causes throttling or kernel panic # 95°C is close to tjmax on most CPUs — triggers before thermal shutdown SYS_WATCHDOG_CPU_TEMP_MAX=95 # ━━━ Check Toggles ━━━ # Disable individual checks without disabling the whole watchdog # All enabled by default except load — transcoding causes legitimate load spikes SYS_WATCHDOG_CHECK_ROOTFS=true SYS_WATCHDOG_CHECK_LOG=true SYS_WATCHDOG_CHECK_RAM=true SYS_WATCHDOG_CHECK_ARC=true SYS_WATCHDOG_CHECK_CPU_TEMP=true SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal SYS_WATCHDOG_CHECK_ZOMBIES=true SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding # ━━━ Abort Toggles ━━━ # Conditions that prevent reboot even when a threshold is hit # true = abort reboot if this condition is active (conservative — avoid data loss) # false = reboot anyway (aggressive — a clean reboot beats a hard crash) # Philosophy: aborting is safer for data, rebooting is safer for stability SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── # ==============================================================================================