added more descriptions to master.conf

2026-04-24 22:52:32 -04:00
parent 63101fdb83
commit f826672b80
3 changed files with 844 additions and 431 deletions
@@ -134,12 +134,12 @@
 # Gitea self-hosted repository — used by git_pull_execute.sh.
 # Detects Gitea container location at runtime — works through failover automatically.
 # Falls back to GITEA_DOMAIN if local and Tailscale both fail.
-    GITEA_CONTAINER="Gitea"
-    GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git"
-    GITEA_DOMAIN=""                                     # e.g. git.gmer4lfe.com — requires NPM + DNS
-    TARGET_DIR="/mnt/user/appdata/unraid_scripts"
-    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
-    SSH_PORT=221
+    GITEA_CONTAINER="Gitea"                             # exact Docker container name
+    GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git"   # repo path on Gitea server
+    GITEA_DOMAIN=""                                     # e.g. git.gmer4lfe.com — requires NPM + DNS setup
+    TARGET_DIR="/mnt/user/appdata/unraid_scripts"       # where scripts are cloned to
+    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"             # SSH key for authenticating to Gitea
+    SSH_PORT=221                                        # Gitea SSH port (default 22, Gitea often uses 222/221)

 # ==============================================================================================
 # ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────────
@@ -261,13 +261,19 @@ MEDIA_MANAGEMENT_JOBS=(
 # Global fallback values used when no profile match is found.
 # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
 # Appdata shares match profiles by directory basename (lowercased).
-    BW_LIMIT=12500
-    RETRY_COUNT=3
-    SLEEP=300
-    CRITICAL_CONTAINER_NAMES=()
-    DELAYED_CONTAINERS=()
-    CONTAINER_DELAY=5
-    EXCLUDE_DIRS=()
+# If a profile key exists it overrides the global. If missing the global is used.
+
+    BW_LIMIT=12500          # KB/s — 12500 ≈ 100Mbit — network transfer speed cap
+    RETRY_COUNT=3           # retry attempts if rsync fails before giving up
+    SLEEP=300               # seconds between retry attempts
+    CRITICAL_CONTAINER_NAMES=()  # containers to stop on REMOTE before rsync — profiles override
+    DELAYED_CONTAINERS=()        # containers needing delay before starting — profiles override
+    CONTAINER_DELAY=5            # seconds to wait before starting delayed containers
+    EXCLUDE_DIRS=()              # directories to exclude from transfer — profiles override
+
+# --delete     removes files on remote that no longer exist on source (mirror behaviour)
+# --inplace    writes directly to destination — better for large files, avoids temp copies
+# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks
    DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)

 # ━━━ Remote Health Checks ━━━
@@ -302,15 +308,18 @@ declare -A PROFILE_RSYNC_OPTS=(
    [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
 )

+# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only
+# Lower for shares running alongside other jobs, higher for time-sensitive critical data
 declare -A PROFILE_BW_LIMIT=(
-    [arrs_stack]=5000
-    [critical-data]=9500
+    [arrs_stack]=5000       # lower — runs alongside other syncs, avoids saturating link
+    [critical-data]=9500    # high — small dataset, get it synced fast and clean
    [gmer4lfe]=8000
-    [important-data]=9500
-    [emby]=8000
-    [emby-failover]=9500
+    [important-data]=9500   # high — database sync needs to be fast
+    [emby]=8000             # medium — large full mirror, steady transfer
+    [emby-failover]=9500    # high — small critical dataset, sync as fast as possible
 )

+# Retry attempts per profile — how many times to retry before giving up on a failed sync
 declare -A PROFILE_RETRY_COUNT=(
    [arrs_stack]=3
    [critical-data]=3
@@ -320,48 +329,60 @@ declare -A PROFILE_RETRY_COUNT=(
    [emby-failover]=3
 )

+# Seconds to wait between retry attempts
+# emby-failover shorter — frequent sync, faster retry on transient failures
 declare -A PROFILE_SLEEP=(
    [arrs_stack]=300
    [critical-data]=300
    [gmer4lfe]=300
    [important-data]=300
    [emby]=300
-    [emby-failover]=120
+    [emby-failover]=120     # shorter — frequent dirty sync, retry faster
 )

 # Containers stopped on BOTH LOCAL and REMOTE servers before rsync.
-# Local stops first — flushes databases cleanly. Remote stops next — prevents writes while receiving.
+# Local stops first — flushes databases cleanly before pushing data out.
+# Remote stops next — prevents writes to destination while receiving.
+# Only containers that were running get restarted — stopped containers stay stopped.
 # Same container names on both servers — consistent naming is required by this ecosystem.
-# If a container is not found it is skipped gracefully, not errored.
+# If a container is not found on a server it is skipped gracefully, not errored.
 # SPACE-SEPARATED STRINGS — converted to array at runtime
 declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
    [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
    [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
    [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
    [important-data]="Postgres-NextCloud NextCloud"
-    [emby]="Emby"
-    [emby-failover]=""
+    [emby]="Emby"           # weekly clean sync — both Emby instances stopped, WAL checkpointed
+    [emby-failover]=""      # dirty sync — Emby stays running both sides, WAL excluded from sync
 )

+# Containers that need a delay before starting after rsync completes.
+# Database containers must be accepting connections before dependent apps start.
+# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres.
 # SPACE-SEPARATED STRINGS — converted to array at runtime
 declare -A PROFILE_DELAYED_CONTAINERS=(
    [arrs_stack]=""
-    [critical-data]="Authelia Authelia-Secondary"
+    [critical-data]="Authelia Authelia-Secondary"   # wait for Mariadb + Redis to be ready
    [gmer4lfe]=""
-    [important-data]="NextCloud"
+    [important-data]="NextCloud"                    # wait for Postgres to accept connections
    [emby]=""
    [emby-failover]=""
 )

+# Seconds to wait before starting delayed containers
+# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts
 declare -A PROFILE_CONTAINER_DELAY=(
    [arrs_stack]=5
-    [critical-data]=15
+    [critical-data]=15      # Mariadb + Redis need time to accept connections
    [gmer4lfe]=5
-    [important-data]=10
+    [important-data]=10     # Postgres needs time before NextCloud
    [emby]=5
    [emby-failover]=5
 )

+# Directories excluded from rsync transfer per profile
+# emby-failover excludes WAL files — safe to sync while Emby is running
+# emby clean sync only excludes logs, transcodes, cache — full metadata mirror
 # SPACE-SEPARATED STRINGS — converted to array at runtime
 declare -A PROFILE_EXCLUDE_DIRS=(
    [arrs_stack]="logs *.tmp"
@@ -369,9 +390,13 @@ declare -A PROFILE_EXCLUDE_DIRS=(
    [important-data]="logs *.tmp"
    [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
    [emby]="logs transcodes cache crash*"
+    # emby-failover: Emby running, WAL excluded — only safe critical data synced
+    # users.db, library.db, authentication.db, config/ — everything else excluded
    [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
 )

+# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata
+# not to array disks, so disk space check is irrelevant and just slows things down
 declare -A PROFILE_SKIP_DISK_CHECK=(
    [arrs_stack]=true
    [critical-data]=true
@@ -489,29 +514,45 @@ FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=(
 )

 # ━━━ Tier Delay Settings ━━━
-# Minutes before each tier activates. Tier 1 is always immediate.
-HOST1_TIER2_DELAY=240
-HOST1_TIER3_DELAY=720
-HOST1_TIER4_DELAY=1440
+# How long the primary server must be down before each tier activates — in minutes.
+# Tier 1 is always immediate — Live TV and media can't wait.
+# Set independently per host — adjust based on hardware and what's worth starting.
+# Longer delays = less resource usage on covering server but slower recovery.
+#
+# HOST1's containers running on HOST2 (HOST1 is down):
+HOST1_TIER2_DELAY=240       # 4 hours  — NextCloud, Immich — can wait
+HOST1_TIER3_DELAY=720       # 12 hours — secondary services — Gitea etc.
+HOST1_TIER4_DELAY=1440      # 24 hours — full workflow — arrs and downloaders

+# HOST2's containers running on HOST1 (HOST2 is down):
 HOST2_TIER2_DELAY=240
 HOST2_TIER3_DELAY=720
 HOST2_TIER4_DELAY=1440

 # ━━━ Rsync Writeback Jobs ━━━
-# Syncs critical appdata back to primary on handback — containers stopped before this runs.
-# Short outages skip writeback — primary state is more reliable than dirty sync data.
-# Tier 4 automatically syncs HOST*_DAILY_SYNC_SHARES — add edge cases here only.
+# Syncs critical appdata BACK to primary server during handback after failover.
+# Containers are stopped before writeback runs — clean source, no competing writes.
+# Purpose: primary comes back online with the state that built up during its outage
+#          (watch states, auth changes, library updates that happened on HOST2)
+#
+# HOST*_TIER1_WRITEBACK_DELAY:
+#   Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data
+#   Only writeback if outage lasted longer than this many minutes
+#   60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby
+#
+# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here
+# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage

-HOST1_TIER1_WRITEBACK_DELAY=60
+HOST1_TIER1_WRITEBACK_DELAY=60      # minutes — skip Emby writeback if outage under 1hr
 HOST2_TIER1_WRITEBACK_DELAY=60

+# HOST1 writeback — run by HOST2 during HOST1 handback
 FAILOVER_HOST1_WRITEBACK_TIER1=(
-    "/mnt/user/Media_Server/Emby"
+    "/mnt/user/Media_Server/Emby"               # watch states, playstates built up during outage
 )

 FAILOVER_HOST1_WRITEBACK_TIER2=(
-    "/mnt/user/appdata-Failover/Important-Data"
+    "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage
 )

 FAILOVER_HOST1_WRITEBACK_TIER3=(
@@ -519,9 +560,11 @@ FAILOVER_HOST1_WRITEBACK_TIER3=(
 )

 FAILOVER_HOST1_WRITEBACK_TIER4=(
-    "/mnt/user/appdata-Failover/Arrs_Stack"
+    # Edge cases outside HOST1_DAILY_SYNC_SHARES
+    "/mnt/user/appdata-Failover/Arrs_Stack"     # arr databases — downloads queued during outage
 )

+# HOST2 writeback — run by HOST1 during HOST2 handback
 FAILOVER_HOST2_WRITEBACK_TIER1=(
    # "/mnt/user/appdata-Failover/Jayred365-Emby"
 )
@@ -535,6 +578,7 @@ FAILOVER_HOST2_WRITEBACK_TIER3=(
 )

 FAILOVER_HOST2_WRITEBACK_TIER4=(
+    # Edge cases outside HOST2_DAILY_SYNC_SHARES
    "/mnt/user/appdata-Failover/Arrs_Stack"
 )

@@ -543,18 +587,25 @@ FAILOVER_HOST2_WRITEBACK_TIER4=(
 # ==============================================================================================

 # ━━━ Docker Daily Restart ━━━
-# Restarted by docker_daily_restart.sh via daily_sync_maintenance.sh — 1am daily.
+# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh.
+# These containers run better with a daily restart — not just "keeping things fresh".
+# Dispatcharr specifically degrades over time without restart — daily is intentional.
+# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window.
+# Case-sensitive — must match exact Docker container names.
 DAILY_RESTART_CONTAINERS=(
    "NginxProxyManager"
    "Authelia"
    "Dispatcharr-Iptv-Users"
-    "Dispatcharr"
+    "Dispatcharr"               # Live TV scheduler — degrades without daily restart
    "Dispatcharr-Basic"
    "ErsatzTV-Emby"
 )

 # ━━━ Docker Weekly Restart ━━━
-# Restarted by docker_weekly_restart.sh via weekly_sync_maintenance.sh — Sunday 2:30am.
+# Less critical services restarted weekly by docker_weekly_restart.sh.
+# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped
+# for the weekly sync window so restart adds zero extra downtime.
+# Weekly restarts also catch any pending image updates not applied during weekly sync.
 WEEKLY_RESTART_CONTAINERS=(
    "NextCloud"
    "Organizrv2-Gmer4Lfe"
@@ -567,14 +618,25 @@ WEEKLY_RESTART_CONTAINERS=(
 # Started by array_start.sh — runs until array stops.
 # Re-sources Master.conf each cycle — add/remove containers without restarting watchdog.
 # Silent when all healthy — only logs when something needs attention.
+# Heartbeat fires periodically as proof of life even when everything is healthy.
 #
-# Tier 1 — strict monitoring of explicitly configured containers
-#   Memory hard limits, CPU thresholds, HTTP responsiveness, required container checks
-# Tier 2 — global health scan of ALL running containers
-#   Unhealthy status, OOM kills, crash loops, dead containers, unexpected exits
+# Tier 1 — strict monitoring of explicitly configured containers:
+#   Memory hard limits    — immediate restart if container exceeds limit
+#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT sustained strikes
+#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT failed checks
+#   Required containers   — must always be running, strike + skip list with auto-clear
+#
+# Tier 2 — global health scan of ALL running containers:
+#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
+#   OOM killed            — kernel killed container → restart + notify
+#   Crash loop detection  — RestartCount climbing → notify, critical above limit
+#   Dead containers       — remove and restart
+#   Unexpected exits      — non-zero exit code → restart

 # Memory hard limits in MB — immediate restart if exceeded
-# 20GB=20480  16GB=16384  12GB=12288  8GB=8192  4GB=4096  2GB=2048  1GB=1024
+# Container restarted the moment it crosses this line — no strike system
+# 20GB=20480  16GB=16384  12GB=12288  10GB=10240
+#  8GB=8192    6GB=6144    4GB=4096    2GB=2048   1GB=1024
 declare -A WATCHDOG_CONTAINERS=(
    ["Emby"]=16384
    ["LidaTube"]=6144
@@ -582,6 +644,9 @@ declare -A WATCHDOG_CONTAINERS=(
    ["Code-Server"]=1024
 )

+# HTTP health check URLs — checked every cycle, strike system before restart
+# Container must respond with HTTP 200 within CURL_TIMEOUT seconds
+# Per-host — HOST1 and HOST2 may run different containers on different ports
 declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
    ["Emby"]="http://localhost:8096"
 )
@@ -590,6 +655,11 @@ declare -A HOST2_WATCHDOG_CONTAINER_URLS=(
    ["Emby"]="http://localhost:8096"
 )

+# Required containers — must always be running
+# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt
+# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window
+# Skip list auto-clears when container recovers — no manual intervention needed
+# Per-host — each server has different critical containers
 HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
    "NginxProxyManager"
    "Lldap-Gmer4Lfe"
@@ -605,33 +675,76 @@ HOST2_WATCHDOG_REQUIRED_CONTAINERS=(
    # add HOST2 required containers here
 )

+# Strike state file — /tmp resets on reboot which is correct
+# Fresh start after reboot means no stale strikes carrying over
    WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
-    SOFT_CPU_THRESHOLD=80
-    HARD_CPU_THRESHOLD=85
-    CPU_FAIL_LIMIT=2
-    SOFT_MEM_THRESHOLD=80
-    RESP_FAIL_LIMIT=2
-    CURL_TIMEOUT=5
-    DOCKER_WATCHDOG_INTERVAL=900

+# CPU thresholds — normalised against total core count automatically at runtime
+# SOFT = warn only, HARD = strike toward restart
+# CPU_FAIL_LIMIT = consecutive HARD strikes before restart
+    SOFT_CPU_THRESHOLD=80   # warn at this % of total system CPU
+    HARD_CPU_THRESHOLD=85   # strike at this % of total system CPU
+    CPU_FAIL_LIMIT=2        # consecutive hard CPU strikes before container restart
+
+# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit
+# Does not trigger restart — informational only
+    SOFT_MEM_THRESHOLD=80
+
+# HTTP responsiveness — consecutive failed checks before restart
+# CURL_TIMEOUT = seconds before curl gives up on a single check
+    RESP_FAIL_LIMIT=2       # consecutive failed checks before restart
+    CURL_TIMEOUT=5          # seconds per check before timeout
+
+# How often the watchdog runs its checks
+# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly
+# Containers have this long to recover before next check
+    DOCKER_WATCHDOG_INTERVAL=900    # seconds between watchdog cycles
+
+# Heartbeat — proof of life logged periodically even when everything is healthy
+# Useful to confirm the watchdog is still running without flooding logs
+    DOCKER_WATCHDOG_HEARTBEAT=true       # true = log heartbeat / false = completely silent
+    DOCKER_WATCHDOG_HEARTBEAT_HOURS=1    # how often to log heartbeat (hours)
+
+# Tier 2 master toggle — set false to disable global container scanning entirely
+# When false only WATCHDOG_CONTAINERS and required containers are monitored
    WATCHDOG_SCAN_ALL=true

+# Containers to skip in Tier 2 scan entirely
+# Useful for containers that legitimately exit/restart frequently
 WATCHDOG_SCAN_IGNORE=(
    # "container-name"
 )

-    WATCHDOG_RESTART_UNHEALTHY=true
-    WATCHDOG_RESTART_DEAD=true
-    WATCHDOG_RESTART_CRASHED=true
-    WATCHDOG_NOTIFY_OOM=true
-    WATCHDOG_NOTIFY_CRASHLOOP=true
+# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2
+    WATCHDOG_RESTART_UNHEALTHY=true     # restart containers with Docker HEALTHCHECK = unhealthy
+    WATCHDOG_RESTART_DEAD=true          # restart containers in dead state
+    WATCHDOG_RESTART_CRASHED=true       # restart containers that exited with non-zero code
+    WATCHDOG_NOTIFY_OOM=true            # notify + restart OOM killed containers
+    WATCHDOG_NOTIFY_CRASHLOOP=true      # notify when Docker RestartCount keeps climbing
+
+# Crash loop threshold — notify critical if Docker has restarted this many times total
+# Above this number the notification escalates to critical — manual intervention needed
    WATCHDOG_CRASH_LIMIT=5
-    WATCHDOG_STARTUP_GRACE=600
-    WATCHDOG_CONTAINER_RESTART_LIMIT=3
-    WATCHDOG_CONTAINER_RESTART_WINDOW=1
+
+# Startup grace period — skip restarts while system is still booting after array start
+# Prevents watchdog from restarting containers that are legitimately still initializing
+    WATCHDOG_STARTUP_GRACE=600      # seconds after boot before watchdog acts on failures
+
+# Restart loop protection — stops hammering a broken container
+# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list
+# Skip list auto-clears when container recovers healthy
+    WATCHDOG_CONTAINER_RESTART_LIMIT=3      # max watchdog restarts allowed in window
+    WATCHDOG_CONTAINER_RESTART_WINDOW=1     # rolling window in hours
    WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db"
+
+# Notification batching — one clean summary per cycle instead of one ping per event
+# true  = batch all events into one notification at end of cycle
+# false = send one notification per event (noisy on busy systems)
    WATCHDOG_BATCH_NOTIFY=true

+# Dependency ordering — skip restarting a container if its dependency is also down
+# Prevents restarting Authelia before its database is ready
+# Space-separated list of dependencies per container
 declare -A WATCHDOG_DEPENDENCIES=(
    ["Authelia"]="Mariadb-Authelia Redis-Authelia"
    ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
@@ -639,14 +752,18 @@ declare -A WATCHDOG_DEPENDENCIES=(
 )

 # ━━━ Docker Network Connect ━━━
-# Connects containers to extra networks on array start via array_start.sh.
+# Connects containers to extra Docker networks on array start via array_start.sh.
+# Useful for containers that need their own custom network but also need to be
+# reachable from your main custom bridge network.
+# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network in
+# NETWORK_CONNECT_NETWORKS — containers not found are skipped gracefully.
 NETWORK_CONNECT_CONTAINERS=(
    "memcached"
    "Npm-CrowdSec"
 )

 NETWORK_CONNECT_NETWORKS=(
-    "high-availability"
+    "high-availability"     # must exist before array start — create in Docker settings
 )

 # ==============================================================================================
@@ -654,33 +771,52 @@ NETWORK_CONNECT_NETWORKS=(
 # ==============================================================================================

 # ━━━ Reboot ━━━
+# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
+# Gives users time to save work — 300s = 5 minutes
    REBOOT_SLEEP=300

 # ━━━ Mover ━━━
+# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
+# Gives mover time to finish current file transfer before being interrupted.
    MOVER_STOP_TIMEOUT=300

 # ━━━ Syslog Filter ━━━
+# Path for the rsyslog filter file that suppresses Docker veth interface noise.
+# Docker creates a new veth interface for each container — generates hundreds of
+# log lines per hour that have no diagnostic value. Filter removes them at source.
    FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"

 # ━━━ PHP-FPM ━━━
+# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
+# Default is very low — increasing it prevents WebGUI slowdowns under load.
+# 250 is safe for servers with 32GB+ RAM.
    PHP_CONF="/etc/php-fpm.d/www.conf"
    PHP_MAX_CHILDREN=250

 # ━━━ Clear Logs ━━━
+# System log files cleared weekly to prevent rootfs fill over time.
+# These grow continuously — without clearing they eventually consume all rootfs space.
    LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)

 # ━━━ WebGUI Watchdog ━━━
+# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
+# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
+# WEBGUI_NGINX_WAIT = seconds after nginx restart before rechecking
+# WEBGUI_EMHTTP_WAIT = seconds after emhttp restart before rechecking
    WEBGUI_URL="http://localhost"
-    WEBGUI_TIMEOUT=5
-    WEBGUI_NGINX_WAIT=15
-    WEBGUI_EMHTTP_WAIT=30
+    WEBGUI_TIMEOUT=5        # seconds before curl gives up on WebGUI check
+    WEBGUI_NGINX_WAIT=15    # seconds after nginx restart before rechecking
+    WEBGUI_EMHTTP_WAIT=30   # seconds after emhttp restart before rechecking

 # ==============================================================================================
 # ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
 # ==============================================================================================

 # ━━━ Media Permissions ━━━
-# Applied recursively by media_shares_permissions.sh via MEDIA_MANAGEMENT_JOBS.
+# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh.
+# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership.
+# 777 mode = read/write/execute for all users — standard for unRAID media shares
+# nobody:users = standard unRAID media share ownership
    PERMISSIONS_MODE="777"
    PERMISSIONS_OWNER="nobody:users"

@@ -759,6 +895,7 @@ MEDIA_FILE_PATTERNS=(
 HOST1_LIDARR_URL="http://192.168.50.2:8686"
 HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
 HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"
+LIDARR_LOCK_WARN_AGE=3600   # 1hr — large libraries take time, not stuck

 # Container path → host path translation
 # Lidarr stores file paths using container paths — script scans host paths
@@ -771,12 +908,18 @@ declare -A HOST2_LIDARR_PATH_MAP=(
    # ["/ext-music"]="/mnt/user/Music-New"
 )

-LIDARR_ORPHAN_AGE=7
+LIDARR_ORPHAN_AGE=7         # days — files must be older than this before eligible for deletion
+                            # protects files that may still be mid-import or recently downloaded
 LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
 LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc")
+                            # NEVER deleted — cover art, metadata, lyrics
+                            # Lidarr generates these but doesn't include them in trackFile API
+                            # Without this protection cleanup would delete all your artwork
 LIDARR_MAX_DELETE_GB=1          # require --i-know-what-im-doing if deletion exceeds this
 LIDARR_MIN_TRACKED_PCT=80       # abort if tracked count drops below this % of last run
+                                # protects against API returning partial data on a bad day
 LIDARR_TRACKED_COUNT_FILE="/boot/config/lidarr_tracked.count"
+                                # persists last known tracked count for percentage comparison

 # ── Sonarr ────────────────────────────────────────────────────────────────────────────────────
 HOST1_SONARR_URL="http://192.168.50.2:8989"
@@ -802,9 +945,11 @@ declare -A HOST2_SONARR_PATH_MAP=(
    # ["/tv"]="/mnt/user/Anime_Shows"
 )

-SONARR_ORPHAN_AGE=7
+SONARR_ORPHAN_AGE=7         # days — files must be older than this before eligible for deletion
 SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
 SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa")
+                            # NEVER deleted — cover art, metadata, subtitles
+                            # Sonarr generates these but doesn't include them in episodefile API

 # ── Radarr ────────────────────────────────────────────────────────────────────────────────────
 HOST1_RADARR_URL="http://192.168.50.2:7878"
@@ -830,9 +975,11 @@ declare -A HOST2_RADARR_PATH_MAP=(
    # ["/anime-movies"]="/mnt/user/Anime_Movies"
 )

-RADARR_ORPHAN_AGE=7
+RADARR_ORPHAN_AGE=7         # days — files must be older than this before eligible for deletion
 RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
 RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa")
+                            # NEVER deleted — cover art, metadata, subtitles
+                            # Radarr generates these but doesn't include them in moviefile API

 # ━━━ Arr Failed/Stalled Recovery ━━━
 # Auto blocklist + re-search failed imports and stalled downloads.
@@ -849,12 +996,16 @@ RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.a
 # Lidarr runs on HOST1 only — exits cleanly on HOST2.

 ARR_IMPORT_RECOVERY_AGE=6      # hours — skip items newer than this
+                               # gives the arr time to retry on its own before we intervene
+                               # matches cron interval — items are eligible after one missed cycle

-HOST1_SONARR_RECOVERY=true
-HOST1_RADARR_RECOVERY=true
-HOST1_LIDARR_RECOVERY=true     # HOST1 only
-HOST2_SONARR_RECOVERY=true
-HOST2_RADARR_RECOVERY=true
+# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron
+# Useful if an arr is having issues and you want to skip it for a few runs
+HOST1_SONARR_RECOVERY=true    # Tv_Shows import recovery
+HOST1_RADARR_RECOVERY=true    # Movies import recovery
+HOST1_LIDARR_RECOVERY=true    # Music import recovery — HOST1 only, exits cleanly on HOST2
+HOST2_SONARR_RECOVERY=true    # Anime_Shows import recovery
+HOST2_RADARR_RECOVERY=true    # Anime_Movies import recovery

 # ==============================================================================================
 # ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
@@ -873,16 +1024,47 @@ HOST2_RADARR_RECOVERY=true
 #   Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently.

 # ━━━ Transcode Manager ━━━
+# tmpfs mount point — created at array start by ramdisk_setup.sh
+# Must exist before Emby starts so the symlink resolves correctly
    RAMDISK_PATH="/mnt/ramdisk_transcodes"
+
+# Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront
+# Set this to a comfortable limit based on your typical concurrent stream count
+# Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom
    RAMDISK_SIZE="8G"
+
+# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state
+# Emby resolves the symlink once per session at start — symlink flips are transparent
+# Must match the container path configured in Emby's Extra Parameters
    TRANSCODE_LINK="/mnt/ram-transcode"
+
+# SSD fallback location — where transcodes land when ramdisk is too full
+# Must have enough free space to handle peak session load
    TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
+
+# Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop
+# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this
+# RAMDISK_LOW_GB:  flip symlink back to ramdisk when usage drops to this
+# Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back
+# Without hysteresis a session right at the threshold causes rapid flipping
    RAMDISK_WARN_GB=6.8
    RAMDISK_LOW_GB=5.5
+
+# Minimum free GB on SSD before allowing a flip to SSD
+# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk
    RAMDISK_SSD_MIN_GB=20
+
+# File age thresholds in minutes before cleanup eligibility
+# TRANSCODE_MAX_AGE:    HLS segment files older than this with no active session = clean up
+# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up
    TRANSCODE_MAX_AGE=20
    TRANSCODE_ORPHAN_AGE=30
+
+# Notify if symlink flips this many times in one hour
+# Frequent flips indicate the ramdisk is too small or thresholds need adjustment
    TRANSCODE_FLIP_WARN=3
+
+# Permissions applied to ramdisk and SSD transcode directories
    TRANSCODE_OWNER="nobody:users"
    TRANSCODE_CHMOD="755"

@@ -895,8 +1077,12 @@ HOST2_RADARR_RECOVERY=true
 #   ssd     — always uses SSD, never flips to ramdisk
 #             use during ramdisk maintenance or after a ramdisk issue
    TRANSCODE_MANAGER_MODE="smart"   # smart | ramdisk | ssd
+
+# Daily statistics log — read by weekly_health_digest.sh for transcode summary
+# Tracks peak usage, flip count, session ratio, files cleaned per day
+# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write
    TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
-    TRANSCODE_LOG_RETENTION=90
+    TRANSCODE_LOG_RETENTION=90      # days before old entries are purged

 # ━━━ Transcode Server Array ━━━
 # All media servers sharing the ramdisk transcode space.
@@ -918,30 +1104,48 @@ TRANSCODE_SERVERS=(
 # ==============================================================================================

 # ━━━ Certificate Monitor ━━━
+# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
+# Checks the actual certificate served by each domain, not what NPM thinks it has.
+# CERT_WARN_DAYS  = notify this many days before expiry
+# CERT_CRIT_DAYS  = escalate to critical this many days before expiry
+# CERT_TIMEOUT    = seconds before giving up on the openssl connection
 CERT_MONITOR_DOMAINS=(
    "Gmer4Lfe.com"
    "Gmer4Lfe.us"
 )
-    CERT_WARN_DAYS=30
-    CERT_CRIT_DAYS=7
-    CERT_TIMEOUT=10
+    CERT_WARN_DAYS=30       # warn when cert expires within this many days
+    CERT_CRIT_DAYS=7        # critical alert within this many days
+    CERT_TIMEOUT=10         # seconds per domain check

 # ━━━ Backup Verify ━━━
-# Leave empty to use HOST*_DAILY_SYNC_SHARES automatically.
+# Verifies rsync mirror health by comparing random file checksums between servers.
+# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
+# Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically.
+# BACKUP_VERIFY_SAMPLE   = number of random files to checksum per share
+# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted)
 BACKUP_VERIFY_SHARES=(
-    # leave empty to use daily sync shares automatically
+    # leave empty to use HOST*_DAILY_SYNC_SHARES automatically
 )
-    BACKUP_VERIFY_SAMPLE=10
-    BACKUP_VERIFY_MIN_SIZE=1M
+    BACKUP_VERIFY_SAMPLE=10         # random files to check per share
+    BACKUP_VERIFY_MIN_SIZE=1M       # minimum file size to include in sample

 # ━━━ SMART Health ━━━
-    SMART_TEMP_WARN=45
-    SMART_TEMP_CRIT=55
+# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*.
+# Reads live SMART data — no persistent writes.
+# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data)
+    SMART_TEMP_WARN=45          # Celsius — warn above this temperature
+    SMART_TEMP_CRIT=55          # Celsius — critical above this temperature
 SMART_IGNORE_DRIVES=(
-    "sda"
+    "sda"   # boot USB — SMART not meaningful on flash drives
 )

 # ━━━ ZFS Memory Snapshot ━━━
+# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken.
+# ZFS_REPORT_ARC_WARN_PCT  = warn if ARC is using more than this % of its max
+# ZFS_REPORT_FREE_WARN_GB  = warn if less than this GB free RAM
+# ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool
+# ZFS_REPORT_DOCKER_TOP    = how many top Docker containers to show by memory usage
+# ZFS_REPORT_IGNORE_POOLS  = individual disk pools to skip (unRAID array disks as ZFS)
    ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
    ZFS_REPORT_ARC_WARN_PCT=90
    ZFS_REPORT_FREE_WARN_GB=10
@@ -956,26 +1160,35 @@ ZFS_REPORT_IGNORE_POOLS=(
 )

 # ━━━ Bandwidth Monitor ━━━
-# Called by rsync.sh after each sync — bounded write, minimal flash wear.
+# Called automatically by rsync.sh after each sync — one bounded write per run.
+# Tracks transfer size, duration and profile per sync for weekly summary reporting.
+# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records
+# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size
    BANDWIDTH_LOG="/boot/config/bandwidth_history.db"
-    BANDWIDTH_LOG_RETENTION=90
-    BANDWIDTH_WARN_GB=50
+    BANDWIDTH_LOG_RETENTION=90  # days before old entries are purged
+    BANDWIDTH_WARN_GB=50        # flag syncs larger than this in weekly report

 # ━━━ Health Digest ━━━
-# Reads existing state files — no new flash writes.
-# Profiles: always | smart | weekly
-    DIGEST_PROFILE="weekly"
-    DIGEST_DAY="Sunday"
-    DIGEST_SMART_ON_WATCHDOG=true
-    DIGEST_SMART_ON_FAILOVER=true
-    DIGEST_SMART_ON_CERT_WARN=true
-    DIGEST_SMART_ON_BANDWIDTH=true
+# Aggregated system health summary — reads existing state files, no new writes.
+# Three profiles control when the digest email is sent:
+#   always  — sends every run regardless of findings
+#   smart   — sends only when DIGEST_SMART_ON_* conditions are found
+#   weekly  — sends once per week on DIGEST_DAY only
+# Smart profile triggers — set true to send digest when finding is detected:
+    DIGEST_PROFILE="weekly"         # always | smart | weekly
+    DIGEST_DAY="Sunday"             # day of week for weekly profile
+    DIGEST_SMART_ON_WATCHDOG=true   # send if any watchdog strikes are active
+    DIGEST_SMART_ON_FAILOVER=true   # send if failover state is not NORMAL
+    DIGEST_SMART_ON_CERT_WARN=true  # send if any cert is under CERT_WARN_DAYS
+    DIGEST_SMART_ON_BANDWIDTH=true  # send if any transfer exceeded BANDWIDTH_WARN_GB

 # ━━━ Emby Session Report ━━━
-# Weekly Emby usage statistics via API — no persistent writes.
-# URL and API key from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY in Host Configuration.
-    EMBY_REPORT_DAYS=7
-    EMBY_REPORT_TOP_N=10
+# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
+# Shows top content, most active users, session counts over the report period.
+# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY
+# defined in Host Configuration at the top of this file — no duplication needed.
+    EMBY_REPORT_DAYS=7          # days to include in the report period
+    EMBY_REPORT_TOP_N=10        # number of top content items to show

 # ==============================================================================================
 # ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
@@ -993,37 +1206,78 @@ ZFS_REPORT_IGNORE_POOLS=(
    SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"    # reboot loop detection

 # ━━━ Strike and Reboot Loop Settings ━━━
-    SYS_WATCHDOG_STRIKE_LIMIT=2
-    SYSTEM_WATCHDOG_INTERVAL=300    # seconds between cycles (5min default)
-    SYS_WATCHDOG_REBOOT_LIMIT=3
-    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
+# Strike system: a check must fail this many consecutive cycles before action is taken
+# Single spikes (one bad reading) are ignored — sustained problems trigger reboot
+    SYS_WATCHDOG_STRIKE_LIMIT=2         # consecutive failures before reboot trigger
+
+# How often checks run — 300s = 5 minutes
+# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot
+    SYSTEM_WATCHDOG_INTERVAL=300        # seconds between watchdog cycles
+
+# Reboot loop protection — if system keeps rebooting something is seriously wrong
+# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot
+# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting
+    SYS_WATCHDOG_REBOOT_LIMIT=3         # max reboots before shutdown instead
+    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12   # rolling window in hours
+
+# Heartbeat — proof of life logged periodically even when everything is healthy
+    SYSTEM_WATCHDOG_HEARTBEAT=true      # true = log heartbeat / false = completely silent
+    SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1   # how often to log heartbeat (hours)

 # ━━━ Thresholds ━━━
+# Set at "about to become unstable" levels — not "things are a bit high"
+# These should be high enough that normal operation never triggers them
+
+# rootfs (/) usage percentage — when array is down rsync writes land on rootfs
+# fills rapidly and can crash the server — 95% is almost too late, act fast
    SYS_WATCHDOG_ROOTFS_PCT=95
+
+# /var/log usage percentage — log spam can fill rootfs, indicates something broken
    SYS_WATCHDOG_LOG_PCT=95
+
+# Free RAM in GB — below this is critically low, OOM or swap imminent
+# Your server has 128GB — 4GB free means something is consuming everything
    SYS_WATCHDOG_MEM_GB=4
+
+# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
+# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger
    SYS_WATCHDOG_ARC_PINNED_PCT=98
    SYS_WATCHDOG_ARC_RELEASE_PCT=95
+
+# Load average multiplier — threshold = MULTIPLIER × CPU core count
+# MULTIPLIER=3 on 16-core = load average of 48 before triggering
+# Set high — transcoding causes legitimate high load spikes
    SYS_WATCHDOG_LOAD_MULTIPLIER=3
+
+# Zombie process count — large numbers indicate serious process management failure
+# A few zombies are normal — 50 means something is very wrong
    SYS_WATCHDOG_ZOMBIE_LIMIT=50
+
+# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic
+# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown
    SYS_WATCHDOG_CPU_TEMP_MAX=95

 # ━━━ Check Toggles ━━━
+# Disable individual checks without disabling the whole watchdog
+# All enabled by default except load — transcoding causes legitimate load spikes
    SYS_WATCHDOG_CHECK_ROOTFS=true
    SYS_WATCHDOG_CHECK_LOG=true
    SYS_WATCHDOG_CHECK_RAM=true
    SYS_WATCHDOG_CHECK_ARC=true
    SYS_WATCHDOG_CHECK_CPU_TEMP=true
-    SYS_WATCHDOG_CHECK_LOAD=false       # disabled — load spikes during transcoding are normal
+    SYS_WATCHDOG_CHECK_LOAD=false           # disabled — load spikes during transcoding are normal
    SYS_WATCHDOG_CHECK_ZOMBIES=true
-    SYS_WATCHDOG_CHECK_CONTAINERS=true
-    SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
+    SYS_WATCHDOG_CHECK_CONTAINERS=true      # checks docker_watchdog persistent skip list
+    SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true   # checks if Docker daemon is responding

 # ━━━ Abort Toggles ━━━
-# true = abort reboot if condition active / false = reboot anyway
-    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
-    SYS_WATCHDOG_ABORT_ON_PARITY=false
-    SYS_WATCHDOG_ABORT_ON_MOVER=false
+# Conditions that prevent reboot even when a threshold is hit
+# true  = abort reboot if this condition is active (conservative — avoid data loss)
+# false = reboot anyway (aggressive — a clean reboot beats a hard crash)
+# Philosophy: aborting is safer for data, rebooting is safer for stability
+    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true    # rebooting with bad pool risks data loss
+    SYS_WATCHDOG_ABORT_ON_PARITY=false           # aborting parity is better than crashing mid-check
+    SYS_WATCHDOG_ABORT_ON_MOVER=false            # aborting move is better than crashing mid-move

 # ==============================================================================================
 # ──────────────────────── End Of User Variables ───────────────────────────────────────────────