#!/bin/bash
# ==============================================================================================
# ================================= MASTER CONFIGURATION =======================================
# ==============================================================================================
# Shared configuration for the unRAID script ecosystem.
# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists.
#
# ── HOW THE THREE-FILE SYSTEM WORKS ──────────────────────────────────────────────────────────
# Scripts source all three files at startup:
#   source master.conf          ← shared config (this file)
#   source master_host1.conf    ← HOST1 credentials, shares, container lists
#   source master_host2.conf    ← HOST2 credentials, shares, container lists
#
# Sparse checkout (git) ensures each server only pulls its own master_host*.conf.
# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials.
#
# What belongs here:      thresholds, toggles, intervals, profiles, job lists
# What belongs in HOST*:  hostnames, SSH keys, API keys, passwords, share paths,
#                         container names, failover lists, watchdog containers
#
# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
# Every script sources all three conf files and common.sh at startup.
# Change a value here and it affects all scripts that use it — no hunting through files.
# To disable something: comment it out with # rather than deleting it.
# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section.
# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
#   Section                         Description
#   ───────────────────────────────────────────────────────────────────────────────────────────
#   SHARED HOST CONFIGURATION       DATA_DIR and shared connection settings
#   PARTNERSHIP                     Mirror relationship lifecycle — onboard/offboard/transfer
#   LOGGING                         Enable or disable verbose logging
#   NOTIFICATIONS                   unRAID native and Discord webhook settings
#   GIT / REPO                      Gitea repository and SSH settings
#
#   ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
#   ARRAY START                     Scripts launched at array start (array_start.sh)
#   DAILY SYNC MAINTENANCE          Job list + media shares (daily_sync_maintenance.sh)
#   WEEKLY SYNC MAINTENANCE         Job list + sync shares + update toggles (weekly_sync_maintenance.sh)
#   CRITICAL SYNC MAINTENANCE       15-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh)
#
#   ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
#   RSYNC ENABLE/DISABLE            Two-tier toggle system — global gate + per-orchestrator
#   RSYNC DEFAULTS                  Global fallback rsync options and limits
#   REMOTE HEALTH CHECKS            Rootfs threshold for pre-flight abort
#   RSYNC PROFILE SYSTEM            Per-profile overrides for appdata syncs
#
#   ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
#   FAILOVER                        Mutual container failover shared settings
#   FAILOVER TEST                   Simulated outage settings for failover_test.sh
#
#   ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   DOWNLOADERS RESET               Retention and thresholds for slskd, SABnzbd, qBittorrent
#   DOCKER DAILY RESTART            Containers restarted daily
#   DOCKER WEEKLY RESTART           Containers restarted weekly
#   DOCKER WATCHDOG                 Thresholds and toggles for container monitoring
#   DOCKER NETWORK CONNECT          Networks to ensure + containers to connect
#
#   ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   INOTIFY TUNING                  inotify limits — raised at array start by inotify_tuning.sh
#   SYSTEM TUNING MONITOR           Tracks inotify + php-fpm usage over time
#   REBOOT                          User warning delay before scheduled reboot
#   MOVER                           Mover stop timeout
#   SYSLOG FILTER                   Docker veth noise filter file path
#   PHP-FPM                         PHP-FPM max children config
#   CLEAR LOGS                      System log file paths
#   WEBGUI WATCHDOG                 WebGUI nginx + emhttp monitoring and restart
#
#   ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
#   MEDIA PERMISSIONS               Share list, mode and owner for permissions script
#   MEDIA CLEANER                   Anime and media folder lists and file patterns
#   ARR CLEANUP                     Lidarr, Sonarr, Radarr orphan file cleanup settings
#   ARR FAILED/STALLED RECOVERY     Auto blocklist + re-search settings
#
#   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
#   TRANSCODE MANAGER               Ramdisk and SSD fallback transcode management
#   TRANSCODE SERVER ARRAY          Multi-server session monitoring (Emby, Jellyfin, Plex)
#
#   ── MONITORS ───────────────────────────────────────────────────────────────────────────────
#   CERTIFICATE MONITOR             SSL certificate expiry monitoring
#   BACKUP VERIFY                   Random sample checksum verification against remote
#   SMART HEALTH                    Drive SMART attribute monitoring
#   ZFS MEMORY SNAPSHOT             Weekly ZFS health and memory diagnostic report
#   BANDWIDTH MONITOR               Daily rsync transfer logging and weekly summary
#   HEALTH DIGEST                   Aggregated system health digest — always/smart/weekly
#   EMBY SESSION REPORT             Weekly Emby usage statistics via API
#
#   ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
#   SYSTEM WATCHDOG                 Continuous system health monitoring — last line of defense
#
# ==============================================================================================

# ==============================================================================================
# ── SHARED HOST CONFIGURATION ─────────────────────────────────────────────────────────────────
# ==============================================================================================
# Hostnames defined in master_host*.conf — detect_hosts() reads them at runtime.
# DATA_DIR is the same path on all servers — persistent script state and statistics.
# Array share — survives reboots, no flash drive wear.
# Created automatically if it doesn't exist.
# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config.
    DATA_DIR="/mnt/user/appdata/unraid_scripts/data"

# ── Version Parity ──
# Controls behaviour when local and remote unRAID versions differ.
# Major version mismatch always aborts regardless of this setting.
# "warn"  — log warning and continue (safe for minor/patch differences)
# "abort" — refuse to continue (strict — ensures both sides always match)
    UNRAID_VERSION_MISMATCH_ACTION="warn"

# ── Remote Docker Daemon ──
# Strike system for remote Docker daemon health checks.
# Scripts that issue remote container commands check the remote daemon first.
# Below limit → skip operation this run | At limit → notify critical + exit
    REMOTE_DOCKER_STRIKE_LIMIT=3       # consecutive failures before critical notify
    REMOTE_DOCKER_RETRY_WAIT=30        # seconds to wait before retrying after failure

# ==============================================================================================
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Manages the relationship lifecycle between two unRAID servers.
# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
# All identity vars (hostnames, SSH keys) live in master_host*.conf.
# Hostnames already match Tailscale device names — IP resolution is automatic.
#
# State files on /boot/config — survives reboots, available before array starts:
#   /boot/config/partnership_HOST1.db ← HOST1 writes only
#   /boot/config/partnership_HOST2.db ← HOST2 writes only
#   Propagated via SSH — no rsync needed
#
# critical_sync_maintenance.sh runs --check every 15min:
#   Reads both state files via SSH
#   Detects offboard requests → finalises from owner side
#   Increments offline counter → auto-offboards after threshold
#   Silent when healthy ✅
#
# See README-Partnership.md for full lifecycle documentation.

    PARTNERSHIP_ENABLED=false
    PARTNERSHIP_OWNER_HOST="HOST1"          # "HOST1" or "HOST2" — flips on --transfer

# Auth containers reconfigured on onboard/offboard — defined per host in master_host*.conf.
# Format: "ContainerName|WebUIPort"
# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS
# On onboard → WebUI pointed at owner's Tailscale IP
# On offboard → WebUI pointed back at localhost

# Paths to collect during the grace window after offboard — defined per host in master_host*.conf.
# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS
# Notified on offboard — no auto-deletion, manual collection.

# Timing — single var controls both Tailscale removal and backup access expiry.
# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless.
    PARTNERSHIP_GRACE_HOURS=6               # hours after offboard before Tailscale removal
                                            # backup access expires at the same time
    PARTNERSHIP_OFFLINE_THRESHOLD=30        # days either server unreachable before auto-offboard
                                            # works both directions independently

# Tailscale removal on offboard.
    PARTNERSHIP_REMOVE_TAILSCALE=true       # remove mirror from Tailscale tailnet on offboard
                                            # false = skip removal (manual or testing)

# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
# Stays in shared conf — only owner uses it, and owner is always running this script.
# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope
    TAILSCALE_API_KEY=""                    # tskey-api-...
    TAILSCALE_TAILNET=""                    # your tailnet name (e.g. yourname.github)

# Transfer safety.
    PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
    PARTNERSHIP_TRANSFER_STRIKES=3          # consecutive health checks required
    PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20    # max health check attempts before giving up

# Onboard settings.
    PARTNERSHIP_ONBOARD_VERIFY=true         # verify WebUI reachable after reconfiguration
    PARTNERSHIP_ONBOARD_NOTIFY=true         # notify both servers on completion
    PARTNERSHIP_SYNC_INTERVAL=15            # minutes — informational, actual schedule in cron

# ==============================================================================================
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Silent-by-default output model — ecosystem only speaks when something is wrong.
# true  = only warn() and error() produce output (default — reduces notification spam)
# false = all output visible — use for monitor scripts or debugging
# Override per-run: script --log sets ENABLE_LOGGING=true for [LOG] detail
# Monitor scripts (coffee_report, health_digest etc.) set SILENT_MODE=false themselves
    SILENT_MODE=true

# Controls verbose [LOG] output across all scripts.
# true  = show detailed [LOG] lines — useful for debugging or first-time setup
# false = show only user-facing output — cleaner for scheduled runs
    ENABLE_LOGGING=false

# ==============================================================================================
# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# unRAID native notification system — integrates with the bell icon in the WebGUI.
# normal = job completed successfully / warning = something failed or needs attention
    NOTIFY_UNRAID=true

# Discord webhook URL — defined per host in master_host*.conf.
# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK
# Allows different webhooks per server, or only one server notifying.

# ==============================================================================================
# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Gitea self-hosted repository — used by git_pull_execute.sh.
# Detects Gitea container location at runtime — works through failover automatically.
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
    GITEA_CONTAINER="Gitea"
    GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git"
    GITEA_DOMAIN=""                         # e.g. git.yourdomain.com — requires NPM + DNS
    TARGET_DIR="/mnt/user/appdata/unraid_scripts"
    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
    SSH_PORT=221                            # Gitea SSH port (default 22, Gitea often uses 221/222)

# ==============================================================================================
# ── ORCHESTRATORS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# All orchestrator job lists live here — edit arrays to add/remove scripts.
# No changes to orchestrator scripts needed when adding or removing jobs.

# ━━━ Array Start ━━━
# Scripts launched by array_start.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
# Continuous scripts (watchdogs, failover) run until array stops.
    ARRAY_START_SCRIPTS=(
        "git_pull_execute.sh"                         # pull latest scripts before anything starts
        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
        "unRAID_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
        "unRAID_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
        "unRAID_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
        "unRAID_Essentials/system_watchdog.sh"        # system health monitor — continuous loop
        "Docker_Essentials/docker_watchdog.sh"        # container health monitor — continuous loop
#        "Failover/failover.sh"                        # mutual failover — enable when HOST2 ready
    )

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs media share sync first, then iterates
# DAILY_MAINTENANCE_SCRIPTS for all jobs.
# Schedule: 0 1 * * * (1am daily)
    DAILY_MAINTENANCE_SCRIPTS=(
        "git_pull_execute.sh"                         # pull latest scripts — always runs first
        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
        "Media/media_cleaner.sh anime"                # remove junk from anime shares
        "Media/media_cleaner.sh media"                # remove junk from media shares
        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
        #"Media/radarr_cleanup.sh"                    # remove orphaned movie files — enable when ready
        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
    )

# Media shares synced daily by daily_sync_maintenance.sh.
# Defined per-host in master_host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
# Each server syncs only the shares it owns — direction is automatic.
# HOST1 pushes its shares to HOST2. HOST2 pushes its shares to HOST1.
# Never both pushing the same share — one server is always the truth holder.
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
# For shares needing custom options or container stops — create a profile in RSYNC section.

# Personal encrypted shares defined per-host in master_host*.conf.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.

# ━━━ Weekly Sync Maintenance ━━━
# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
# Schedule: 30 2 * * 0 (Sunday 2:30am)
    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"  # weekly container restarts after sync
    )

# Shares synced during the weekly maintenance window — defined per host in master_host*.conf.
# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
# Order matters — Emby first (larger), then Critical-Data (auth stack).

# Container update toggles for the weekly sync window.
# Containers already stopped for sync — updates pull at no extra downtime.
# Both false → sync only, no updates.
    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH

# ━━━ Critical Sync Maintenance ━━━
# critical_sync_maintenance.sh runs every 15 minutes.
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
# partnership --check always runs last regardless of rsync gate.

# Jobs run every 15 minutes before the rsync shares.
# Comment out to disable without removing.
    CRITICAL_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 15min
    )

# Shares synced every 15 minutes — defined per host in master_host*.conf.
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
# Format: "/path/to/share" or "/path/to/share|profile-name"
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.

# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Rsync Enable/Disable ━━━
# Two-tier toggle system — Tier 1 overrides Tier 2.
#
# Tier 1 — Global gate:
#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
#   Use when: remote completely offline, major maintenance, disaster recovery
#
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
#   Fine grained control — disable specific orchestrators while keeping others
#   Use when: rebuilding secondary, testing, per-window bandwidth management
#
# Example — HOST2 data rebuild:
#   RSYNC_ENABLED=true           ← rsync works, individual scripts run fine
#   DAILY_RSYNC_ENABLED=false    ← skip daily HDD syncs during rebuild
#   WEEKLY_RSYNC_ENABLED=true    ← Emby + Critical-Data still sync (NVMe)
#   CRITICAL_RSYNC_ENABLED=true  ← 15min auth stack sync still runs
#   FAILOVER_RSYNC_ENABLED=true  ← handback writeback still works
#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
#   → When ready: DAILY_RSYNC_ENABLED=true
    RSYNC_ENABLED=true              # Tier 1 — global gate, overrides everything below
    DAILY_RSYNC_ENABLED=false       # Tier 2 — HOST2 rebuild in progress, re-enable when ready
    WEEKLY_RSYNC_ENABLED=true       # Tier 2 — weekly_sync_maintenance.sh rsync section
    CRITICAL_RSYNC_ENABLED=false    # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
    FAILOVER_RSYNC_ENABLED=true     # Tier 2 — failover.sh writeback jobs on handback

# ━━━ Rsync Defaults ━━━
# Global fallback values used when no profile match is found.
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
# Appdata shares match profiles by directory basename (lowercased).
    BW_LIMIT=12500                  # KB/s — 12500 ≈ 100Mbit
    RETRY_COUNT=3                   # retry attempts before giving up
    SLEEP=300                       # seconds between retry attempts
    CRITICAL_CONTAINER_NAMES=()     # containers stopped on REMOTE before rsync — profiles override
    DELAYED_CONTAINERS=()           # containers needing delay before starting — profiles override
    CONTAINER_DELAY=5               # seconds before starting delayed containers
    EXCLUDE_DIRS=()                 # directories excluded from transfer — profiles override

# --delete       removes files on remote not on source (mirror behaviour)
# --inplace      writes directly to destination — better for large files
# --no-whole-file forces delta transfer — sends only changed blocks
    DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)

# ━━━ Remote Health Checks ━━━
# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
# When remote array is down, rsync writes land on rootfs and fill it rapidly.
    ROOTFS_WARN=75

# ━━━ Rsync Profile System ━━━
# Profiles allow per-share rsync behaviour without touching script logic.
# Profile key matched by basename of directory passed to rsync.sh (lowercased).
# Override with --profile=name flag.
#
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
#            List ALL desired options explicitly when defining a profile.
#
# Current profiles:
#   arrs_stack        — arr databases — lower bandwidth, containers stopped for consistency
#   critical-data     — auth stack — full stop both sides, Authelia delayed start
#                       called by weekly_sync_maintenance.sh — full clean sync weekly
#   critical-failover — dirty sync — auth stays running both sides, WAL excluded
#                       called by critical_sync_maintenance.sh every 15min
#   host1-appdata     — HOST1 server-specific appdata — defined in master_host1.conf
#   host2-appdata     — HOST2 server-specific appdata — defined in master_host2.conf
#   important-data    — NextCloud + Postgres — NextCloud delayed start after Postgres
#   emby              — weekly clean sync — both Emby stopped, full mirror
#                       called by weekly_sync_maintenance.sh only — do NOT schedule separately
#   emby-failover     — dirty sync — Emby stays running, WAL excluded
#                       called by critical_sync_maintenance.sh every 15min

    declare -A PROFILE_RSYNC_OPTS=(
        [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
        [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
        [critical-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
        [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
        [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
        [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
    )

# Per-profile bandwidth limits in KB/s
    declare -A PROFILE_BW_LIMIT=(
        [arrs_stack]=5000           # lower — runs alongside other syncs
        [critical-data]=9500        # high — small dataset, sync fast
        [critical-failover]=9500    # high — small dataset, sync fast
        [important-data]=9500       # high — database sync
        [emby]=8000                 # medium — large full mirror
        [emby-failover]=9500        # high — small critical dataset
    )

# Retry attempts per profile
    declare -A PROFILE_RETRY_COUNT=(
        [arrs_stack]=3
        [critical-data]=3
        [critical-failover]=3
        [important-data]=3
        [emby]=3
        [emby-failover]=3
    )

# Seconds between retry attempts
    declare -A PROFILE_SLEEP=(
        [arrs_stack]=300
        [critical-data]=300
        [critical-failover]=120     # shorter — frequent dirty sync, retry faster
        [important-data]=300
        [emby]=300
        [emby-failover]=120         # shorter — frequent dirty sync, retry faster
    )

# Containers stopped on BOTH LOCAL and REMOTE before rsync.
# Local stops first — flushes databases cleanly. Remote stops next — prevents writes.
# Only running containers get restarted — stopped containers stay stopped.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
        [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
        [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
        [critical-failover]=""      # dirty sync — auth stays running both sides
        [important-data]="Postgres-NextCloud NextCloud"
        [emby]="Emby"
        [emby-failover]=""          # dirty sync — Emby stays running both sides
    )

# Containers needing a delay after rsync before starting.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_DELAYED_CONTAINERS=(
        [arrs_stack]=""
        [critical-data]="Authelia Authelia-Secondary"   # wait for Mariadb + Redis
        [critical-failover]=""
        [important-data]="NextCloud"                    # wait for Postgres
        [emby]=""
        [emby-failover]=""
    )

# Seconds before starting delayed containers
    declare -A PROFILE_CONTAINER_DELAY=(
        [arrs_stack]=5
        [critical-data]=15          # Mariadb + Redis need time to accept connections
        [critical-failover]=5
        [important-data]=10         # Postgres needs time before NextCloud
        [emby]=5
        [emby-failover]=5
    )

# Directories excluded from rsync per profile.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_EXCLUDE_DIRS=(
        [arrs_stack]="logs *.tmp"
        [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
        [critical-failover]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
        [important-data]="logs *.tmp"
        [emby]="logs transcodes cache crash*"
        [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
    )

# Remote restart after dirty sync — restart these on remote IF they were running before sync.
# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted.
# Used by dirty sync profiles (critical-failover, emby-failover) so remote picks up changes.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_REMOTE_RESTART_CONTAINERS=(
        [critical-failover]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary"
        [emby-failover]="Emby"
    )

# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed.
# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically.

# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs Failover/failover.sh independently via array_start.sh.
# All decisions based on two pings: remote reachable + internet reachable.
#
# States: NORMAL | FAILOVER | NO_INTERNET | DARK
#
# DDNS rules — absolute:
#   Internet loss → stop own DDNS immediately
#   Failover → start remote DDNS first (Tier 1)
#   Handback → stop remote DDNS → rsync → start containers → start local DDNS last
#
# Per-host container lists and tier delays live in master_host*.conf.
# Shared settings (intervals, state file, thresholds) live here.

    EXTERNAL_IP="8.8.8.8"
    FAILOVER_CHECK_INTERVAL=120     # seconds between failover state checks
    FAILOVER_HANDBACK_STRIKES=2     # consecutive healthy checks before initiating handback
    FAILOVER_STATE_FILE="/boot/config/failover_state.db"
    FAILOVER_ENABLED=false          # HOST2 being rebuilt — set true when back online and tested
                                    # false = suppresses "not running" warnings in status scripts

# ━━━ Failover Test ━━━
# Controlled simulation of a failover event — run manually via failover_test.sh.
    FAILOVER_TEST_BLOCK_WAIT=150    # seconds to wait after blocking connectivity
    FAILOVER_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback

# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Downloaders Reset ━━━
# Runs every 15 minutes via CRITICAL_MAINTENANCE_SCRIPTS.
# Clears stuck states, purges old history, prepares each download client for a clean cycle.
# Per-host URLs and API keys live in master_host*.conf.
    DOWNLOADER_RETENTION_DAYS=7     # days — purge history older than this

# qBittorrent failsafe — removes torrents older than threshold regardless of ratio
# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files)
    QBIT_FAILSAFE_MIN_DAYS=180      # days — minimum age before failsafe deletion
    QBIT_FAILSAFE_MIN_RATIO=0       # 0 = age only, no ratio requirement

# ━━━ Docker Daily Restart ━━━
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
# Per-host lists live in master_host*.conf:
#   HOST1_DAILY_RESTART_CONTAINERS
#   HOST2_DAILY_RESTART_CONTAINERS
# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime.

# ━━━ Docker Weekly Restart ━━━
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
# Containers already stopped for weekly sync — restart adds zero extra downtime.
# Per-host lists live in master_host*.conf:
#   HOST1_WEEKLY_RESTART_CONTAINERS
#   HOST2_WEEKLY_RESTART_CONTAINERS

# ━━━ Docker Watchdog ━━━
# Continuous two-tier self-healing container monitoring.
# Started by array_start.sh — runs until array stops.
# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog.
#
# Tier 1 — strict monitoring of explicitly configured containers:
#   Memory hard limits    — immediate restart if exceeded
#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT sustained strikes
#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT failed checks
#   Required containers   — must always be running, strike + skip list with auto-clear
#
# Tier 2 — global health scan of ALL running containers:
#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
#   OOM killed            — kernel killed → restart + notify
#   Crash loop detection  — RestartCount climbing → notify, critical above limit
#   Dead containers       — remove and restart
#   Unexpected exits      — non-zero exit code → restart
#
# All per-host container lists live in master_host*.conf:
#   HOST*_WATCHDOG_CONTAINERS         — memory hard limits per container
#   HOST*_WATCHDOG_CONTAINER_URLS     — HTTP health check URLs
#   HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
#   HOST*_WATCHDOG_SCAN_IGNORE        — skip in Tier 2 scan
#   HOST*_WATCHDOG_DEPENDENCIES       — dependency ordering for restart decisions

# Strike state file — /tmp resets on reboot (correct — no stale strikes after reboot)
    WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"

# CPU thresholds — normalised against total core count at runtime
    SOFT_CPU_THRESHOLD=80           # warn at this % of total system CPU
    HARD_CPU_THRESHOLD=85           # strike at this % of total system CPU
    CPU_FAIL_LIMIT=2                # consecutive hard CPU strikes before container restart

# Memory soft threshold — warn when container reaches this % of its hard limit
    SOFT_MEM_THRESHOLD=80

# HTTP responsiveness
    RESP_FAIL_LIMIT=2               # consecutive failed checks before restart
    CURL_TIMEOUT=5                  # seconds per check before timeout

# Watchdog cycle interval
    DOCKER_WATCHDOG_INTERVAL=900    # seconds between watchdog cycles (15 minutes)

# Heartbeat — proof of life logged periodically even when everything is healthy
    DOCKER_WATCHDOG_HEARTBEAT=true
    DOCKER_WATCHDOG_HEARTBEAT_HOURS=1

# Tier 2 master toggle
    WATCHDOG_SCAN_ALL=true          # false = only WATCHDOG_CONTAINERS + required containers

# Individual Tier 2 check toggles
    WATCHDOG_RESTART_UNHEALTHY=true
    WATCHDOG_RESTART_DEAD=true
    WATCHDOG_RESTART_CRASHED=true
    WATCHDOG_NOTIFY_OOM=true
    WATCHDOG_NOTIFY_CRASHLOOP=true

# Crash loop threshold
    WATCHDOG_CRASH_LIMIT=5          # RestartCount above this = critical crash loop

# Startup grace period — skip restarts while system is still booting
    WATCHDOG_STARTUP_GRACE=600      # seconds after boot before watchdog acts on failures

# Restart loop protection — prevents watchdog from endlessly restarting a broken container
    WATCHDOG_CONTAINER_RESTART_LIMIT=3
    WATCHDOG_CONTAINER_RESTART_WINDOW=1     # rolling window in hours
    WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"

# Notification batching — one summary per cycle instead of one ping per event
    WATCHDOG_BATCH_NOTIFY=true

# ━━━ Docker Network Connect ━━━
# Ensures custom networks exist and connects containers at array start.
# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run.
# Container and network lists are host-specific — defined in master_host*.conf:
#   HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS
#   HOST1_NETWORK_CONNECT_NETWORKS   / HOST2_NETWORK_CONNECT_NETWORKS

# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ inotify Tuning ━━━
# Linux inotify limits — applied at every array start by inotify_tuning.sh.
# Default unRAID values are very low — with many Docker containers watching files
# you can silently exhaust the limit causing containers to miss file events.
# Symptoms: containers miss file events, downloads not detected, library not updated.
# Live TV stutter is a known symptom of inotify exhaustion.
# These settings are lost on reboot — reapplied automatically at array start.
    INOTIFY_MAX_INSTANCES=1024      # default: 128   — max inotify instances per user
                                    # 1024 handles ~20-30 containers watching files simultaneously
    INOTIFY_MAX_WATCHES=1048576     # default: 8192  — SHARED budget across ALL users/containers
                                    # 524288 (512K) was previous value — raised to 1M (1048576)
                                    # VSCode/Code-Server alone needs ~50K-200K for large workspaces
                                    # with node_modules. All arr containers + Emby + VSCode share
                                    # this budget. 1M is safe on 128GB RAM (~128MB kernel memory)
                                    # If VSCode shows "unable to watch for file changes" → too low
    INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping

# ━━━ System Tuning Monitor ━━━
# Tracks inotify and php-fpm usage over time.
# Snapshot written every 6 hours by system_tuning_monitor.sh.
# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
    INOTIFY_WARN_PCT=80             # warn if inotify instances exceed this % of limit
    PHP_FPM_WARN_PCT=80             # warn if php-fpm workers exceed this % of max_children
    TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
    TUNING_LOG_RETENTION=30         # days before old entries are purged

# ━━━ Reboot ━━━
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
    REBOOT_SLEEP=300

# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
# Gives mover time to finish current file transfer before being interrupted.
    MOVER_STOP_TIMEOUT=300

# ━━━ Syslog Filter ━━━
# Path for the rsyslog filter file that suppresses Docker veth interface noise.
# Docker creates a new veth interface for each container — generates hundreds of
# log lines per hour that have no diagnostic value. Filter removes them at source.
    FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"

# ━━━ PHP-FPM ━━━
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
# Default is very low — increasing it prevents WebGUI slowdowns under load.
# 250 is safe for servers with 32GB+ RAM.
    PHP_CONF="/etc/php-fpm.d/www.conf"
    PHP_MAX_CHILDREN=250

# ━━━ Clear Logs ━━━
# System log files cleared weekly to prevent rootfs fill over time.
# These grow continuously — without clearing they eventually consume all rootfs space.
# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
#
# Size threshold approach — only clear if log exceeds threshold.
# Avoids destroying useful recent diagnostic context when logs are small.
# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing)
# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size
#   Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr)
#   100MB per container × 30 containers = 3GB before clearing kicks in
    LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
    LOG_MIN_SIZE_MB=10          # skip system log if under this size (already small)
    LOG_DOCKER_MAX_MB=100       # clear Docker container log only if over this size (MB)

# ━━━ WebGUI Watchdog ━━━
# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
    WEBGUI_URL="http://localhost"
    WEBGUI_TIMEOUT=5        # seconds before curl gives up on WebGUI check
    WEBGUI_NGINX_WAIT=15    # seconds after nginx restart before rechecking
    WEBGUI_EMHTTP_WAIT=30   # seconds after emhttp restart before rechecking

# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Media Permissions ━━━
# Applied recursively by media_shares_permissions.sh.
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
#
# Why split directory vs file permissions:
#   Directories need execute bit to enter — 755 allows owner+group+others to traverse
#   Files should NOT be executable — 664 allows owner+group read/write, others read
#   This is POSIX best practice for media servers, not a blanket 777 band-aid
#
# Why nobody:users ownership:
#   All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID
#   Files owned by nobody:users are accessible to all containers without world-write
#   If this script fixes many files each run → a container has wrong PUID/PGID set
#   Fix: add PUID=99 PGID=100 to the container's environment variables
#
# This script runs daily as a failsafe — even with correct container config:
#   rsync may bring files with source ownership if not run with --chown
#   Manual admin copies create root:root files
#   New containers may not have PUID/PGID set yet
    PERMISSIONS_DIR_MODE="755"    # directories — traverse + list, no world-write
    PERMISSIONS_FILE_MODE="664"   # files — group read/write, no execute
    PERMISSIONS_OWNER="nobody:users"

# Share list defined per host in master_host*.conf:
#   HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES

# ━━━ Media Cleaner ━━━
# Removes junk files from media shares — two profiles: anime and media.
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
# Folder lists defined per host in master_host*.conf:
#   HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS
#   HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS

    ANIME_FILE_PATTERNS=(
        # ── Checksums ────────────────────────────────────────────────────────────────────────
        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download

        # ── Scene / download metadata ─────────────────────────────────────────────────────────
        '*.url' '*.lnk'                     # scene links
        '*.nfo'                             # scene info files (arrs regenerate their own)
        '*.info' '*.diz'                    # scene description files
        '*.nzb'                             # usenet download files
        '*.torrent'                         # torrent files left by download clients

        # ── Archives and segments ─────────────────────────────────────────────────────────────
        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
        '*.r04' '*.r05' '*.r06' '*.r07'
        '*.r08' '*.r09' '*.srr'             # scene repair files
        '*.001' '*.002' '*.003'             # split archive parts
        '*.gz' '*.tar' '*.bz2'             # linux archives

        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
        '*.sample*' '*.proof*'              # scene samples — never needed in library

        # ── Executables and scripts ───────────────────────────────────────────────────────────
        '*.exe' '*.scr' '*.com'             # Windows executables
        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
        '*.sh'                              # shell scripts in media folders = suspicious

        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
        '*.part'                            # partial download files

        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
        '*sync-conflict*'                   # Syncthing conflict copies
    )

    MEDIA_FILE_PATTERNS=(
        # ── Checksums ────────────────────────────────────────────────────────────────────────
        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download

        # ── Scene / download metadata ─────────────────────────────────────────────────────────
        '*.url' '*.lnk'                     # scene links
        '*.nfo'                             # scene info files (arrs regenerate their own)
        '*.info' '*.diz'                    # scene description files
        '*.nzb'                             # usenet download files
        '*.torrent'                         # torrent files left by download clients

        # ── Archives and segments ─────────────────────────────────────────────────────────────
        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
        '*.r04' '*.r05' '*.r06' '*.r07'
        '*.r08' '*.r09' '*.srr'             # scene repair files
        '*.001' '*.002' '*.003'             # split archive parts
        '*.gz' '*.tar' '*.bz2'             # linux archives

        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
        '*.sample*' '*.proof*'              # scene samples — never needed in library

        # ── Executables and scripts ───────────────────────────────────────────────────────────
        '*.exe' '*.scr' '*.com'             # Windows executables
        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
        '*.sh'                              # shell scripts in media folders = suspicious

        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
        '*.part'                            # partial download files

        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
        '*sync-conflict*'                   # Syncthing conflict copies

        # ── Media-specific extras ─────────────────────────────────────────────────────────────
        '*.iso'                             # disc images — extracted content is in library
        '*.lrc'                             # lyrics files — handled by music apps not Emby
    )

# ━━━ Arr Cleanup ━━━
# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
# Per-host URLs, API keys, and path maps live in master_host*.conf.
# detect_hosts() selects correct host vars at runtime.
#
# API versions — update MAJOR version here when script is updated for a new arr version:
#   Sonarr v4  → /api/v3/series  → /api/v3/episodefile?seriesId=X
#   Radarr v6  → /api/v3/movie   → /api/v3/moviefile?movieId=X
#   Lidarr v3  → /api/v1/artist  → /api/v1/trackFile?artistId=X
    SONARR_VERSION_MAJOR=4
    RADARR_VERSION_MAJOR=6
    LIDARR_VERSION_MAJOR=3

# Lidarr shared settings
    LIDARR_LOCK_WARN_AGE=3600       # 1hr — large libraries take time, not stuck
    LIDARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    LIDARR_MAX_DELETE_GB=5          # require --i-know-what-im-doing if deletion exceeds this
    LIDARR_MIN_TRACKED_PCT=80       # abort if tracked count drops below this % of last run
                                    # protects against API returning partial data on a bad day
    LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
    LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
    LIDARR_PROTECTED_PATTERNS=(
        # Metadata
        "*.nfo" "*.tbn"
        # Images — album art, artist images, Emby artwork
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        "folder.*" "cover.*" "album.*" "artist.*" "disc.*"
        # Lyrics
        "*.lrc"
    )

# Sonarr shared settings
    SONARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    SONARR_MAX_DELETE_GB=10         # require --i-know-what-im-doing if deletion exceeds this
    SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
    SONARR_PROTECTED_PATTERNS=(
        # Subtitles
        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
        # Metadata
        "*.nfo" "*.tbn"
        # Images
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        # Kodi/Emby extras — not tracked by Sonarr API
        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
        "*-clip.*" "*-other.*"
        # Theme songs — stored in show folder, not tracked by arr
        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
    )

# Radarr shared settings
    RADARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    RADARR_MAX_DELETE_GB=15         # require --i-know-what-im-doing if deletion exceeds this
    RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
    RADARR_PROTECTED_PATTERNS=(
        # Subtitles
        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
        # Metadata
        "*.nfo" "*.tbn"
        # Images
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        # Kodi/Emby extras — not tracked by Radarr API
        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
        "*-clip.*" "*-other.*"
        # Theme songs — stored in movie folder, not tracked by arr
        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
    )

# ━━━ Arr Failed/Stalled Recovery ━━━
# Auto blocklist + re-search failed imports and stalled downloads.
# Runs every 6 hours — schedule: 0 */6 * * *
#
# Targets four problem types:
#   importFailed   — downloaded but arr couldn't import
#   importPending  — downloaded, stuck waiting to import (won't self-resolve)
#   error status   — serious failure not covered above
#   stalled        — download stuck with no connections or progress
#
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in master_host*.conf.
    ARR_IMPORT_RECOVERY_AGE=6           # hours — skip items newer than this
                                        # matches cron interval — items eligible after one missed cycle

# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem symlink indirection.
# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected.
# Symlink flips between ramdisk and SSD are transparent to active streams.
#
# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container:
#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
#   Standard path mappings use rprivate — symlink changes are NOT visible inside container.

# ━━━ Transcode Manager ━━━
# tmpfs mount point — created at array start by ramdisk_setup.sh.
# Must exist before Emby starts so the symlink resolves correctly.
    RAMDISK_PATH="/mnt/ramdisk_transcodes"

# Ramdisk size and flip thresholds — defined per host in master_host*.conf.
# All three are coupled — if size changes, thresholds must change with it.
#   HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE
#   HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB  ← flip to SSD at this usage
#   HOST1_RAMDISK_LOW_GB  / HOST2_RAMDISK_LOW_GB   ← flip back to ramdisk at this usage

# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state.
# Emby resolves the symlink once per session at start — symlink flips are transparent.
# Must match the container path configured in Emby's Extra Parameters.
    TRANSCODE_LINK="/mnt/ram-transcode"

# SSD fallback location — defined per host in master_host*.conf (cache path differs per server):
#   HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD

# Minimum free GB on SSD before allowing flip from ramdisk to SSD.
    RAMDISK_SSD_MIN_GB=20

    TRANSCODE_MAX_AGE=20            # minutes — HLS segment age before cleanup eligibility
    TRANSCODE_ORPHAN_AGE=30         # minutes — files with no matching active session
    TRANSCODE_FLIP_WARN=3           # notify if symlink flips this many times in one hour

    TRANSCODE_OWNER="nobody:users"
    TRANSCODE_CHMOD="755"

# Operating mode — controls symlink direction behaviour.
#   smart   — auto-flips between ramdisk and SSD based on thresholds (default)
#   ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds
#   ssd     — always uses SSD, never flips to ramdisk
    TRANSCODE_MANAGER_MODE="smart"

# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
    TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
    TRANSCODE_LOG_RETENTION=90      # days before old entries purged

    TRANSCODE_CHECK_EMBY=true

# ━━━ Transcode Server Array ━━━
# All media servers sharing the ramdisk transcode space.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
# Defined per host in master_host*.conf — Emby container names and keys differ per server:
#   HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS

# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Certificate Monitor ━━━
# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
# Checks the actual certificate served by each domain, not what NPM thinks it has.
# Domains defined per host in master_host*.conf — each server monitors its own domains:
#   HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS
    CERT_WARN_DAYS=30           # warn when cert expires within this many days
    CERT_CRIT_DAYS=7            # critical alert within this many days
    CERT_TIMEOUT=10             # seconds per domain before giving up

# ━━━ Backup Verify ━━━
# Verifies rsync mirror health by comparing random file checksums between servers.
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
# Defined per host in master_host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically:
#   HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
    BACKUP_VERIFY_SAMPLE=10         # random files to check per share
    BACKUP_VERIFY_MIN_SIZE=1M       # minimum file size to include in sample

# ━━━ SMART Health ━━━
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime
# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found.
    SMART_TEMP_WARN=45              # fallback — Celsius warn threshold
    SMART_TEMP_CRIT=55              # fallback — Celsius critical threshold
# Drives to ignore defined per host in master_host*.conf — hardware is server-specific:
#   HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES

# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS pool health and memory diagnostic report — informational only.
    ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
    ZFS_REPORT_ARC_WARN_PCT=90      # warn if ARC using more than this % of its max
    ZFS_REPORT_FREE_WARN_GB=10      # warn if less than this GB free RAM
    ZFS_REPORT_AVAIL_WARN_GB=20     # warn if less than this GB available on ZFS pool
    ZFS_REPORT_DOCKER_TOP=10        # how many top Docker containers to show by memory
# Pool ignore list defined per host in master_host*.conf — pool names are server-specific:
#   HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS

# ━━━ Bandwidth Monitor ━━━
# Called automatically by rsync.sh after each sync — one bounded write per run.
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
    BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
    BANDWIDTH_LOG_RETENTION=90      # days before old entries purged
    BANDWIDTH_WARN_GB=50            # flag syncs larger than this in weekly report

# Stats files — written by cleanup and recovery scripts, read by coffee report.
# All in DATA_DIR — array always running when these are written.
    ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db"     # lidarr/sonarr/radarr orphan stats
    ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db"   # blocklist + re-search stats

# ━━━ Health Digest ━━━
# Aggregated system health summary — reads existing state files, no new writes.
# Three profiles control when the digest is sent:
#   always  — sends every run regardless of findings
#   smart   — sends only when DIGEST_SMART_ON_* conditions are found
#   weekly  — sends once per week on DIGEST_DAY only
    DIGEST_PROFILE="weekly"         # always | smart | weekly
    DIGEST_DAY="Sunday"
    DIGEST_SMART_ON_WATCHDOG=true   # send if any watchdog strikes are active
    DIGEST_SMART_ON_FAILOVER=true   # send if failover state is not NORMAL
    DIGEST_SMART_ON_CERT_WARN=true  # send if any cert is under CERT_WARN_DAYS
    DIGEST_SMART_ON_BANDWIDTH=true  # send if any transfer exceeded BANDWIDTH_WARN_GB

# ━━━ Emby Session Report ━━━
# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in master_host*.conf.
    EMBY_REPORT_DAYS=7              # days to include in the report period
    EMBY_REPORT_TOP_N=10            # number of top content items to show

# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Continuous system health monitoring — last line of defense before a crash.
# Started by array_start.sh — runs until array stops.
# Re-sources all three conf files each cycle — config changes take effect on next cycle.
#
# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
#   CRITICAL  — bypass ALL strikes, reboot immediately
#               Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only
#
#   URGENT    — bypass strikes only when OOM confirms active crisis
#               RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW
#               Without OOM confirmation → normal strike system
#
#   STANDARD  — strike system (N consecutive failures → reboot)
#               RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
#
# ── RAM TIERS ─────────────────────────────────────────────────────────────────────────────────
#   MEM_WARN_GB     — warn + notify only (informational)
#   MEM_SHUTDOWN_GB — stop non-essential containers, recover above MEM_RECOVER_GB
#   MEM_GB          — strike system → reboot (or bypass with OOM)
#
# ── CONTAINER SHUTDOWN ────────────────────────────────────────────────────────────────────────
#   At MEM_SHUTDOWN_GB: stop all containers NOT in MEM_SHUTDOWN_EXCLUDED list
#   Excluded containers stay running — DNS, auth, Emby, Dispatcharr
#   Stopped containers stay stopped until RAM recovers above MEM_RECOVER_GB
#   Strike list applied — doesn't flip-flop every cycle

# ━━━ State Files ━━━
    SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"       # /tmp — resets on reboot ✅
    SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
    SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
    SYS_WATCHDOG_OOM_FILE="/tmp/system_watchdog_oom.db"           # /tmp — resets on reboot ✅

# ━━━ Strike and Reboot Loop Settings ━━━
# Strike system: a check must fail this many consecutive cycles before action is taken.
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
    SYS_WATCHDOG_STRIKE_LIMIT=2         # consecutive failures before reboot trigger

# How often checks run — 300s = 5 minutes.
# At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot.
    SYSTEM_WATCHDOG_INTERVAL=300

# Reboot loop protection — if system keeps rebooting something is seriously wrong.
# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot.
    SYS_WATCHDOG_REBOOT_LIMIT=3
    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12

# Heartbeat — proof of life logged periodically even when everything is healthy.
    SYSTEM_WATCHDOG_HEARTBEAT=true
    SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1

# ━━━ RAM Tiers ━━━
# Three-level RAM response — graduated action instead of single threshold.
# HOST1 has 128GB, HOST2 has 64GB — adjust accordingly.
# MEM_WARN_GB > MEM_SHUTDOWN_GB > MEM_GB always
    SYS_WATCHDOG_MEM_WARN_GB=10         # warn + notify — informational only
    SYS_WATCHDOG_MEM_SHUTDOWN_GB=6      # stop non-essential containers
    SYS_WATCHDOG_MEM_GB=4              # strike system → reboot
    SYS_WATCHDOG_MEM_RECOVER_GB=30     # RAM must recover above this before restarting containers

# Containers excluded from RAM emergency shutdown.
# These stay running regardless of RAM pressure.
# DNS and auth must stay up, Emby and Dispatcharr for Live TV continuity.
    SYS_WATCHDOG_MEM_SHUTDOWN_EXCLUDED=(
        "NginxProxyManager"             # DNS / reverse proxy — internet access
        "Authelia"                      # auth — without this nothing is accessible
        "Mariadb"                       # Authelia dependency
        "Redis"                         # Authelia dependency
        "Emby"                          # media server — Live TV buffering
        "Dispatcharr"                   # Live TV scheduler — loses state if stopped
    )

# ━━━ OOM Bypass Settings ━━━
# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
# in a single cycle → bypass strike system and reboot immediately.
# Rate-based: kills per 5-minute cycle, not absolute count.
# Rationale: 1-2 kills = docker_watchdog handles it ✅
#            3+ kills while RAM critical = system dying faster than watchdogs can heal ✅
    SYS_WATCHDOG_OOM_LIMIT=3           # OOM kills in one cycle to trigger bypass

# ━━━ Thresholds ━━━
# Set at "about to become unstable" — not "things are a bit high".

# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99%
    SYS_WATCHDOG_ROOTFS_PCT=95         # standard strike threshold
    SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing

# /var/log usage — log spam indicates something broken
    SYS_WATCHDOG_LOG_PCT=95

# /tmp usage — tmpfs fills from downloads, lock files fail at 100%
    SYS_WATCHDOG_TMP_PCT=90            # warn + attempt clear
    SYS_WATCHDOG_TMP_CRITICAL_PCT=98   # bypass strikes if clear failed

# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
    SYS_WATCHDOG_ARC_PINNED_PCT=98
    SYS_WATCHDOG_ARC_RELEASE_PCT=95

# Load average multiplier — threshold = MULTIPLIER × CPU core count
# MULTIPLIER=3 on 16-core = load of 48 before triggering
    SYS_WATCHDOG_LOAD_MULTIPLIER=3

# Zombie process count — large numbers = serious process management failure
    SYS_WATCHDOG_ZOMBIE_LIMIT=50

# CPU temperature — sustained high temp causes throttling or kernel panic
    SYS_WATCHDOG_CPU_TEMP_MAX=95

# File descriptor exhaustion — system-wide FD limit near exhaustion
# New connections fail silently, Docker can't spawn processes, SSH fails
    SYS_WATCHDOG_FD_CRITICAL_PCT=95    # bypass strikes — critical tier

# Runaway process — single non-container process consuming excessive CPU
# Multiple strikes before action — single spikes are normal
    SYS_WATCHDOG_RUNAWAY_CPU_PCT=90    # % single process must sustain
    SYS_WATCHDOG_RUNAWAY_STRIKES=3     # consecutive cycles before warning

# Array disk errors — accumulating mdstat errors = disk failing NOW
    SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5  # new errors in one cycle before acting

# ━━━ Check Toggles ━━━
# Per-host — moved to master_host*.conf
# Different servers may have different hardware, NICs, and check requirements
# See HOST*_SYS_WATCHDOG_CHECK_* in master_host*.conf

# ━━━ Abort Toggles ━━━
# Conditions that prevent reboot even when a threshold is hit.
# CRITICAL tier bypasses these — a truly critical condition reboots regardless.
# true  = abort standard reboot if this condition is active
# false = reboot anyway
    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true    # rebooting with bad pool risks data loss
    SYS_WATCHDOG_ABORT_ON_PARITY=false           # aborting parity better than crashing mid-check
    SYS_WATCHDOG_ABORT_ON_MOVER=false            # aborting move better than crashing mid-move

# ==============================================================================================
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
# ==============================================================================================