#!/bin/bash
# ==============================================================================================
# ================================= MASTER CONFIGURATION =======================================
# ==============================================================================================
# Shared configuration for the unRAID script ecosystem.
# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists.
#
# ── HOW THE THREE-FILE SYSTEM WORKS ──────────────────────────────────────────────────────────
# Scripts source all three files at startup:
#   source master.conf          ← shared config (this file)
#   source host1.conf    ← HOST1 credentials, shares, container lists
#   source host2.conf    ← HOST2 credentials, shares, container lists
#
# Sparse checkout (git) ensures each server only pulls its own host*.conf.
# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials.
#
# What belongs here:      thresholds, toggles, intervals, profiles, job lists
# What belongs in HOST*:  hostnames, SSH keys, API keys, passwords, share paths,
#                         container names, failover lists, watchdog containers
#
# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
# Every script sources all three conf files and common.sh at startup.
# Change a value here and it affects all scripts that use it — no hunting through files.
# To disable something: comment it out with # rather than deleting it.
# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section.
# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
#   Section                         Description
#   ───────────────────────────────────────────────────────────────────────────────────────────
#   SHARED HOST CONFIGURATION       DATA_DIR and shared connection settings
#   PARTNERSHIP                     Mirror relationship lifecycle — onboard/offboard/transfer
#   LOGGING                         Enable or disable verbose logging
#   NOTIFICATIONS                   unRAID native and Discord webhook settings
#   GIT / REPO                      Gitea repository and SSH settings
#
#   ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
#   ARRAY START                     Scripts launched at array start (array_started.sh)
#   ARRAY STOP                      Scripts run at planned shutdown (array_stopping.sh)
#   WATCHDOG ORCHESTRATOR           Per-minute watchdog runner (watchdog_orchestrator.sh)
#   SYSTEM WATCHDOG                 Sub-scripts called by watchdog_orchestrator.sh
#   CRITICAL SYNC MAINTENANCE       30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh)
#   INTERMEDIATE SYNC MAINTENANCE   Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh)
#   DAILY SYNC MAINTENANCE          Job list + media shares (daily_sync_maintenance.sh)
#   WEEKLY SYNC MAINTENANCE         Job list + sync shares + update toggles (weekly_sync_maintenance.sh)
#   MONTHLY MAINTENANCE             Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh)
#
#   ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
#   RSYNC ENABLE/DISABLE            Two-tier toggle system — global gate + per-orchestrator
#   RSYNC DEFAULTS                  Global fallback rsync options and limits
#   REMOTE HEALTH CHECKS            Rootfs threshold for pre-flight abort
#   RSYNC PROFILE SYSTEM            Per-profile overrides for appdata syncs
#
#   ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
#   FALLBACK                        Mutual container failover shared settings
#   FALLBACK TEST                   Simulated outage settings for fallback_test.sh
#
#   ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   DOWNLOADERS RESET               Retention and thresholds for slskd, SABnzbd, qBittorrent
#   DOCKER DAILY RESTART            Containers restarted daily
#   DOCKER WEEKLY RESTART           Containers restarted weekly
#   DOCKER WATCHDOG                 Thresholds and toggles for container monitoring
#   DOCKER NETWORK CONNECT          Networks to ensure + containers to connect
#
#   ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   INOTIFY TUNING                  inotify limits — raised at array start by inotify_tuning.sh
#   SYSTEM TUNING MONITOR           Tracks inotify + php-fpm usage over time
#   REBOOT                          User warning delay before scheduled reboot
#   MOVER                           Mover stop timeout
#   SYSLOG FILTER                   Docker veth noise filter file path
#   PHP-FPM                         PHP-FPM max children config
#   CLEAR LOGS                      System log file paths
#   WEBGUI WATCHDOG                 WebGUI nginx + emhttp monitoring and restart
#
#   ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
#   MEDIA PERMISSIONS               Share list, mode and owner for permissions script
#   MEDIA CLEANER                   Anime and media folder lists and file patterns
#   ARR CLEANUP                     Lidarr, Sonarr, Radarr orphan file cleanup settings
#   ARR FAILED/STALLED RECOVERY     Auto blocklist + re-search settings
#
#   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
#   TRANSCODE MANAGER               Ramdisk and SSD fallback transcode management
#   TRANSCODE SERVER ARRAY          Multi-server session monitoring (Emby, Jellyfin, Plex)
#
#   ── MONITORS ───────────────────────────────────────────────────────────────────────────────
#   CERTIFICATE MONITOR             SSL certificate expiry monitoring
#   BACKUP VERIFY                   Random sample checksum verification against remote
#   SMART HEALTH                    Drive SMART attribute monitoring
#   ZFS MEMORY SNAPSHOT             Weekly ZFS health and memory diagnostic report
#   BANDWIDTH MONITOR               Daily rsync transfer logging and weekly summary
#   HEALTH DIGEST                   Aggregated system health digest — always/smart/weekly
#   EMBY SESSION REPORT             Weekly Emby usage statistics via API
#
#   ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
#   RESOURCE MANAGER                Containers paused/stopped under RAM pressure
#
#   ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
#   SYSTEM WATCHDOG                 Continuous system health monitoring — last line of defense
#
# ==============================================================================================

# ==============================================================================================
# ── HOST IDENTITIES ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Hostnames for every server in the ecosystem — not credentials, safe for all machines.
# Must match the exact unRAID hostname AND Tailscale device name (case sensitive).
# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID.
# Tailscale IP resolution uses these names — no hardcoded IPs needed.
# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf.
    HOST1="unRAID-Gmer4Lfe"
    HOST2="unRAID-Jayred365"

# ==============================================================================================
# ── SHARED HOST CONFIGURATION ─────────────────────────────────────────────────────────────────
# ==============================================================================================
# Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash —
# scripts, state, and data are all available before the array mounts.
# Both directories are created automatically if they don't exist.
#
#   DATA_DIR   — historical logs, statistics, discovery histories, blocklists
#   STATE_DIR  — runtime state files for all scripts (watchdogs, fallback, transcode, etc.)
#                Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root.
    DATA_DIR="${SCRIPTS_DIR}/data"
    STATE_DIR="${SCRIPTS_DIR}/State_Files"
    PERSISTENT_CONF_CACHE="${SCRIPTS_DIR}/.cache/vv/d"

# ── Version Parity ──
# Controls behaviour when local and remote unRAID versions differ.
# Major version mismatch always aborts regardless of this setting.
# "warn"  — log warning and continue (safe for minor/patch differences)
# "abort" — refuse to continue (strict — ensures both sides always match)
    UNRAID_VERSION_MISMATCH_ACTION="warn"

# ── Arr Sync ──
# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
# Runs before rsync — all nodes agree on tracked library before files are transferred.
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
    CONF_SYNC_ENABLED=false             # HOST2 rebuild in progress — re-enable when HOST2 is back online

    ARR_SYNC_ENABLED=true
    ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
    ARR_SYNC_CONNECT_TIMEOUT=10     # seconds — SSH connect timeout per node
    ARR_SYNC_API_TIMEOUT=60         # seconds — curl timeout for library fetches
    DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-}"   # set by platform adapter; override in host*.conf if needed
    ARR_SYNC_LIDARR_PORT=8686
    ARR_SYNC_SONARR_PORT=8989
    ARR_SYNC_RADARR_PORT=7878

# ── Remote Docker Daemon ──
# Strike system for remote Docker daemon health checks.
# Scripts that issue remote container commands check the remote daemon first.
# Below limit → skip operation this run | At limit → notify critical + exit
    REMOTE_DOCKER_STRIKE_LIMIT=3       # consecutive failures before critical notify

# ==============================================================================================
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Manages the relationship lifecycle between two unRAID servers.
# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
# All identity vars (hostnames, SSH keys) live in host*.conf.
# Hostnames already match Tailscale device names — IP resolution is automatic.
#
# State files in STATE_DIR — survives reboots, array must be up (scripts run after array start):
#   STATE_DIR/partnership_HOST1.db ← HOST1 writes only
#   STATE_DIR/partnership_HOST2.db ← HOST2 writes only
#   STATE_DIR/partnership_blocklist.db ← shared blocklist
#   STATE_DIR/varaverk_setup.db ← onboarding progress state
#   Propagated via SSH — no rsync needed
#
# critical_sync_maintenance.sh runs --check every 30min:
#   Reads both state files via SSH
#   Detects offboard requests → finalises from owner side
#   Increments offline counter → auto-offboards after threshold
#   Silent when healthy ✅
#
# See README-Partnership.md for full lifecycle documentation.

    PARTNERSHIP_ENABLED=false
    PARTNERSHIP_OWNER_HOST="HOST1"          # "HOST1" or "HOST2" — flips on --transfer

# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf.
# Format: "ContainerName|WebUIPort"
# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS
# On onboard → WebUI pointed at owner's Tailscale IP
# On offboard → WebUI pointed back at localhost

# Paths to collect during the grace window after offboard — defined per host in host*.conf.
# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS
# Notified on offboard — no auto-deletion, manual collection.

# Timing — single var controls both Tailscale removal and backup access expiry.
# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless.
    PARTNERSHIP_GRACE_HOURS=6               # hours after offboard before Tailscale removal
                                            # backup access expires at the same time
    PARTNERSHIP_OFFLINE_THRESHOLD=30        # days either server unreachable before auto-offboard
                                            # works both directions independently

# Partnership and setup state files — all in STATE_DIR per the project requirement.
# Scripts use these variables; do not hardcode /boot/config paths in scripts.
    PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db"
    VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"

# Tailscale removal on offboard.
    PARTNERSHIP_REMOVE_TAILSCALE=true       # remove mirror from Tailscale tailnet on offboard
                                            # false = skip removal (manual or testing)

# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
# Stays in shared conf — only owner uses it, and owner is always running this script.
# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope
    TAILSCALE_API_KEY=""                    # tskey-api-...
    TAILSCALE_TAILNET=""                    # your tailnet name (e.g. yourname.github)

# Transfer safety.
    PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
    PARTNERSHIP_TRANSFER_STRIKES=3          # consecutive health checks required
    PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20    # max health check attempts before giving up

# Onboard settings.
    PARTNERSHIP_ONBOARD_VERIFY=true         # verify WebUI reachable after reconfiguration
    PARTNERSHIP_ONBOARD_NOTIFY=true         # notify both servers on completion
    PARTNERSHIP_SYNC_INTERVAL=15            # minutes — informational, actual schedule in cron

# SSH key lifecycle — managed by Partnership/ssh_setup.sh.
# Key named after this server: hostname lowercased, unraid- prefix stripped.
#   unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation
# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically.
    SSH_MAX_STRIKES=5                       # consecutive SSH auth failures before critical notify
    SSH_STRIKE_RESET_HRS=24                 # hours since last failure before strike counter resets

# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato.
# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard.
# Folder name derived from remote hostname (strip unraid- prefix case-insensitively).
# Plugin config: /boot/config/plugins/folder.view3/docker.json
    PARTNERSHIP_FOLDERVIEW3=true            # create/remove FolderView3 folder on onboard/offboard
    PARTNERSHIP_FOLDERVIEW3_URL=""          # CA plugin URL — leave empty to skip auto-install

# ==============================================================================================
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Controls verbose [LOG] output across all scripts.
# true  = show detailed [LOG] lines — useful for debugging or first-time setup
# false = show only user-facing output — cleaner for scheduled runs
    ENABLE_LOGGING=false

# ==============================================================================================
# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# unRAID native notification system — integrates with the bell icon in the WebGUI.
# normal = job completed successfully / warning = something failed or needs attention
    NOTIFY_UNRAID=true

# Discord webhook URL — defined per host in host*.conf.
# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK
# Allows different webhooks per server, or only one server notifying.

# Upgrade webhook — standalone PHP listener that receives OnUpgrade events from
# Sonarr/Radarr/Lidarr and immediately pushes the upgraded file to all mesh nodes.
# Bypasses Unraid nginx auth — the secret in the URL is the only gate.
# Run Tools/webhook_setup.sh once to register the connection in each arr.
# WEBHOOK_PORT 0 disables the listener.
    WEBHOOK_PORT=7821
    WEBHOOK_SECRET="cdbfde3c13468b7ae1cc1adc466b0e3db28592dec9d99c33689702ce0431e073"                       # auto-generated on first start if empty

# ==============================================================================================
# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Gitea self-hosted repository — used by git_pull_execute.sh.
# Detects Gitea container location at runtime — works through failover automatically.
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
    GITEA_CONTAINER="Gitea"
    GITEA_REPO_PATH="FailedProxy/Varaverk.git"
    GITEA_DOMAIN=""                         # e.g. git.yourdomain.com — requires NPM + DNS
    TARGET_DIR="/boot/config/plugins/varaverk"
    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
    SSH_PORT=221                            # Gitea SSH port (default 22, Gitea often uses 221/222)
    GITEA_HTTP_PORT=3000                    # Gitea web/API port — used by gitea_ssh_setup.sh

# ==============================================================================================
# ── ORCHESTRATORS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# All orchestrator job lists live here — edit arrays to add/remove scripts.
# No changes to orchestrator scripts needed when adding or removing jobs.
# Sections ordered by run frequency: array events first, then shortest interval to longest.

# ━━━ Array Start ━━━
# Scripts launched by array_started.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
# Continuous scripts (failover) run until array stops.
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
# watchdog_orchestrator.sh — NOT launched here.
    ARRAY_START_SCRIPTS=(
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.cache/vv/d/ RAM cache
        "System_Essentials/conf_cache_restore.sh"     # load partner confs from persistent backup if conf_sync couldn't reach partner
        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
        "Media/start_webhook_listener.sh"             # arr upgrade webhook listener — continuous
        "Fallback/fallback.sh"                        # mutual failover — continuous
    )

# ━━━ Array Stop ━━━
# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
# Run sequentially (foreground) — each must complete before the next starts.
# Order matters: user scripts first (prevents new ops), then data movement, then containers.
    ARRAY_STOP_SCRIPTS=(
        "System_Essentials/conf_cache_save.sh"          # snapshot partner conf RAM cache → /boot before anything stops
        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last
    )

# ━━━ Watchdog Orchestrator ━━━
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: */15 * * * * (every 15 minutes)
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
# Order matters — resource first (frees pressure), docker second (heals with freed resources),
# system third (storage + webgui component health), stability last (last line of defense).
    WATCHDOG_ORCHESTRATOR_SCRIPTS=(
        "Watchdogs/resource_watchdog.sh"              # reduce system pressure before healing attempts
        "Watchdogs/docker_watchdog.sh"                # heal containers with freed resources
        "Watchdogs/system_watchdog.sh"                # system component health — storage + webgui
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register API key if registry lost it
        "Watchdogs/stability_watchdog.sh"             # reboot if all else fails — last line of defense
    )

# ━━━ System Watchdog ━━━
# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
# Called by watchdog_orchestrator.sh — not scheduled directly.
    SYSTEM_WATCHDOG_SCRIPTS=(
        "Watchdogs/System/storage_watchdog.sh"        # pool growth + runaway log detection
        "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh"    # WebGUI availability — nginx → php-fpm → emhttp
        "Watchdogs/System/network_watchdog.sh"        # internet, DDNS, Tailscale, NPM proxy
        "Watchdogs/System/conf_cache_watchdog.sh"     # maintain persistent conf backup while partner is offline
    )

# ━━━ Critical Sync Maintenance ━━━
# critical_sync_maintenance.sh runs every 30 minutes.
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
# partnership --check always runs last regardless of rsync gate.
# Comment out entries to disable without removing.
    CRITICAL_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 30min
        "Media/play_state_sync.sh"                      # sync watched/resume state across Emby + Jellyfin
    )

# Shares synced every 30 minutes — defined per host in host*.conf.
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
# Format: "/path/to/share" or "/path/to/share|profile-name"
# Order matters — Critical-Data first (auth stack), then maintenance scripts.

# ━━━ Intermediate Sync Maintenance ━━━
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
# and optional mid-day rsync for any shares that need sub-daily propagation.
# Schedule: 0 */4 * * *
    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs media share sync first, then iterates
# DAILY_MAINTENANCE_SCRIPTS for all jobs.
# Schedule: 0 1 * * * (1am daily)
    DAILY_MAINTENANCE_SCRIPTS=(
        "git_pull_execute.sh"                         # pull latest scripts — always runs first
        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
        "Media/media_cleaner.sh anime"                # remove junk from anime shares
        "Media/media_cleaner.sh media"                # remove junk from media shares
        "Media/lidarr_cleanup.sh"                     # remove orphaned music files
        "Media/sonarr_cleanup.sh"                     # remove orphaned TV files
        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
    )

# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
# Containers keep running during pull — no extra downtime.
# Set false to skip updates while still running the daily restart.
    DAILY_CONTAINER_UPDATES=true

# Media shares synced daily by daily_sync_maintenance.sh.
# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
# Any node can download content to any share — it propagates to all nodes on the next cycle.
# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
# For shares needing custom options or container stops — create a profile in RSYNC section.

# Personal encrypted shares defined per-host in host*.conf.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.

# ━━━ Weekly Sync Maintenance ━━━
# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
# Schedule: 30 2 * * 0 (Sunday 2:30am)
    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_update.sh --weekly"   # pull latest images for WEEKLY_RESTART_CONTAINERS before restart
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
    )

# Pull updates for WEEKLY_RESTART_CONTAINERS before docker_weekly_restart.sh runs.
# Set false to skip — docker_weekly_restart.sh still runs regardless.
    WEEKLY_CONTAINER_UPDATES=true

# Pull updates for all running containers NOT in daily/weekly managed lists.
# Ensures every deployed container receives at least one image pull per month.
# Set false to skip.
    MONTHLY_REMAINING_UPDATES=true

# Shares synced during the weekly maintenance window — defined per host in host*.conf.
# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
# Order matters — Emby first (larger), then Critical-Data (auth stack).

# Container update toggles for the weekly sync window.
# Containers already stopped for sync — updates pull at no extra downtime.
# Both false → sync only, no updates.
    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH

# ━━━ Monthly Maintenance ━━━
# monthly_maintenance.sh fires only when BOTH gates pass:
#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe)
# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry.
# Add scripts that require a long-stable settled system — scrubs, extended drive tests.
# State file on /boot/config — survives reboots (interval gate independent of uptime gate).
    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_update.sh --remainder"  # update all containers not in daily/weekly managed lists
        "Tools/docker_prune_images.sh --all"              # remove stopped containers then all unused images (tagged orphans)
        "Tools/zfs_pool_scrub.sh"                         # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"                        # SMART extended self-test — all drives, sequential
    )

    MONTHLY_UPTIME_THRESHOLD_DAYS=30    # minimum uptime in days before maintenance fires
    MONTHLY_RUN_INTERVAL_DAYS=30        # minimum days since last run before running again
    MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db"

# ━━━ Sunday Morning Coffee Report ━━━
# Orchestrator that runs all Sunday monitor scripts in sequence.
# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am)
# Each script runs independently and notifies on its own findings.
    COFFEE_REPORT_SCRIPTS=(
        "Monitors/zfs_memory_snapshot.sh"       # ZFS pool health + ARC + Docker memory snapshot
        "Monitors/smart_health.sh"              # drive SMART attributes — reallocated, pending, temp
        "Monitors/cert_monitor.sh"              # SSL certificate expiry for all configured domains
        "Monitors/backup_verify.sh"             # rsync mirror integrity via independent MD5 checksums
        "Monitors/bandwidth_monitor.sh"         # weekly rsync transfer totals and per-share breakdown
        "Monitors/emby_session_report.sh"       # Emby usage — streams, users, library, transcode ratio
        "Monitors/weekly_health_digest.sh"      # aggregated digest — watchdogs, fallback, skip list
    )

# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Rsync Enable/Disable ━━━
# Two-tier toggle system — Tier 1 overrides Tier 2.
#
# Tier 1 — Global gate:
#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
#   Use when: remote completely offline, major maintenance, disaster recovery
#
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
#   Fine grained control — disable specific orchestrators while keeping others
#   Use when: rebuilding secondary, testing, per-window bandwidth management
#
# Example — HOST2 data rebuild:
#   RSYNC_ENABLED=true                ← rsync works, individual scripts run fine
#   CRITICAL_RSYNC_ENABLED=true       ← 30min auth stack sync still runs
#   INTERMEDIATE_RSYNC_ENABLED=false  ← skip 4h arr/mid-day rsync during rebuild
#   DAILY_RSYNC_ENABLED=false         ← skip daily HDD syncs during rebuild
#   WEEKLY_RSYNC_ENABLED=true         ← Emby + Critical-Data still sync (NVMe)
#   FALLBACK_RSYNC_ENABLED=true       ← handback writeback still works
#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
#   → When ready: INTERMEDIATE_RSYNC_ENABLED=true  DAILY_RSYNC_ENABLED=true
    RSYNC_ENABLED=true                  # Tier 1 — global gate, overrides everything below
    CRITICAL_RSYNC_ENABLED=true         # Tier 2 — critical_sync_maintenance.sh rsync section
    INTERMEDIATE_RSYNC_ENABLED=true     # Tier 2 — intermediate_sync_maintenance.sh rsync section
    DAILY_RSYNC_ENABLED=true            # Tier 2 — daily_sync_maintenance.sh rsync section
    WEEKLY_RSYNC_ENABLED=true           # Tier 2 — weekly_sync_maintenance.sh rsync section
    MONTHLY_RSYNC_ENABLED=true          # Tier 2 — monthly_maintenance.sh rsync section
    FALLBACK_RSYNC_ENABLED=true         # Tier 2 — fallback.sh writeback jobs on handback

# ━━━ Download Webhook ━━━
# Immediate push to remote nodes on every Sonarr/Radarr/Lidarr Download event.
# Prevents remotes from searching for content that is already on disk locally.
    DOWNLOAD_WEBHOOK_ENABLED=true

# ━━━ Rsync Merge Auto-Promote ━━━
# When enabled, rsync.sh pre-scans the remote before syncing and promotes to merge
# mode (pull-then-push) if ≥75% of remote top-level entries exist locally.
# Named profiles are always excluded from auto-promote regardless of this toggle.
    RSYNC_MERGE_ENABLED=true

# ━━━ Rsync Defaults ━━━
# Global fallback values used when no profile match is found.
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
# Appdata shares match profiles by directory basename (lowercased).
    BW_LIMIT=12500                  # KB/s — 12500 ≈ 100Mbit
    RETRY_COUNT=3                   # retry attempts before giving up
    SLEEP=300                       # seconds between retry attempts
    CRITICAL_CONTAINER_NAMES=()     # containers stopped on REMOTE before rsync — profiles override
    DELAYED_CONTAINERS=()           # containers needing delay before starting — profiles override
    CONTAINER_DELAY=5               # seconds before starting delayed containers
    RESTART_VERIFY_WAIT=3           # seconds to wait after restart before checking container is running
    EXCLUDE_DIRS=()                 # directories excluded from transfer — profiles override

# --inplace      writes directly to destination — delta against existing file, better for large media
# --partial      keep partial file on interrupted transfer so next run resumes, not re-transfers
# --timeout      kill stalled transfers instead of hanging indefinitely
# --numeric-ids  use UIDs/GIDs numerically — prevents ownership mismatches between servers
# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync.
# Media shares use this default; rsync spreads files only, never removes them.
# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default).
# Note: arr_cleanup only catches true orphans under the union model — intentional removals
#       require arr_sync.sh --blocklist-add first, then manual file deletion.
    DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids)

# ━━━ Remote Health Checks ━━━
# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
# When remote array is down, rsync writes land on rootfs and fill it rapidly.
    ROOTFS_WARN=75

# ━━━ Rsync Profile System ━━━
# Profiles allow per-share rsync behaviour without touching script logic.
# Profile key matched by basename of directory passed to rsync.sh (lowercased).
# Override with --profile=name flag.
#
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
#            List ALL desired options explicitly when defining a profile.
#
# Current profiles:
#   arrs_stack        — arr databases — lower bandwidth, containers stopped for consistency
#   critical-data     — auth stack — full stop both sides, Authelia delayed start
#                       called by weekly_sync_maintenance.sh — full clean sync weekly
#   critical-fallback — dirty sync — auth stays running both sides, WAL excluded
#                       called by critical_sync_maintenance.sh every 30min
#   host1-appdata     — HOST1 server-specific appdata — defined in host1.conf
#   host2-appdata     — HOST2 server-specific appdata — defined in host2.conf
#   important-data    — NextCloud + Postgres — NextCloud delayed start after Postgres
#   emby              — weekly clean sync — both Emby stopped, full mirror
#                       called by weekly_sync_maintenance.sh only — do NOT schedule separately
#   emby-fallback     — dirty sync — Emby stays running, WAL excluded
#                       unused — replaced by play_state_sync

    declare -A PROFILE_RSYNC_OPTS=(
        [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
        [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
        [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
        [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
        [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
        [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
    )

# Per-profile bandwidth limits in KB/s
    declare -A PROFILE_BW_LIMIT=(
        [arrs_stack]=5000           # lower — runs alongside other syncs
        [critical-data]=9500        # high — small dataset, sync fast
        [critical-fallback]=9500    # high — small dataset, sync fast
        [important-data]=9500       # high — database sync
        [emby]=8000                 # medium — large full mirror
        [emby-fallback]=9500        # high — small critical dataset
    )

# Retry attempts per profile
    declare -A PROFILE_RETRY_COUNT=(
        [arrs_stack]=3
        [critical-data]=3
        [critical-fallback]=3
        [important-data]=3
        [emby]=3
        [emby-fallback]=3
    )

# Seconds between retry attempts
    declare -A PROFILE_SLEEP=(
        [arrs_stack]=300
        [critical-data]=300
        [critical-fallback]=120     # shorter — frequent dirty sync, retry faster
        [important-data]=300
        [emby]=300
        [emby-fallback]=120         # shorter — retry faster (unused)
    )

# Containers stopped on BOTH LOCAL and REMOTE before rsync.
# Local stops first — flushes databases cleanly. Remote stops next — prevents writes.
# Only running containers get restarted — stopped containers stay stopped.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
        [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
        [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary"
        [critical-fallback]=""      # dirty sync — auth stays running both sides
        [important-data]="Postgres-NextCloud NextCloud"
        [emby]="Emby"
        [emby-fallback]=""          # unused — no containers stopped
    )

# Containers needing a delay after rsync before starting.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_DELAYED_CONTAINERS=(
        [arrs_stack]=""
        [critical-data]="Authelia Authelia-Secondary"   # wait for Mariadb + Redis
        [critical-fallback]=""
        [important-data]="NextCloud"                    # wait for Postgres
        [emby]=""
        [emby-fallback]=""
    )

# Seconds before starting delayed containers
    declare -A PROFILE_CONTAINER_DELAY=(
        [arrs_stack]=5
        [critical-data]=15          # Mariadb + Redis need time to accept connections
        [critical-fallback]=5
        [important-data]=10         # Postgres needs time before NextCloud
        [emby]=5
        [emby-fallback]=5
    )

# Directories excluded from rsync per profile.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_EXCLUDE_DIRS=(
        [arrs_stack]="logs *.tmp"
        [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
        [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
        [important-data]="logs *.tmp"
        [emby]="logs transcodes cache crash*"
        [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
    )

# Remote restart after live sync — restart these on remote IF they were running before sync.
# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted.
# Used by live sync profiles (critical-fallback) so remote picks up changes.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_REMOTE_RESTART_CONTAINERS=(
        [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary"
        [emby-fallback]="Emby"
    )

# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed.
# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically.

# ==============================================================================================
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs Fallback/fallback.sh independently via array_started.sh.
# All decisions based on two pings: remote reachable + internet reachable.
#
# States: NORMAL | FALLBACK | NO_INTERNET | DARK
#
# DDNS rules — absolute:
#   Internet loss → stop own DDNS immediately
#   Failover → start remote DDNS first (Tier 1)
#   Handback → stop remote DDNS → rsync → start containers → start local DDNS last
#
# Per-host container lists and tier delays live in host*.conf.
# Shared settings (intervals, state file, thresholds) live here.

    EXTERNAL_IP="8.8.8.8"
    FALLBACK_CHECK_INTERVAL=30      # seconds between fallback state checks
    FALLBACK_HANDBACK_STRIKES=3     # consecutive healthy checks before initiating handback (3×30s = 90s)
    FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
    FALLBACK_ENABLED=true           # HOST2 back online
                                    # false = suppresses "not running" warnings in status scripts
    FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120      # minutes without active partnership before suspending
                                                # 0 = suspend immediately when partnership goes inactive

# ━━━ Play State Sync ━━━
    PLAY_SYNC_ENABLED=true
    PLAY_SYNC_REMOTE=true               # sync across hosts via Tailscale
    PLAY_SYNC_DAYS=90                   # days back to check for played items (0 = all time)
    PLAY_SYNC_TYPES="Movie,Episode"     # Audio excluded — music library too large; favorites handled separately
    PLAY_SYNC_FAV_TYPES="MusicArtist,MusicAlbum,Movie,Series"  # union sync — never unmarks; Audio tracks future

# ━━━ Play State Sync — Handback ━━━
# play_state_sync runs before DNS cutover so users land on current watch state.
# Retried until success — DNS is held until sync completes or retries are exhausted.
    PLAY_SYNC_HANDBACK_RETRIES=5        # attempts before giving up and proceeding to DNS cutover
    PLAY_SYNC_HANDBACK_RETRY_DELAY=60   # seconds between retry attempts

# ━━━ Failover Test ━━━
# Controlled simulation of a failover event — run manually via fallback_test.sh.
    FALLBACK_TEST_BLOCK_WAIT=150    # seconds to wait after blocking connectivity
    FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback

# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Downloaders Reset ━━━
# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS.
# Clears stuck states, purges old history, prepares each download client for a clean cycle.
# Per-host URLs and API keys live in host*.conf.
    DOWNLOADER_RETENTION_DAYS=7     # days — purge history older than this

# qBittorrent failsafe — removes torrents older than threshold regardless of ratio
# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files)
    QBIT_FAILSAFE_MIN_DAYS=180      # days — minimum age before failsafe deletion
    QBIT_FAILSAFE_MIN_RATIO=0       # 0 = age only, no ratio requirement

# ━━━ Docker Daily Restart ━━━
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
# Per-host lists live in host*.conf:
#   HOST1_DAILY_RESTART_CONTAINERS
#   HOST2_DAILY_RESTART_CONTAINERS
# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime.

# ━━━ Docker Weekly Restart ━━━
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
# Containers already stopped for weekly sync — restart adds zero extra downtime.
# Per-host lists live in host*.conf:
#   HOST1_WEEKLY_RESTART_CONTAINERS
#   HOST2_WEEKLY_RESTART_CONTAINERS

# ━━━ Docker Watchdog ━━━
# Continuous two-tier self-healing container monitoring.
# Started by array_started.sh — runs until array stops.
# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog.
#
# Tier 1 — strict monitoring of explicitly configured containers:
#   Memory hard limits    — immediate restart if exceeded
#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT sustained strikes
#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT failed checks
#   Required containers   — must always be running, strike + skip list with auto-clear
#
# Tier 2 — global health scan of ALL running containers:
#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
#   OOM killed            — kernel killed → restart + notify
#   Crash loop detection  — RestartCount climbing → notify, critical above limit
#   Dead containers       — remove and restart
#   Unexpected exits      — non-zero exit code → restart
#
# All per-host container lists live in host*.conf:
#   HOST*_WATCHDOG_CONTAINERS         — memory hard limits per container
#   HOST*_WATCHDOG_CONTAINER_URLS     — HTTP health check URLs
#   HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
#   HOST*_WATCHDOG_SCAN_IGNORE        — skip in Tier 2 scan
#   HOST*_WATCHDOG_DEPENDENCIES       — dependency ordering for restart decisions

# Strike state file — persistent in STATE_DIR
    WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db"

# CPU thresholds — normalised against total core count at runtime
    SOFT_CPU_THRESHOLD=80           # warn at this % of total system CPU
    HARD_CPU_THRESHOLD=85           # strike at this % of total system CPU
    CPU_FAIL_LIMIT=2                # consecutive hard CPU strikes before container restart

# Memory soft threshold — warn when container reaches this % of its hard limit
    SOFT_MEM_THRESHOLD=80

# HTTP responsiveness
    RESP_FAIL_LIMIT=2               # consecutive failed checks before restart
    CURL_TIMEOUT=5                  # seconds per check before timeout

# Tier 2 master toggle
    WATCHDOG_SCAN_ALL=true          # false = only WATCHDOG_CONTAINERS + required containers

# Individual Tier 2 check toggles
    WATCHDOG_RESTART_UNHEALTHY=true
    WATCHDOG_RESTART_DEAD=true
    WATCHDOG_RESTART_CRASHED=true
    WATCHDOG_NOTIFY_OOM=true
    WATCHDOG_NOTIFY_CRASHLOOP=true

# Crash loop threshold
    WATCHDOG_CRASH_LIMIT=5          # RestartCount above this = critical crash loop

# Startup grace period — skip restarts while system is still booting
    WATCHDOG_STARTUP_GRACE=600      # seconds after boot before watchdog acts on failures

# Restart loop protection — prevents watchdog from endlessly restarting a broken container
    WATCHDOG_CONTAINER_RESTART_LIMIT=3
    WATCHDOG_CONTAINER_RESTART_WINDOW=1     # rolling window in hours
    WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"

# Notification batching — one summary per cycle instead of one ping per event
    WATCHDOG_BATCH_NOTIFY=true

# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files.
#
# Part 1 — Growth rate (zero-config):
#   Reads per-container dir totals each cycle via du, compares to previous cycle.
#   Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan
#   inside that container. No per-container config required — new containers are covered
#   automatically. Baseline built on first run after boot; growth detection starts cycle 2.
#
# Part 2 — Absolute log size:
#   Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths.
#   Catches logs that have already stabilised at a large size and are no longer actively growing.
#
# Strike system (reuses existing watchdog infrastructure):
#   Strike 1 — warn + notify: condition first detected
#   Strike 2 — warn + escalated notify: still present next cycle
#   Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
#     If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes
#     If false: critical notify only, strikes held until condition resolves
#   Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
#
# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a
#   container until its dir exceeds the configured ceiling. Only needed when a container
#   legitimately has large stable data and you want to guarantee it never triggers a false alarm.
    WATCHDOG_CHECK_APPDATA=true
    WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata")
    WATCHDOG_APPDATA_GROWTH_GB=2            # flag containers growing more than this per cycle
    WATCHDOG_APPDATA_LOG_MAX_GB=2           # flag *.log files exceeding this size (absolute)
    WATCHDOG_APPDATA_TRUNCATE_LOGS=false    # set true to auto-truncate oversized *.log files on action cycle
    WATCHDOG_APPDATA_STRIKE_LIMIT=3         # cycles before action fires (matches existing watchdog pattern)
    WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db"
    STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db"

# ━━━ Docker Network Connect ━━━
# Ensures custom networks exist and connects containers at array start.
# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run.
# Container and network lists are host-specific — defined in host*.conf:
#   HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS
#   HOST1_NETWORK_CONNECT_NETWORKS   / HOST2_NETWORK_CONNECT_NETWORKS

# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ inotify Tuning ━━━
# Linux inotify limits — applied at every array start by inotify_tuning.sh.
# Default unRAID values are very low — with many Docker containers watching files
# you can silently exhaust the limit causing containers to miss file events.
# Symptoms: containers miss file events, downloads not detected, library not updated.
# Live TV stutter is a known symptom of inotify exhaustion.
# These settings are lost on reboot — reapplied automatically at array start.
    INOTIFY_MAX_INSTANCES=1024      # default: 128   — max inotify instances per user
                                    # 1024 handles ~20-30 containers watching files simultaneously
    INOTIFY_MAX_WATCHES=1048576     # default: 8192  — SHARED budget across ALL users/containers
                                    # 524288 (512K) was previous value — raised to 1M (1048576)
                                    # VSCode/Code-Server alone needs ~50K-200K for large workspaces
                                    # with node_modules. All arr containers + Emby + VSCode share
                                    # this budget. 1M is safe on 128GB RAM (~128MB kernel memory)
                                    # If VSCode shows "unable to watch for file changes" → too low
    INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping

# ━━━ System Tuning Monitor ━━━
# Tracks inotify and php-fpm usage over time.
# Snapshot written every 6 hours by system_tuning_monitor.sh.
# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
    INOTIFY_WARN_PCT=80             # warn if inotify instances exceed this % of limit
    PHP_FPM_WARN_PCT=80             # warn if php-fpm workers exceed this % of max_children
    TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
    TUNING_LOG_RETENTION=30         # days before old entries are purged

# ━━━ Reboot ━━━
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
    REBOOT_SLEEP=300

# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
# Gives mover time to finish current file transfer before being interrupted.
    MOVER_STOP_TIMEOUT=300

# ━━━ Syslog Filter ━━━
# Path for the rsyslog filter file that suppresses Docker veth interface noise.
# Docker creates a new veth interface for each container — generates hundreds of
# log lines per hour that have no diagnostic value. Filter removes them at source.
    FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"

# ━━━ PHP-FPM ━━━
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
# Default is very low — increasing it prevents WebGUI slowdowns under load.
# 250 is safe for servers with 32GB+ RAM.
    PHP_CONF="/etc/php-fpm.d/www.conf"
    PHP_MAX_CHILDREN=250

# ━━━ Clear Logs ━━━
# System log files cleared weekly to prevent rootfs fill over time.
# These grow continuously — without clearing they eventually consume all rootfs space.
# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
#
# Size threshold approach — only clear if log exceeds threshold.
# Avoids destroying useful recent diagnostic context when logs are small.
# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing)
# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size
#   Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr)
#   100MB per container × 30 containers = 3GB before clearing kicks in
    LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
    LOG_MIN_SIZE_MB=10          # skip system log if under this size (already small)
    LOG_DOCKER_MAX_MB=100       # clear Docker container log only if over this size (MB)

# ━━━ Network Watchdog ━━━
# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy.
# Host-specific values (domain, container, NPM URL) live in host*.conf.
    NETWORK_WATCHDOG_ENABLED=true
    NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1"
    NETWORK_WATCHDOG_INTERNET_TIMEOUT=5
    NETWORK_WATCHDOG_CHECK_TAILSCALE=true
    NETWORK_WATCHDOG_NPM_TIMEOUT=10
    NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2
    NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db"

# ━━━ WebGUI Watchdog ━━━
# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
    WEBGUI_URL="http://localhost"
    WEBGUI_TIMEOUT=5        # seconds before curl gives up on WebGUI check
    WEBGUI_NGINX_WAIT=15    # seconds after nginx restart before rechecking
    WEBGUI_EMHTTP_WAIT=30   # seconds after emhttp restart before rechecking

# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Media Permissions ━━━
# Applied recursively by media_shares_permissions.sh.
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
#
# Why split directory vs file permissions:
#   Directories need execute bit to enter — 755 allows owner+group+others to traverse
#   Files should NOT be executable — 664 allows owner+group read/write, others read
#   This is POSIX best practice for media servers, not a blanket 777 band-aid
#
# Why nobody:users ownership:
#   All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID
#   Files owned by nobody:users are accessible to all containers without world-write
#   If this script fixes many files each run → a container has wrong PUID/PGID set
#   Fix: add PUID=99 PGID=100 to the container's environment variables
#
# This script runs daily as a failsafe — even with correct container config:
#   rsync may bring files with source ownership if not run with --chown
#   Manual admin copies create root:root files
#   New containers may not have PUID/PGID set yet
    PERMISSIONS_DIR_MODE="755"    # directories — traverse + list, no world-write
    PERMISSIONS_FILE_MODE="664"   # files — group read/write, no execute
    PERMISSIONS_OWNER="nobody:users"

# Share list defined per host in host*.conf:
#   HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES

# ━━━ Media Cleaner ━━━
# Removes junk files from media shares — two profiles: anime and media.
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
# Folder lists defined per host in host*.conf:
#   HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS
#   HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS

    ANIME_FILE_PATTERNS=(
        # ── Checksums ────────────────────────────────────────────────────────────────────────
        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download

        # ── Scene / download metadata ─────────────────────────────────────────────────────────
        '*.url' '*.lnk'                     # scene links
        '*.info' '*.diz'                    # scene description files
        '*.nzb'                             # usenet download files
        '*.torrent'                         # torrent files left by download clients

        # ── Archives and segments ─────────────────────────────────────────────────────────────
        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
        '*.r04' '*.r05' '*.r06' '*.r07'
        '*.r08' '*.r09' '*.srr'             # scene repair files
        '*.001' '*.002' '*.003'             # split archive parts
        '*.gz' '*.tar' '*.bz2'             # linux archives

        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
        '*.sample*' '*.proof*'              # scene samples — never needed in library

        # ── Executables and scripts ───────────────────────────────────────────────────────────
        '*.exe' '*.scr' '*.com'             # Windows executables
        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
        '*.sh'                              # shell scripts in media folders = suspicious

        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
        '*.part'                            # partial download files

        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
        '*sync-conflict*'                   # Syncthing conflict copies
    )

    MEDIA_FILE_PATTERNS=(
        # ── Checksums ────────────────────────────────────────────────────────────────────────
        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download

        # ── Scene / download metadata ─────────────────────────────────────────────────────────
        '*.url' '*.lnk'                     # scene links
        '*.info' '*.diz'                    # scene description files
        '*.nzb'                             # usenet download files
        '*.torrent'                         # torrent files left by download clients

        # ── Archives and segments ─────────────────────────────────────────────────────────────
        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
        '*.r04' '*.r05' '*.r06' '*.r07'
        '*.r08' '*.r09' '*.srr'             # scene repair files
        '*.001' '*.002' '*.003'             # split archive parts
        '*.gz' '*.tar' '*.bz2'             # linux archives

        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
        '*.sample*' '*.proof*'              # scene samples — never needed in library

        # ── Executables and scripts ───────────────────────────────────────────────────────────
        '*.exe' '*.scr' '*.com'             # Windows executables
        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
        '*.sh'                              # shell scripts in media folders = suspicious

        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
        '*.part'                            # partial download files

        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
        '*sync-conflict*'                   # Syncthing conflict copies

        # ── Media-specific extras ─────────────────────────────────────────────────────────────
        '*.iso'                             # disc images — extracted content is in library
        '*.lrc'                             # lyrics files — handled by music apps not Emby
    )

# ━━━ Arr Cleanup ━━━
# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
# Per-host URLs, API keys, and path maps live in host*.conf.
# detect_hosts() selects correct host vars at runtime.
#
# API versions — update MAJOR version here when script is updated for a new arr version:
#   Sonarr v4  → /api/v3/series  → /api/v3/episodefile?seriesId=X
#   Radarr v6  → /api/v3/movie   → /api/v3/moviefile?movieId=X
#   Lidarr v3  → /api/v1/artist  → /api/v1/trackFile?artistId=X
    SONARR_VERSION_MAJOR=4
    RADARR_VERSION_MAJOR=6
    LIDARR_VERSION_MAJOR=3

# arr_profile_enforcer.sh — quality profile names by root folder type
# Root folder paths containing "kids" or "anime" → ARR_KIDS_PROFILE_NAME
# All other root folders                          → ARR_*_DEFAULT_PROFILE
    ARR_KIDS_PROFILE_NAME="Kids shows"
    ARR_SONARR_DEFAULT_PROFILE="Any"
    ARR_RADARR_DEFAULT_PROFILE="Any (mine)"

# Lidarr shared settings
    LIDARR_LOCK_WARN_AGE=3600       # 1hr — large libraries take time, not stuck
    LIDARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    LIDARR_MAX_DELETE_GB=5          # require --i-know-what-im-doing if deletion exceeds this
    LIDARR_MIN_TRACKED_PCT=80       # abort if tracked count drops below this % of last run
                                    # protects against API returning partial data on a bad day
    LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
    LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
    LIDARR_PROTECTED_PATTERNS=(
        # Metadata
        "*.nfo" "*.tbn"
        # Images — album art, artist images, Emby artwork
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        "folder.*" "cover.*" "album.*" "artist.*" "disc.*"
        # Lyrics
        "*.lrc"
        # Cue sheets and rip logs
        "*.cue" "*.log"
        # Theme music — Emby downloads artist themes into artist folders
        "theme.mp3" "theme.flac" "theme.m4a" "theme.wav"
    )

# Lidarr artwork fetcher settings
    LIDARR_ART_MIN_SIZE=10000           # bytes — reject downloads smaller than this
    LIDARR_ART_MAX_PARALLEL=4           # concurrent background download jobs
    LIDARR_ART_RETRIES=2                # download retry attempts per image
    LIDARR_ART_SLEEP_BETWEEN=0.2        # seconds between fanart.tv API calls
#   HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf

# Lidarr discovery settings (playback_aware_lidarr_discovery.sh)
    LIDARR_DISCOVERY_THRESHOLD=70         # score to accept candidate (0-100)
    LIDARR_DISCOVERY_LOOKBACK_DAYS=7      # Emby play history window in days
    LIDARR_DISCOVERY_MIN_PLAYS=3          # min plays in window before evaluating an artist
    LIDARR_DISCOVERY_USER_CAP_PCT=35      # max % any single user can contribute to play score (prevents one listener dominating)
    LIDARR_DISCOVERY_MAX_ADDS=5           # max artists to add per run — quality over bulk
    LIDARR_DISCOVERY_REJECT_COOLDOWN=30   # days before re-evaluating a rejected artist
    LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db"

# Sonarr discovery settings (playback_aware_sonarr_discovery.sh)
    SONARR_DISCOVERY_THRESHOLD=52         # score to accept candidate (0-100)
    SONARR_DISCOVERY_LOOKBACK_DAYS=14     # Emby episode play history window in days (shorter than movies — TV watched more frequently)
    SONARR_DISCOVERY_MAX_SEEDS=5          # max seed series from Stage 1
    SONARR_DISCOVERY_MAX_ADDS=3           # max shows to add per run — TV is a larger commitment than movies
    SONARR_DISCOVERY_MIN_VOTE_COUNT=50    # min TMDB votes (TV has fewer votes than movies at same popularity)
    SONARR_DISCOVERY_MIN_RATING=65        # min TMDB vote_average × 10 (65 = 6.5/10)
    SONARR_DISCOVERY_REJECT_COOLDOWN=60   # days before re-evaluating a rejected show
    SONARR_DISCOVERY_USER_EPISODE_CAP=8   # max episodes any one user contributes to seed volume score
    SONARR_DISCOVERY_MONITOR_MODE="all"    # Sonarr monitor mode on add: all | future | first | latest | none
    SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db"

# Radarr discovery shared settings
    RADARR_DISCOVERY_THRESHOLD=52         # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap
    RADARR_DISCOVERY_LOOKBACK_DAYS=30     # Emby watch history window in days (movies rewatched less often)
    RADARR_DISCOVERY_MAX_SEEDS=5          # max seed movies from Stage 1
    RADARR_DISCOVERY_MAX_ADDS=5           # max movies to add per run
    RADARR_DISCOVERY_MIN_VOTE_COUNT=100   # min TMDB votes to be considered a candidate
    RADARR_DISCOVERY_MIN_RATING=60        # min TMDB vote_average × 10 (60 = 6.0/10)
    RADARR_DISCOVERY_REJECT_COOLDOWN=60   # days before re-evaluating a rejected movie
    RADARR_DISCOVERY_SEED_LIBRARIES=("Movies")  # Emby libraries to draw seed movies from
    RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db"

# Emby → arr sync library allowlists
# Only these Emby library names will be considered by the sync tools.
# Names must match exactly as shown in Emby > Dashboard > Libraries.
    SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows")
    RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies")

# Sonarr shared settings
    SONARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    SONARR_MAX_DELETE_GB=10         # require --i-know-what-im-doing if deletion exceeds this
    SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
    SONARR_PROTECTED_PATTERNS=(
        # Subtitles
        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
        # Metadata
        "*.nfo" "*.tbn"
        # Images
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        # Kodi/Emby extras — not tracked by Sonarr API
        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
        "*-clip.*" "*-other.*"
        # Theme songs — stored in show folder, not tracked by arr
        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
        "theme.mkv" "theme.mp4"
        "trailer.*" "trailer *.*"
    )

# Radarr shared settings
    RADARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    RADARR_MAX_DELETE_GB=15         # require --i-know-what-im-doing if deletion exceeds this
    RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
    RADARR_PROTECTED_PATTERNS=(
        # Subtitles
        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
        # Metadata
        "*.nfo" "*.tbn"
        # Images
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        # Kodi/Emby extras — not tracked by Radarr API
        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
        "*-clip.*" "*-other.*"
        # Theme songs — stored in movie folder, not tracked by arr
        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
        "theme.mkv" "theme.mp4"
        "trailer.*" "trailer *.*"
    )

# Radarr/Sonarr TMDb/TVDB removed entry cleanup
    RADARR_DROPPED_ADD_EXCLUSION=true   # add removed movies to import exclusion list
    SONARR_DROPPED_ADD_EXCLUSION=true   # add removed series to import exclusion list

# ━━━ Arr Failed/Stalled Recovery ━━━
# Auto blocklist + re-search failed imports and stalled downloads.
# Runs every 6 hours — schedule: 0 */6 * * *
#
# Targets four problem types:
#   importFailed   — downloaded but arr couldn't import
#   importPending  — downloaded, stuck waiting to import (won't self-resolve)
#   error status   — serious failure not covered above
#   stalled        — download stuck with no connections or progress
#
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf.
    ARR_IMPORT_RECOVERY_AGE=6           # hours — skip items newer than this
                                        # matches cron interval — items eligible after one missed cycle

# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem symlink indirection.
# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected.
# Symlink flips between ramdisk and SSD are transparent to active streams.
#
# ⚠️ Docker mount — must use shared propagation so symlink flips work inside container:
#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
#   Standard path mappings use rprivate — symlink changes are NOT visible inside container.

# ━━━ Transcode Manager ━━━
# tmpfs mount point — created at array start by ramdisk_setup.sh.
# Must exist before Emby starts so the symlink resolves correctly.
    RAMDISK_PATH="/mnt/ramdisk_transcodes"

# Ramdisk size and flip thresholds — defined per host in host*.conf.
# All three are coupled — if size changes, thresholds must change with it.
#   HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE
#   HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB  ← flip to SSD at this usage
#   HOST1_RAMDISK_LOW_GB  / HOST2_RAMDISK_LOW_GB   ← flip back to ramdisk at this usage

# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state.
# Emby resolves the symlink once per session at start — symlink flips are transparent.
# Must match the container path configured in Emby's Extra Parameters.
    TRANSCODE_LINK="/mnt/ram-transcode"

# SSD fallback location — defined per host in host*.conf (cache path differs per server):
#   HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD

# Minimum free GB on SSD before allowing flip from ramdisk to SSD.
    RAMDISK_SSD_MIN_GB=20

    TRANSCODE_MAX_AGE=20            # minutes — HLS segment age before cleanup eligibility
    TRANSCODE_ORPHAN_AGE=30         # minutes — files with no matching active session
    TRANSCODE_FLIP_WARN=3           # notify if symlink flips this many times in one hour

    TRANSCODE_OWNER="nobody:users"
    TRANSCODE_CHMOD="755"

# Operating mode — controls symlink direction behaviour.
#   smart   — auto-flips between ramdisk and SSD based on thresholds (default)
#   ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds
#   ssd     — always uses SSD, never flips to ramdisk
    TRANSCODE_MANAGER_MODE="smart"

# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
    TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db"
    TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
    TRANSCODE_LOG_RETENTION=90      # days before old entries purged

    TRANSCODE_CHECK_EMBY=true

# ━━━ Transcode Server Array ━━━
# All media servers sharing the ramdisk transcode space.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
# Defined per host in host*.conf — Emby container names and keys differ per server:
#   HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS

# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Certificate Monitor ━━━
# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
# Checks the actual certificate served by each domain, not what NPM thinks it has.
# Domains defined per host in host*.conf — each server monitors its own domains:
#   HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS
    CERT_WARN_DAYS=30           # warn when cert expires within this many days
    CERT_CRIT_DAYS=7            # critical alert within this many days
    CERT_TIMEOUT=10             # seconds per domain before giving up

# ━━━ Backup Verify ━━━
# Verifies rsync mirror health by comparing random file checksums between servers.
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically:
#   HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
    BACKUP_VERIFY_SAMPLE=10         # random files to check per share
    BACKUP_VERIFY_MIN_SIZE=1M       # minimum file size to include in sample

# ━━━ SMART Health ━━━
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime
# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found.
    SMART_TEMP_WARN=45              # fallback — Celsius warn threshold
    SMART_TEMP_CRIT=55              # fallback — Celsius critical threshold
# Drives to ignore defined per host in host*.conf — hardware is server-specific:
#   HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES

# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS pool health and memory diagnostic report — informational only.
    ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log"
    ZFS_REPORT_ARC_WARN_PCT=90      # warn if ARC using more than this % of its max
    ZFS_REPORT_FREE_WARN_GB=10      # warn if less than this GB free RAM
    ZFS_REPORT_AVAIL_WARN_GB=20     # warn if less than this GB available on ZFS pool
    ZFS_REPORT_DOCKER_TOP=10        # how many top Docker containers to show by memory
# Pool ignore list defined per host in host*.conf — pool names are server-specific:
#   HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS

# ━━━ Bandwidth Monitor ━━━
# Called automatically by rsync.sh after each sync — one bounded write per run.
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
    BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
    BANDWIDTH_LOG_RETENTION=90      # days before old entries purged
    BANDWIDTH_WARN_GB=50            # flag syncs larger than this in weekly report

# Stats files — written by cleanup and recovery scripts, read by coffee report.
# All in DATA_DIR — array always running when these are written.
    ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db"     # lidarr/sonarr/radarr orphan stats
    ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db"   # blocklist + re-search stats

# ━━━ Health Digest ━━━
# Aggregated system health summary — reads existing state files, no new writes.
# Three profiles control when the digest is sent:
#   always  — sends every run regardless of findings
#   smart   — sends only when DIGEST_SMART_ON_* conditions are found
#   weekly  — sends once per week on DIGEST_DAY only
    DIGEST_PROFILE="weekly"         # always | smart | weekly
    DIGEST_DAY="Sunday"
    DIGEST_SMART_ON_WATCHDOG=true   # send if any watchdog strikes are active
    DIGEST_SMART_ON_FALLBACK=true   # send if fallback state is not NORMAL
    DIGEST_SMART_ON_CERT_WARN=true  # send if any cert is under CERT_WARN_DAYS
    DIGEST_SMART_ON_BANDWIDTH=true  # send if any transfer exceeded BANDWIDTH_WARN_GB

# ━━━ Emby Session Report ━━━
# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf.
    EMBY_REPORT_DAYS=7              # days to include in the report period
    EMBY_REPORT_TOP_N=10            # number of top content items to show

# ==============================================================================================
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Pressure reduction layer — keeps the system comfortable before things break.
# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop.
#
# ── PRESSURE LEVELS ───────────────────────────────────────────────────────────────────────────
#   Level 1 (soft)   — throttle SABnzbd + qBit download speeds
#   Level 2 (medium) — further throttle + docker pause background containers
#   Level 3 (hard)   — docker stop optional containers, signal docker_watchdog to defer
#
# ── PER-HOST CONTAINER LISTS ──────────────────────────────────────────────────────────────────
#   HOST*_RW_PAUSE_CONTAINERS  — docker pause at medium pressure (in host*.conf)
#   HOST*_RW_STOP_CONTAINERS   — docker stop at hard pressure (in host*.conf)

    RW_ENABLED=true
    RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db"

# ━━━ Pressure Thresholds ━━━
# Graduated RAM response — resource_watchdog acts before system_watchdog reboots.
# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always
    RW_RAM_SOFT_GB=12           # throttle start — reduce background load
    RW_RAM_MEDIUM_GB=8          # pause background containers
    RW_RAM_HARD_GB=6            # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB)
    RW_RAM_RECOVER_GB=20        # RAM must reach this before restoring hard-stopped containers

# Load average thresholds — multiplier × core count
    RW_LOAD_SOFT_MULTIPLIER=2.0     # soft pressure: 2× cores sustained
    RW_LOAD_MEDIUM_MULTIPLIER=3.0   # medium pressure: 3× cores sustained

# Consecutive runs at lower pressure before de-escalating
    RW_RECOVER_CYCLES=3

# ━━━ SABnzbd Throttle ━━━
# Speed values: "50M" = 50 MB/s, "0" = unlimited
    RW_SABNZBD_ENABLED=true
    RW_SABNZBD_SPEED_SOFT="50M"
    RW_SABNZBD_SPEED_MEDIUM="10M"

# ━━━ qBittorrent Throttle ━━━
# KB/s — 0 = unlimited
    RW_QBIT_ENABLED=true
    RW_QBIT_DL_SOFT=51200       # 50 MB/s
    RW_QBIT_DL_MEDIUM=10240     # 10 MB/s

# ━━━ Critical Containers ━━━
# Never paused or stopped regardless of pressure level.
# Keep DNS, auth, media serving, and live TV always running.
    RW_CRITICAL_CONTAINERS=(
        "NginxProxyManager"             # reverse proxy — internet access
        "Authelia"                      # auth — nothing accessible without it
        "Authelia-Secondary"
        "Mariadb-Authelia"              # Authelia dependency
        "Mariadb-Authelia-Secondary"
        "Redis-Authelia"                # Authelia dependency
        "Redis-Authelia-Secondary"
        "AdGuard-Home"                  # DNS — all LAN resolution
        "Emby"                          # media server — Live TV buffering
        "Dispatcharr"                   # Live TV scheduler — loses state if stopped
        "Dispatcharr-Basic"
        "Dispatcharr-Iptv-Users"
    )

# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Single-pass system health check — last line of defense before a crash.
# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh.
# Re-sources config at each orchestrator run — config changes take effect immediately.
#
# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
#   CRITICAL  — bypass ALL strikes, reboot immediately
#               Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only
#
#   URGENT    — bypass strikes only when OOM confirms active crisis
#               RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW
#               Without OOM confirmation → normal strike system
#
#   STANDARD  — strike system (N consecutive failures → reboot)
#               RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
#
# ── RAM ───────────────────────────────────────────────────────────────────────────────────────
#   SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass)
#   Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh

# ━━━ State Files ━━━
    SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
    DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
    SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
    SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"

# ━━━ Strike and Reboot Loop Settings ━━━
# Strike system: a check must fail this many consecutive cycles before action is taken.
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
    SYS_WATCHDOG_STRIKE_LIMIT=2         # consecutive failures before reboot trigger

# Reboot loop protection — if system keeps rebooting something is seriously wrong.
# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot.
    SYS_WATCHDOG_REBOOT_LIMIT=3
    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12

# ━━━ RAM Reboot Threshold ━━━
# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh
# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots)
    SYS_WATCHDOG_MEM_GB=4              # strike system → reboot

# ━━━ OOM Bypass Settings ━━━
# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
# in a single cycle → bypass strike system and reboot immediately.
# Rate-based: kills per 5-minute cycle, not absolute count.
# Rationale: 1-2 kills = docker_watchdog handles it ✅
#            3+ kills while RAM critical = system dying faster than watchdogs can heal ✅
    SYS_WATCHDOG_OOM_LIMIT=3           # OOM kills in one cycle to trigger bypass

# ━━━ Thresholds ━━━
# Set at "about to become unstable" — not "things are a bit high".

# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99%
    SYS_WATCHDOG_ROOTFS_PCT=95         # standard strike threshold
    SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing

# /var/log usage — log spam indicates something broken
    SYS_WATCHDOG_LOG_PCT=95

# /tmp usage — tmpfs fills from downloads, lock files fail at 100%
    SYS_WATCHDOG_TMP_PCT=90            # warn + attempt clear
    SYS_WATCHDOG_TMP_CRITICAL_PCT=98   # bypass strikes if clear failed

# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
    SYS_WATCHDOG_ARC_PINNED_PCT=98
    SYS_WATCHDOG_ARC_RELEASE_PCT=95

# Load average multiplier — threshold = MULTIPLIER × CPU core count
# MULTIPLIER=3 on 16-core = load of 48 before triggering
    SYS_WATCHDOG_LOAD_MULTIPLIER=3

# Zombie process count — large numbers = serious process management failure
    SYS_WATCHDOG_ZOMBIE_LIMIT=50

# CPU temperature — sustained high temp causes throttling or kernel panic
    SYS_WATCHDOG_CPU_TEMP_MAX=95

# File descriptor exhaustion — system-wide FD limit near exhaustion
# New connections fail silently, Docker can't spawn processes, SSH fails
    SYS_WATCHDOG_FD_CRITICAL_PCT=95    # bypass strikes — critical tier

# Runaway process — single non-container process consuming excessive CPU
# Multiple strikes before action — single spikes are normal
    SYS_WATCHDOG_RUNAWAY_CPU_PCT=90    # % single process must sustain
    SYS_WATCHDOG_RUNAWAY_STRIKES=3     # consecutive cycles before warning

# Array disk errors — accumulating mdstat errors = disk failing NOW
    SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5  # new errors in one cycle before acting

# ━━━ Check Toggles ━━━
# Per-host — moved to host*.conf
# Different servers may have different hardware, NICs, and check requirements
# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf

# ━━━ Abort Toggles ━━━
# Conditions that prevent reboot even when a threshold is hit.
# CRITICAL tier bypasses these — a truly critical condition reboots regardless.
# true  = abort standard reboot if this condition is active
# false = reboot anyway
    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true    # rebooting with bad pool risks data loss
    SYS_WATCHDOG_ABORT_ON_PARITY=false           # aborting parity better than crashing mid-check
    SYS_WATCHDOG_ABORT_ON_MOVER=false            # aborting move better than crashing mid-move

# ==============================================================================================
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
# ==============================================================================================
