#!/bin/bash
# ==============================================================================================
# ================================= MASTER CONFIGURATION =======================================
# ==============================================================================================
# All user-facing variables for the unRAID script ecosystem.
# Scripts source this file — edit here, changes apply everywhere on next git pull.
#
# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
# Every script sources Master.conf and common.sh at startup.
# Change a value here and it affects all scripts that use it — no hunting through files.
# To disable something: comment it out with # rather than deleting it.
# To add a new rsync profile: add a key to each PROFILE_* array.
# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
#   Section                         Description
#   ───────────────────────────────────────────────────────────────────────────────────────────
#   HOST CONFIGURATION              Server hostnames, SSH keys, and Emby connection details
#   LOGGING                         Enable or disable verbose logging
#   NOTIFICATIONS                   unRAID native and Discord webhook settings
#   GIT / REPO                      Gitea repository and SSH settings
#
#   ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
#   ARRAY START                     Scripts launched at array start (array_start.sh)
#   DAILY SYNC MAINTENANCE          Job list + media shares (daily_sync_maintenance.sh)
#   WEEKLY SYNC MAINTENANCE         Job list + sync jobs + sync settings (weekly_sync_maintenance.sh)
#   MEDIA MANAGEMENT                Job list for media_management.sh
#
#   ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
#   RSYNC DEFAULTS                  Global fallback rsync options and limits
#   REMOTE HEALTH CHECKS            Rootfs threshold for pre-flight abort
#   RSYNC PROFILE SYSTEM            Per-profile overrides for appdata syncs
#
#   ── FAILOVER ───────────────────────────────────────────────────────────────────────────────
#   FAILOVER                        Mutual container failover between two servers
#   FAILOVER TEST                   Simulated outage settings for failover_test.sh
#   DDNS                            Script-controlled DDNS — absolute rules
#   INTERNET LOSS                   Containers to stop when internet is lost
#   TIERED CONTAINER LISTS          What each server runs for the other per tier
#   TIER DELAY SETTINGS             How long before each tier activates (minutes)
#   RSYNC WRITEBACK JOBS            Appdata synced back to primary on handback
#
#   ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   DOCKER DAILY RESTART            Containers restarted daily
#   DOCKER WEEKLY RESTART           Containers restarted weekly
#   DOCKER WATCHDOG                 Continuous two-tier self-healing container monitoring
#   DOCKER NETWORK CONNECT          Connect containers to extra networks on array start
#
#   ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   REBOOT                          User warning delay before scheduled reboot
#   MOVER                           Mover stop timeout
#   SYSLOG FILTER                   Docker veth noise filter file path
#   PHP-FPM                         PHP-FPM max children config
#   CLEAR LOGS                      System log file paths
#   WEBGUI WATCHDOG                 WebGUI nginx + emhttp monitoring and restart
#
#   ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
#   MEDIA PERMISSIONS               Share list, mode and owner for permissions script
#   MEDIA CLEANER                   Anime and media folder lists and file patterns
#   ARR CLEANUP                     Lidarr, Sonarr, Radarr orphan file cleanup
#   ARR FAILED/STALLED RECOVERY     Auto blocklist + re-search failed imports and stalled downloads
#
#   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
#   TRANSCODE MANAGER               Ramdisk and SSD fallback transcode management
#   TRANSCODE SERVER ARRAY          Multi-server session monitoring (Emby, Jellyfin, Plex)
#
#   ── MONITORS ───────────────────────────────────────────────────────────────────────────────
#   CERTIFICATE MONITOR             SSL certificate expiry monitoring
#   BACKUP VERIFY                   Random sample checksum verification against remote
#   SMART HEALTH                    Drive SMART attribute monitoring
#   ZFS MEMORY SNAPSHOT             Weekly ZFS health and memory diagnostic report
#   BANDWIDTH MONITOR               Daily rsync transfer logging and weekly summary
#   HEALTH DIGEST                   Aggregated system health digest — always/smart/weekly
#   EMBY SESSION REPORT             Weekly Emby usage statistics via API
#
#   ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
#   SYSTEM WATCHDOG                 Continuous system health monitoring — last line of defense
#
# ==============================================================================================

# ==============================================================================================
# ── HOST CONFIGURATION ────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Hosts ━━━
# Hostnames must match exact Docker/unRAID hostnames — case sensitive.
# Used by detect_hosts() in common.sh to determine which server is local and which is remote.
# Both servers run identical scripts — host detection makes them bidirectional.
    HOST1="unRAID-Gmer4Lfe"
    HOST2="unRAID-Jayred365"

# SSH keys for server-to-server rsync and failover container operations.
# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys.
    HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
    HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"

# ━━━ Emby ━━━
# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh,
# emby_database_repair.sh, weekly_sync_maintenance.sh, and TRANSCODE_SERVERS array.
# API key: Emby Dashboard → API Keys → + New Key
    HOST1_EMBY_CONTAINER="Emby"
    HOST1_EMBY_URL="http://localhost:8096"
    HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829"

    HOST2_EMBY_CONTAINER="Emby-Jayred365"
    HOST2_EMBY_URL="http://localhost:8096"       # same port — different server, different key
    HOST2_EMBY_API_KEY="your-host2-emby-api-key"

# ==============================================================================================
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Controls verbose [LOG] output across all scripts.
# true  = show detailed [LOG] lines — useful for debugging or first-time setup
# false = show only user-facing output — cleaner for scheduled runs
    ENABLE_LOGGING=true

# ==============================================================================================
# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# unRAID native notification system — integrates with the bell icon in the WebGUI.
# normal = job completed successfully / warning = something failed or needs attention
    NOTIFY_UNRAID=true

# Discord webhook URL — leave blank to disable
    DISCORD_WEBHOOK=""

# ==============================================================================================
# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Gitea self-hosted repository — used by git_pull_execute.sh.
# Detects Gitea container location at runtime — works through failover automatically.
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
    GITEA_CONTAINER="Gitea"
    GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git"
    GITEA_DOMAIN=""                                     # e.g. git.gmer4lfe.com — requires NPM + DNS
    TARGET_DIR="/mnt/user/appdata/unraid_scripts"
    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
    SSH_PORT=221

# ==============================================================================================
# ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# All orchestrator job lists live here — edit arrays to add/remove scripts.
# No changes to orchestrator scripts needed when adding or removing jobs.

# ━━━ Array Start ━━━
# Scripts launched by array_start.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally.
# Continuous scripts (watchdogs, failover) run until array stops.

ARRAY_START_SCRIPTS=(
    "unRAID_Essentials/ramdisk_setup.sh"          # creates ramdisk + symlink before Emby starts
    "unRAID_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
    "unRAID_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
    #"Docker_Essentials/docker_network_connect.sh" # connect containers to extra networks
    "unRAID_Essentials/system_watchdog.sh"        # system health monitor — continuous loop
    "Docker_Essentials/docker_watchdog.sh"        # container health monitor — continuous loop
    #"Failover/failover.sh"                        # mutual failover — continuous loop
)

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs the media share sync built into the script first,
# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs.
# Schedule: 0 1 * * * (1am daily)

DAILY_MAINTENANCE_SCRIPTS=(
    "git_pull_execute.sh"                         # pull latest scripts — always runs first
    "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts
)

# Media shares synced daily by daily_sync_maintenance.sh.
# Each server syncs only the shares it owns (source of truth) — direction is automatic.
# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1.
# Never both pushing the same share — one server is always the truth holder.
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
# For shares needing custom options or container stops — create a profile in the RSYNC section.

HOST1_DAILY_SYNC_SHARES=(
    /mnt/user/Books
    /mnt/user/Intros
    /mnt/user/Kids_Movies
    /mnt/user/Kids_Tv_Shows
    /mnt/user/Movies
    /mnt/user/Music
    /mnt/user/Music_Videos
    /mnt/user/Nextcloud
    /mnt/user/stand-up_comedy
    /mnt/user/Sports
    /mnt/user/Tv_Shows
    /mnt/user/Anime_Shows-Old
    /mnt/user/Anime_Movies-Old
)

HOST2_DAILY_SYNC_SHARES=(
    /mnt/user/Anime_Movies
    /mnt/user/Anime_Shows
)

# Job list run directly by daily_sync_maintenance.sh after the media share sync.
# Runs sequentially — permissions first, then cleaners, then arr cleanup.
# Comment out any job to disable without removing it.
# Each individual script can still be run manually for one-off maintenance.

MEDIA_MANAGEMENT_JOBS=(
    "Media/media_shares_permissions.sh"           # apply permissions — runs first
    "Media/media_cleaner.sh anime"                # remove junk from anime shares
    "Media/media_cleaner.sh media"                # remove junk from media shares
    "Media/lidarr_cleanup.sh"                     # remove orphaned music files
    "Media/sonarr_cleanup.sh"                     # remove orphaned TV files
    "Media/radarr_cleanup.sh"                     # remove orphaned movie files
)

# Personal encrypted shares — synced for offsite backup, independent of media shares.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
HOST1_PERSONAL_SHARES=(
    # /mnt/user/Gmer4Lfe-Personal     # uncomment after creating encrypted dataset
)

HOST2_PERSONAL_SHARES=(
    # /mnt/user/Jayred365-Personal    # uncomment after creating encrypted dataset
)

# ━━━ Weekly Sync Maintenance ━━━
# weekly_sync_maintenance.sh handles the critical sync built into the script first:
#   stop containers both sides → pull updates → sync Emby + Critical-Data → restart
# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs.
# Schedule: 30 2 * * 0 (Sunday 2:30am)

WEEKLY_MAINTENANCE_SCRIPTS=(
    "Docker_Essentials/docker_weekly_restart.sh"  # weekly container restarts after sync
)

# Shares synced by weekly_sync_maintenance.sh during the maintenance window.
# Containers are stopped both sides before these sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
# Order matters — Emby first, then auth stack.
WEEKLY_SYNC_JOBS=(
    "/mnt/user/Media_Server/Emby"               # emby profile — full clean mirror
    "/mnt/user/appdata-Failover/Critical-Data"  # critical-data profile — auth stack
)

# Container update toggles for the weekly sync window.
# Containers are already stopped for the sync — updates pull at no extra downtime.
# Both false → sync only, no updates.
# Toggle false temporarily to skip updates without changing the schedule.
    CRITICAL_SYNC_UPDATES=true          # pull container updates locally
    CRITICAL_SYNC_UPDATES_REMOTE=false   # pull container updates on remote via SSH

# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Rsync Defaults ━━━
# Global fallback values used when no profile match is found.
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
# Appdata shares match profiles by directory basename (lowercased).
    BW_LIMIT=12500
    RETRY_COUNT=3
    SLEEP=300
    CRITICAL_CONTAINER_NAMES=()
    DELAYED_CONTAINERS=()
    CONTAINER_DELAY=5
    EXCLUDE_DIRS=()
    DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)

# ━━━ Remote Health Checks ━━━
# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage.
# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server.
    ROOTFS_WARN=75

# ━━━ Rsync Profile System ━━━
# Profiles allow per-share rsync behaviour without touching script logic.
# Profile key matched by basename of directory passed to rsync.sh (lowercased).
# Override with --profile=name flag.
#
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
#            List ALL desired options explicitly when defining a profile.
#
# Current profiles:
#   arrs_stack    — arr databases — lower bandwidth, containers stopped for consistency
#   critical-data — auth stack — containers stopped both sides, Authelia delayed start
#   gmer4lfe      — server-specific appdata — no container stops needed
#   important-data — NextCloud + Postgres — NextCloud delayed start after Postgres
#   emby          — weekly clean sync — both Emby stopped, full mirror, minimal excludes
#                   called by weekly_sync_maintenance.sh only — do NOT schedule separately
#   emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only
#                   also used for failover writeback on handback

declare -A PROFILE_RSYNC_OPTS=(
    [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
    [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
    [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT"
    [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
    [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
    [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
)

declare -A PROFILE_BW_LIMIT=(
    [arrs_stack]=5000
    [critical-data]=9500
    [gmer4lfe]=8000
    [important-data]=9500
    [emby]=8000
    [emby-failover]=9500
)

declare -A PROFILE_RETRY_COUNT=(
    [arrs_stack]=3
    [critical-data]=3
    [gmer4lfe]=3
    [important-data]=3
    [emby]=3
    [emby-failover]=3
)

declare -A PROFILE_SLEEP=(
    [arrs_stack]=300
    [critical-data]=300
    [gmer4lfe]=300
    [important-data]=300
    [emby]=300
    [emby-failover]=120
)

# Containers stopped on BOTH LOCAL and REMOTE servers before rsync.
# Local stops first — flushes databases cleanly. Remote stops next — prevents writes while receiving.
# Same container names on both servers — consistent naming is required by this ecosystem.
# If a container is not found it is skipped gracefully, not errored.
# SPACE-SEPARATED STRINGS — converted to array at runtime
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
    [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
    [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
    [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe"
    [important-data]="Postgres-NextCloud NextCloud"
    [emby]="Emby"
    [emby-failover]=""
)

# SPACE-SEPARATED STRINGS — converted to array at runtime
declare -A PROFILE_DELAYED_CONTAINERS=(
    [arrs_stack]=""
    [critical-data]="Authelia Authelia-Secondary"
    [gmer4lfe]=""
    [important-data]="NextCloud"
    [emby]=""
    [emby-failover]=""
)

declare -A PROFILE_CONTAINER_DELAY=(
    [arrs_stack]=5
    [critical-data]=15
    [gmer4lfe]=5
    [important-data]=10
    [emby]=5
    [emby-failover]=5
)

# SPACE-SEPARATED STRINGS — converted to array at runtime
declare -A PROFILE_EXCLUDE_DIRS=(
    [arrs_stack]="logs *.tmp"
    [gmer4lfe]="logs *.tmp"
    [important-data]="logs *.tmp"
    [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
    [emby]="logs transcodes cache crash*"
    [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
)

declare -A PROFILE_SKIP_DISK_CHECK=(
    [arrs_stack]=true
    [critical-data]=true
    [gmer4lfe]=true
    [important-data]=true
    [emby]=true
    [emby-failover]=true
)

# ==============================================================================================
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs Failover/failover.sh independently via array_start.sh.
# All decisions based on two pings: remote reachable + internet reachable.
#
# States: NORMAL | FAILOVER | NO_INTERNET | DARK
#
# DDNS rules — absolute:
#   Internet loss → stop own DDNS immediately
#   Failover → start remote DDNS first (Tier 1)
#   Handback → stop remote DDNS → rsync → start containers → start local DDNS last
#
# Tiers:
#   Tier 1 — Immediate — vital services + Live TV
#   Tier 2 — configurable delay — productivity services
#   Tier 3 — configurable delay — secondary services
#   Tier 4 — configurable delay — arrs + downloaders

    EXTERNAL_IP="8.8.8.8"
    FAILOVER_CHECK_INTERVAL=120
    FAILOVER_HANDBACK_STRIKES=2
    FAILOVER_STATE_FILE="/boot/config/failover_state.db"

# ━━━ Failover Test ━━━
    FAILOVER_TEST_BLOCK_WAIT=150
    FAILOVER_TEST_HANDBACK_WAIT=360

# ━━━ DDNS ━━━
HOST1_DDNS_CONTAINERS=(
    "Gmer4Lfe.com"
)

HOST2_DDNS_CONTAINERS=(
    "Gmer4Lfe.us"
)

# ━━━ Internet Loss ━━━
FAILOVER_HOST1_STOP_ON_NO_NET=(
    "Gmer4Lfe.com"
)

FAILOVER_HOST2_STOP_ON_NO_NET=(
    "Gmer4Lfe.us"
)

# ━━━ Tiered Container Lists ━━━

# HOST1 runs for HOST2
FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=(
    "Gmer4Lfe.us"
    "VaultWarden-Jayred365"
    # "container-placeholder"
)

FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=(
    # "container-placeholder"
)

FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=(
    # "container-placeholder"
)

FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=(
    # "container-placeholder"
)

# HOST2 runs for HOST1
FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=(
    "Gmer4Lfe.com"
    "Emby"
    "VaultWarden-Gmer4Lfe"
    "Dispatcharr"
    "Dispatcharr-Basic"
    "Dispatcharr-Iptv-Users"
    "ErsatzTV-Emby"
)

FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=(
    "Postgres-NextCloud"
    "NextCloud"
    "PostgreSQL_Immich"
    "Immich-Gmer4Lfe"
    # "container-placeholder"
)

FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=(
    "Gitea"
    # "container-placeholder"
)

FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=(
    "Sonarr"
    "Radarr"
    "Lidarr"
    "Readarr"
    "Prowlarr"
    "Bazarr"
    "SABnzbd-Gmer4Lfe"
    "Qbittorrent-Gmer4Lfe"
    "LidaTube"
    "Pinchflat"
    "ChannelTube"
    # "container-placeholder"
)

# ━━━ Tier Delay Settings ━━━
# Minutes before each tier activates. Tier 1 is always immediate.
HOST1_TIER2_DELAY=240
HOST1_TIER3_DELAY=720
HOST1_TIER4_DELAY=1440

HOST2_TIER2_DELAY=240
HOST2_TIER3_DELAY=720
HOST2_TIER4_DELAY=1440

# ━━━ Rsync Writeback Jobs ━━━
# Syncs critical appdata back to primary on handback — containers stopped before this runs.
# Short outages skip writeback — primary state is more reliable than dirty sync data.
# Tier 4 automatically syncs HOST*_DAILY_SYNC_SHARES — add edge cases here only.

HOST1_TIER1_WRITEBACK_DELAY=60
HOST2_TIER1_WRITEBACK_DELAY=60

FAILOVER_HOST1_WRITEBACK_TIER1=(
    "/mnt/user/Media_Server/Emby"
)

FAILOVER_HOST1_WRITEBACK_TIER2=(
    "/mnt/user/appdata-Failover/Important-Data"
)

FAILOVER_HOST1_WRITEBACK_TIER3=(
    # "location-placeholder"
)

FAILOVER_HOST1_WRITEBACK_TIER4=(
    "/mnt/user/appdata-Failover/Arrs_Stack"
)

FAILOVER_HOST2_WRITEBACK_TIER1=(
    # "/mnt/user/appdata-Failover/Jayred365-Emby"
)

FAILOVER_HOST2_WRITEBACK_TIER2=(
    # "/mnt/user/appdata-Failover/Jayred365-Important"
)

FAILOVER_HOST2_WRITEBACK_TIER3=(
    # "location-placeholder"
)

FAILOVER_HOST2_WRITEBACK_TIER4=(
    "/mnt/user/appdata-Failover/Arrs_Stack"
)

# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Docker Daily Restart ━━━
# Restarted by docker_daily_restart.sh via daily_sync_maintenance.sh — 1am daily.
DAILY_RESTART_CONTAINERS=(
    "NginxProxyManager"
    "Authelia"
    "Dispatcharr-Iptv-Users"
    "Dispatcharr"
    "Dispatcharr-Basic"
    "ErsatzTV-Emby"
)

# ━━━ Docker Weekly Restart ━━━
# Restarted by docker_weekly_restart.sh via weekly_sync_maintenance.sh — Sunday 2:30am.
WEEKLY_RESTART_CONTAINERS=(
    "NextCloud"
    "Organizrv2-Gmer4Lfe"
    "AdGuard-Home"
    "Immich-Gmer4Lfe"
)

# ━━━ Docker Watchdog ━━━
# Continuous two-tier self-healing container monitoring.
# Started by array_start.sh — runs until array stops.
# Re-sources Master.conf each cycle — add/remove containers without restarting watchdog.
# Silent when all healthy — only logs when something needs attention.
#
# Tier 1 — strict monitoring of explicitly configured containers
#   Memory hard limits, CPU thresholds, HTTP responsiveness, required container checks
# Tier 2 — global health scan of ALL running containers
#   Unhealthy status, OOM kills, crash loops, dead containers, unexpected exits

# Memory hard limits in MB — immediate restart if exceeded
# 20GB=20480  16GB=16384  12GB=12288  8GB=8192  4GB=4096  2GB=2048  1GB=1024
declare -A WATCHDOG_CONTAINERS=(
    ["Emby"]=16384
    ["LidaTube"]=6144
    ["Tdarr"]=6144
    ["Code-Server"]=1024
)

declare -A HOST1_WATCHDOG_CONTAINER_URLS=(
    ["Emby"]="http://localhost:8096"
)

declare -A HOST2_WATCHDOG_CONTAINER_URLS=(
    ["Emby"]="http://localhost:8096"
)

HOST1_WATCHDOG_REQUIRED_CONTAINERS=(
    "NginxProxyManager"
    "Lldap-Gmer4Lfe"
    "Authelia"
    "Mariadb-Authelia"
    "Redis-Authelia"
    "Authelia-Secondary"
    "Redis-Authelia-Secondary"
)

HOST2_WATCHDOG_REQUIRED_CONTAINERS=(
    "NginxProxyManager"
    # add HOST2 required containers here
)

    WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
    SOFT_CPU_THRESHOLD=80
    HARD_CPU_THRESHOLD=85
    CPU_FAIL_LIMIT=2
    SOFT_MEM_THRESHOLD=80
    RESP_FAIL_LIMIT=2
    CURL_TIMEOUT=5
    DOCKER_WATCHDOG_INTERVAL=900

    WATCHDOG_SCAN_ALL=true

WATCHDOG_SCAN_IGNORE=(
    # "container-name"
)

    WATCHDOG_RESTART_UNHEALTHY=true
    WATCHDOG_RESTART_DEAD=true
    WATCHDOG_RESTART_CRASHED=true
    WATCHDOG_NOTIFY_OOM=true
    WATCHDOG_NOTIFY_CRASHLOOP=true
    WATCHDOG_CRASH_LIMIT=5
    WATCHDOG_STARTUP_GRACE=600
    WATCHDOG_CONTAINER_RESTART_LIMIT=3
    WATCHDOG_CONTAINER_RESTART_WINDOW=1
    WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db"
    WATCHDOG_BATCH_NOTIFY=true

declare -A WATCHDOG_DEPENDENCIES=(
    ["Authelia"]="Mariadb-Authelia Redis-Authelia"
    ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary"
    ["NextCloud"]="Postgres-NextCloud"
)

# ━━━ Docker Network Connect ━━━
# Connects containers to extra networks on array start via array_start.sh.
NETWORK_CONNECT_CONTAINERS=(
    "memcached"
    "Npm-CrowdSec"
)

NETWORK_CONNECT_NETWORKS=(
    "high-availability"
)

# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Reboot ━━━
    REBOOT_SLEEP=300

# ━━━ Mover ━━━
    MOVER_STOP_TIMEOUT=300

# ━━━ Syslog Filter ━━━
    FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"

# ━━━ PHP-FPM ━━━
    PHP_CONF="/etc/php-fpm.d/www.conf"
    PHP_MAX_CHILDREN=250

# ━━━ Clear Logs ━━━
    LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)

# ━━━ WebGUI Watchdog ━━━
    WEBGUI_URL="http://localhost"
    WEBGUI_TIMEOUT=5
    WEBGUI_NGINX_WAIT=15
    WEBGUI_EMHTTP_WAIT=30

# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Media Permissions ━━━
# Applied recursively by media_shares_permissions.sh via MEDIA_MANAGEMENT_JOBS.
    PERMISSIONS_MODE="777"
    PERMISSIONS_OWNER="nobody:users"

MEDIA_PERMISSION_SHARES=(
    /mnt/user/Anime_Movies
    /mnt/user/Anime_Movies-Old
    /mnt/user/Anime_Shows
    /mnt/user/Anime_Shows-Old
    /mnt/user/appcache
    /mnt/user/Books
    /mnt/user/Downloads
    /mnt/user/Games
    /mnt/user/Intros
    /mnt/user/Kids_Movies
    /mnt/user/Kids_Tv_Shows
    /mnt/user/Movie_Recordings
    /mnt/user/Movies
    /mnt/user/Music
    /mnt/user/Music_Videos
    /mnt/user/Photo
    /mnt/user/Sports
    /mnt/user/stand-up_comedy
    /mnt/user/Temp_Storage
    /mnt/user/Tv_Recordings
    /mnt/user/Tv_Shows
    /mnt/user/YouTube
)

# ━━━ Media Cleaner ━━━
# Removes junk files from media shares — two profiles: anime and media.
# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media

ANIME_CLEAN_FOLDERS=(
    /mnt/user/Anime_Movies
    /mnt/user/Anime_Movies-Old
    /mnt/user/Anime_Shows
    /mnt/user/Anime_Shows-Old
)

MEDIA_CLEAN_FOLDERS=(
    /mnt/user/Kids_Movies
    /mnt/user/Kids_Tv_Shows
    /mnt/user/Movies
    /mnt/user/Music
    /mnt/user/Sports
    /mnt/user/stand-up_comedy
    /mnt/user/Tv_Shows
)

ANIME_FILE_PATTERNS=(
    '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
    '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
    '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
    '*.log' '*.json'
)

MEDIA_FILE_PATTERNS=(
    '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
    '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
    '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
    '*.log' '*.json' '*.iso' '*.lrc'
)

# ━━━ Arr Cleanup ━━━
# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE.
# detect_hosts() selects correct URL, API key, and root path at runtime.
#
# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr
# are not included in the tracked file API response but must not be deleted.
#
# API versions: Sonarr v4 → /api/v3/   Radarr v6 → /api/v3/   Lidarr v3 → /api/v1/
# Lidarr runs on HOST1 only.

# ── Lidarr ────────────────────────────────────────────────────────────────────────────────────
HOST1_LIDARR_URL="http://192.168.50.2:8686"
HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc"
HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New"

LIDARR_ORPHAN_AGE=7
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc")
LIDARR_MAX_DELETE_GB=1          # require --i-know-what-im-doing if deletion exceeds this
LIDARR_MIN_TRACKED_PCT=80       # abort if tracked count drops below this % of last run
LIDARR_TRACKED_COUNT_FILE="/boot/config/lidarr_tracked.count"

# ── Sonarr ────────────────────────────────────────────────────────────────────────────────────
HOST1_SONARR_URL="http://192.168.50.2:8989"
HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f"
HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows"

HOST2_SONARR_URL="http://localhost:8989"
HOST2_SONARR_API_KEY="your-host2-sonarr-api-key"
HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows"

SONARR_ORPHAN_AGE=7
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa")

# ── Radarr ────────────────────────────────────────────────────────────────────────────────────
HOST1_RADARR_URL="http://192.168.50.2:7878"
HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9"
HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies"

HOST2_RADARR_URL="http://localhost:7878"
HOST2_RADARR_API_KEY="your-host2-radarr-api-key"
HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies"

RADARR_ORPHAN_AGE=7
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa")

# ━━━ Arr Failed/Stalled Recovery ━━━
# Auto blocklist + re-search failed imports and stalled downloads.
# Runs every 6 hours — schedule: 0 */6 * * *
#
# Targets four problem types:
#   importFailed   — downloaded but arr couldn't import
#   importPending  — downloaded, stuck waiting to import (won't self-resolve)
#   error status   — serious failure not covered above
#   stalled        — download stuck with no connections or progress
#
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/
# Lidarr runs on HOST1 only — exits cleanly on HOST2.

ARR_IMPORT_RECOVERY_AGE=6      # hours — skip items newer than this

HOST1_SONARR_RECOVERY=true
HOST1_RADARR_RECOVERY=true
HOST1_LIDARR_RECOVERY=true     # HOST1 only
HOST2_SONARR_RECOVERY=true
HOST2_RADARR_RECOVERY=true

# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem symlink indirection.
# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected.
#
# How it works:
#   ramdisk_setup.sh        — creates tmpfs and symlink at array start via array_start.sh
#   transcode_management.sh — every 3min, runs cleanup then manager in correct order
#   transcode_cleanup.sh    — removes old inactive files
#   transcode_manager.sh    — manages symlink direction based on usage thresholds
#
# ⚠️ Docker mount — must use shared propagation:
#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
#   Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently.

# ━━━ Transcode Manager ━━━
    RAMDISK_PATH="/mnt/ramdisk_transcodes"
    RAMDISK_SIZE="8G"
    TRANSCODE_LINK="/mnt/ram-transcode"
    TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/"
    RAMDISK_WARN_GB=6.8
    RAMDISK_LOW_GB=5.5
    RAMDISK_SSD_MIN_GB=20
    TRANSCODE_MAX_AGE=20
    TRANSCODE_ORPHAN_AGE=30
    TRANSCODE_FLIP_WARN=3
    TRANSCODE_OWNER="nobody:users"
    TRANSCODE_CHMOD="755"
    TRANSCODE_MANAGER_MODE="smart"   # smart | ramdisk | ssd
    TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
    TRANSCODE_LOG_RETENTION=90

# ━━━ Transcode Server Array ━━━
# All media servers sharing the ramdisk transcode space.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD.

TRANSCODE_SERVERS=(
    "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby"
    # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby"
    # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin"
    # "Plex|http://localhost:32400|plex-token|plex"
)

    TRANSCODE_CHECK_EMBY=true

# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Certificate Monitor ━━━
CERT_MONITOR_DOMAINS=(
    "Gmer4Lfe.com"
    "Gmer4Lfe.us"
)
    CERT_WARN_DAYS=30
    CERT_CRIT_DAYS=7
    CERT_TIMEOUT=10

# ━━━ Backup Verify ━━━
# Leave empty to use HOST*_DAILY_SYNC_SHARES automatically.
BACKUP_VERIFY_SHARES=(
    # leave empty to use daily sync shares automatically
)
    BACKUP_VERIFY_SAMPLE=10
    BACKUP_VERIFY_MIN_SIZE=1M

# ━━━ SMART Health ━━━
    SMART_TEMP_WARN=45
    SMART_TEMP_CRIT=55
SMART_IGNORE_DRIVES=(
    "sda"
)

# ━━━ ZFS Memory Snapshot ━━━
    ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
    ZFS_REPORT_ARC_WARN_PCT=90
    ZFS_REPORT_FREE_WARN_GB=10
    ZFS_REPORT_AVAIL_WARN_GB=20
    ZFS_REPORT_DOCKER_TOP=10
ZFS_REPORT_IGNORE_POOLS=(
    "disk10"
    "disk9"
    "disk8"
    "disk6"
    "disk5"
)

# ━━━ Bandwidth Monitor ━━━
# Called by rsync.sh after each sync — bounded write, minimal flash wear.
    BANDWIDTH_LOG="/boot/config/bandwidth_history.db"
    BANDWIDTH_LOG_RETENTION=90
    BANDWIDTH_WARN_GB=50

# ━━━ Health Digest ━━━
# Reads existing state files — no new flash writes.
# Profiles: always | smart | weekly
    DIGEST_PROFILE="weekly"
    DIGEST_DAY="Sunday"
    DIGEST_SMART_ON_WATCHDOG=true
    DIGEST_SMART_ON_FAILOVER=true
    DIGEST_SMART_ON_CERT_WARN=true
    DIGEST_SMART_ON_BANDWIDTH=true

# ━━━ Emby Session Report ━━━
# Weekly Emby usage statistics via API — no persistent writes.
# URL and API key from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY in Host Configuration.
    EMBY_REPORT_DAYS=7
    EMBY_REPORT_TOP_N=10

# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Continuous system health monitoring — last line of defense before a crash.
# Started by array_start.sh — runs until array stops.
# Re-sources Master.conf each cycle — config changes take effect on next cycle.
# Strike system: sustained threshold hits trigger reboot — single spikes ignored.
# Reboot loop protection: shuts down instead if reboot limit hit in rolling window.
# Silent when healthy — logs only when a threshold is triggered.

# ━━━ State Files ━━━
    SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"       # /tmp resets on reboot ✅
    SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"    # survives reboots
    SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"    # reboot loop detection

# ━━━ Strike and Reboot Loop Settings ━━━
    SYS_WATCHDOG_STRIKE_LIMIT=2
    SYSTEM_WATCHDOG_INTERVAL=300    # seconds between cycles (5min default)
    SYS_WATCHDOG_REBOOT_LIMIT=3
    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12

# ━━━ Thresholds ━━━
    SYS_WATCHDOG_ROOTFS_PCT=95
    SYS_WATCHDOG_LOG_PCT=95
    SYS_WATCHDOG_MEM_GB=4
    SYS_WATCHDOG_ARC_PINNED_PCT=98
    SYS_WATCHDOG_ARC_RELEASE_PCT=95
    SYS_WATCHDOG_LOAD_MULTIPLIER=3
    SYS_WATCHDOG_ZOMBIE_LIMIT=50
    SYS_WATCHDOG_CPU_TEMP_MAX=95

# ━━━ Check Toggles ━━━
    SYS_WATCHDOG_CHECK_ROOTFS=true
    SYS_WATCHDOG_CHECK_LOG=true
    SYS_WATCHDOG_CHECK_RAM=true
    SYS_WATCHDOG_CHECK_ARC=true
    SYS_WATCHDOG_CHECK_CPU_TEMP=true
    SYS_WATCHDOG_CHECK_LOAD=false       # disabled — load spikes during transcoding are normal
    SYS_WATCHDOG_CHECK_ZOMBIES=true
    SYS_WATCHDOG_CHECK_CONTAINERS=true
    SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true

# ━━━ Abort Toggles ━━━
# true = abort reboot if condition active / false = reboot anyway
    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
    SYS_WATCHDOG_ABORT_ON_PARITY=false
    SYS_WATCHDOG_ABORT_ON_MOVER=false

# ==============================================================================================
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
# ==============================================================================================