1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= MASTER CONFIGURATION =======================================
4	# ==============================================================================================
5	# Shared configuration for the unRAID script ecosystem.
6	# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists.
7	#
8	# ── HOW THE THREE-FILE SYSTEM WORKS ──────────────────────────────────────────────────────────
9	# Scripts source all three files at startup:
10	#   source master.conf          ← shared config (this file)
11	#   source host1.conf    ← HOST1 credentials, shares, container lists
12	#   source host2.conf    ← HOST2 credentials, shares, container lists
13	#
14	# Sparse checkout (git) ensures each server only pulls its own host*.conf.
15	# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials.
16	#
17	# What belongs here:      thresholds, toggles, intervals, profiles, job lists
18	# What belongs in HOST*:  hostnames, SSH keys, API keys, passwords, share paths,
19	#                         container names, failover lists, watchdog containers
20	#
21	# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
22	# Every script sources all three conf files and common.sh at startup.
23	# Change a value here and it affects all scripts that use it — no hunting through files.
24	# To disable something: comment it out with # rather than deleting it.
25	# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section.
26	# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
27	#
28	# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
29	#
30	#   Section                         Description
31	#   ───────────────────────────────────────────────────────────────────────────────────────────
32	#   SHARED HOST CONFIGURATION       DATA_DIR and shared connection settings
33	#   PARTNERSHIP                     Mirror relationship lifecycle — onboard/offboard/transfer
34	#   LOGGING                         Enable or disable verbose logging
35	#   NOTIFICATIONS                   unRAID native and Discord webhook settings
36	#   GIT / REPO                      Gitea repository and SSH settings
37	#
38	#   ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
39	#   ARRAY START                     Scripts launched at array start (array_started.sh)
40	#   ARRAY STOP                      Scripts run at planned shutdown (array_stopping.sh)
41	#   WATCHDOG ORCHESTRATOR           Per-minute watchdog runner (watchdog_orchestrator.sh)
42	#   SYSTEM WATCHDOG                 Sub-scripts called by watchdog_orchestrator.sh
43	#   CRITICAL SYNC MAINTENANCE       30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh)
44	#   INTERMEDIATE SYNC MAINTENANCE   Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh)
45	#   DAILY SYNC MAINTENANCE          Job list + media shares (daily_sync_maintenance.sh)
46	#   WEEKLY SYNC MAINTENANCE         Job list + sync shares + update toggles (weekly_sync_maintenance.sh)
47	#   MONTHLY MAINTENANCE             Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh)
48	#
49	#   ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
50	#   RSYNC ENABLE/DISABLE            Two-tier toggle system — global gate + per-orchestrator
51	#   RSYNC DEFAULTS                  Global fallback rsync options and limits
52	#   REMOTE HEALTH CHECKS            Rootfs threshold for pre-flight abort
53	#   RSYNC PROFILE SYSTEM            Per-profile overrides for appdata syncs
54	#
55	#   ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
56	#   FALLBACK                        Mutual container failover shared settings
57	#   FALLBACK TEST                   Simulated outage settings for fallback_test.sh
58	#
59	#   ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
60	#   DOWNLOADERS RESET               Retention and thresholds for slskd, SABnzbd, qBittorrent
61	#   DOCKER DAILY RESTART            Containers restarted daily
62	#   DOCKER WEEKLY RESTART           Containers restarted weekly
63	#   DOCKER WATCHDOG                 Thresholds and toggles for container monitoring
64	#   DOCKER NETWORK CONNECT          Networks to ensure + containers to connect
65	#
66	#   ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
67	#   INOTIFY TUNING                  inotify limits — raised at array start by inotify_tuning.sh
68	#   SYSTEM TUNING MONITOR           Tracks inotify + php-fpm usage over time
69	#   REBOOT                          User warning delay before scheduled reboot
70	#   MOVER                           Mover stop timeout
71	#   SYSLOG FILTER                   Docker veth noise filter file path
72	#   PHP-FPM                         PHP-FPM max children config
73	#   CLEAR LOGS                      System log file paths
74	#   WEBGUI WATCHDOG                 WebGUI nginx + emhttp monitoring and restart
75	#
76	#   ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
77	#   MEDIA PERMISSIONS               Share list, mode and owner for permissions script
78	#   MEDIA CLEANER                   Anime and media folder lists and file patterns
79	#   ARR CLEANUP                     Lidarr, Sonarr, Radarr orphan file cleanup settings
80	#   ARR FAILED/STALLED RECOVERY     Auto blocklist + re-search settings
81	#
82	#   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
83	#   TRANSCODE MANAGER               Ramdisk and SSD fallback transcode management
84	#   TRANSCODE SERVER ARRAY          Multi-server session monitoring (Emby, Jellyfin, Plex)
85	#
86	#   ── MONITORS ───────────────────────────────────────────────────────────────────────────────
87	#   CERTIFICATE MONITOR             SSL certificate expiry monitoring
88	#   BACKUP VERIFY                   Random sample checksum verification against remote
89	#   SMART HEALTH                    Drive SMART attribute monitoring
90	#   ZFS MEMORY SNAPSHOT             Weekly ZFS health and memory diagnostic report
91	#   BANDWIDTH MONITOR               Daily rsync transfer logging and weekly summary
92	#   HEALTH DIGEST                   Aggregated system health digest — always/smart/weekly
93	#   EMBY SESSION REPORT             Weekly Emby usage statistics via API
94	#
95	#   ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
96	#   SYSTEM WATCHDOG                 Continuous system health monitoring — last line of defense
97	#
98	# ==============================================================================================
99
100	# ==============================================================================================
101	# ── HOST IDENTITIES ───────────────────────────────────────────────────────────────────────────
102	# ==============================================================================================
103	# Hostnames for every server in the ecosystem — not credentials, safe for all machines.
104	# Must match the exact unRAID hostname AND Tailscale device name (case sensitive).
105	# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID.
106	# Tailscale IP resolution uses these names — no hardcoded IPs needed.
107	# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf.
108	    HOST1="unRAID-Gmer4Lfe"
109	    HOST2="unRAID-Jayred365"
110
111	# ==============================================================================================
112	# ── SHARED HOST CONFIGURATION ─────────────────────────────────────────────────────────────────
113	# ==============================================================================================
114	# Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash —
115	# scripts, state, and data are all available before the array mounts.
116	# Both directories are created automatically if they don't exist.
117	#
118	#   DATA_DIR   — historical logs, statistics, discovery histories, blocklists
119	#   STATE_DIR  — runtime state files for all scripts (watchdogs, fallback, transcode, etc.)
120	#                Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root.
121	    DATA_DIR="/boot/config/plugins/varaverk/data"
122	    STATE_DIR="/boot/config/plugins/varaverk/State_Files"
123
124	# ── Version Parity ──
125	# Controls behaviour when local and remote unRAID versions differ.
126	# Major version mismatch always aborts regardless of this setting.
127	# "warn"  — log warning and continue (safe for minor/patch differences)
128	# "abort" — refuse to continue (strict — ensures both sides always match)
129	    UNRAID_VERSION_MISMATCH_ACTION="warn"
130
131	# ── Arr Sync ──
132	# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
133	# Runs before rsync — all nodes agree on tracked library before files are transferred.
134	# Remote API keys are read live from each node's config.xml via SSH — never stored here.
135	    ARR_SYNC_ENABLED=true
136	    ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
137	    ARR_SYNC_CONNECT_TIMEOUT=10     # seconds — SSH connect timeout per node
138	    ARR_SYNC_API_TIMEOUT=60         # seconds — curl timeout for library fetches
139	    DOCKER_APPDATA_BASE="/mnt/user/appdata"
140	    ARR_SYNC_LIDARR_PORT=8686
141	    ARR_SYNC_SONARR_PORT=8989
142	    ARR_SYNC_RADARR_PORT=7878
143
144	# ── Remote Docker Daemon ──
145	# Strike system for remote Docker daemon health checks.
146	# Scripts that issue remote container commands check the remote daemon first.
147	# Below limit → skip operation this run | At limit → notify critical + exit
148	    REMOTE_DOCKER_STRIKE_LIMIT=3       # consecutive failures before critical notify
149
150	# ==============================================================================================
151	# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
152	# ==============================================================================================
153	# Manages the relationship lifecycle between two unRAID servers.
154	# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
155	# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
156	# All identity vars (hostnames, SSH keys) live in host*.conf.
157	# Hostnames already match Tailscale device names — IP resolution is automatic.
158	#
159	# State files in STATE_DIR — survives reboots, array must be up (scripts run after array start):
160	#   STATE_DIR/partnership_HOST1.db ← HOST1 writes only
161	#   STATE_DIR/partnership_HOST2.db ← HOST2 writes only
162	#   STATE_DIR/partnership_blocklist.db ← shared blocklist
163	#   STATE_DIR/varaverk_setup.db ← onboarding progress state
164	#   Propagated via SSH — no rsync needed
165	#
166	# critical_sync_maintenance.sh runs --check every 30min:
167	#   Reads both state files via SSH
168	#   Detects offboard requests → finalises from owner side
169	#   Increments offline counter → auto-offboards after threshold
170	#   Silent when healthy ✅
171	#
172	# See README-Partnership.md for full lifecycle documentation.
173
174	    PARTNERSHIP_ENABLED=false
175	    PARTNERSHIP_OWNER_HOST="HOST1"          # "HOST1" or "HOST2" — flips on --transfer
176
177	# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf.
178	# Format: "ContainerName|WebUIPort"
179	# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS
180	# On onboard → WebUI pointed at owner's Tailscale IP
181	# On offboard → WebUI pointed back at localhost
182
183	# Paths to collect during the grace window after offboard — defined per host in host*.conf.
184	# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS
185	# Notified on offboard — no auto-deletion, manual collection.
186
187	# Timing — single var controls both Tailscale removal and backup access expiry.
188	# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless.
189	    PARTNERSHIP_GRACE_HOURS=6               # hours after offboard before Tailscale removal
190	                                            # backup access expires at the same time
191	    PARTNERSHIP_OFFLINE_THRESHOLD=30        # days either server unreachable before auto-offboard
192	                                            # works both directions independently
193
194	# Partnership and setup state files — all in STATE_DIR per the project requirement.
195	# Scripts use these variables; do not hardcode /boot/config paths in scripts.
196	    PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db"
197	    VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"
198
199	# Tailscale removal on offboard.
200	    PARTNERSHIP_REMOVE_TAILSCALE=true       # remove mirror from Tailscale tailnet on offboard
201	                                            # false = skip removal (manual or testing)
202
203	# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
204	# Stays in shared conf — only owner uses it, and owner is always running this script.
205	# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope
206	    TAILSCALE_API_KEY=""                    # tskey-api-...
207	    TAILSCALE_TAILNET=""                    # your tailnet name (e.g. yourname.github)
208
209	# Transfer safety.
210	    PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
211	    PARTNERSHIP_TRANSFER_STRIKES=3          # consecutive health checks required
212	    PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20    # max health check attempts before giving up
213
214	# Onboard settings.
215	    PARTNERSHIP_ONBOARD_VERIFY=true         # verify WebUI reachable after reconfiguration
216	    PARTNERSHIP_ONBOARD_NOTIFY=true         # notify both servers on completion
217	    PARTNERSHIP_SYNC_INTERVAL=15            # minutes — informational, actual schedule in cron
218
219	# SSH key lifecycle — managed by Partnership/ssh_setup.sh.
220	# Key named after this server: hostname lowercased, unraid- prefix stripped.
221	#   unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation
222	# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically.
223	    SSH_MAX_STRIKES=5                       # consecutive SSH auth failures before critical notify
224	    SSH_STRIKE_RESET_HRS=24                 # hours since last failure before strike counter resets
225
226	# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato.
227	# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard.
228	# Folder name derived from remote hostname (strip unraid- prefix case-insensitively).
229	# Plugin config: /boot/config/plugins/folder.view3/docker.json
230	    PARTNERSHIP_FOLDERVIEW3=true            # create/remove FolderView3 folder on onboard/offboard
231	    PARTNERSHIP_FOLDERVIEW3_URL=""          # CA plugin URL — leave empty to skip auto-install
232
233	# ==============================================================================================
234	# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
235	# ==============================================================================================
236
237	# Controls verbose [LOG] output across all scripts.
238	# true  = show detailed [LOG] lines — useful for debugging or first-time setup
239	# false = show only user-facing output — cleaner for scheduled runs
240	    ENABLE_LOGGING=false
241
242	# ==============================================================================================
243	# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
244	# ==============================================================================================
245
246	# unRAID native notification system — integrates with the bell icon in the WebGUI.
247	# normal = job completed successfully / warning = something failed or needs attention
248	    NOTIFY_UNRAID=true
249
250	# Discord webhook URL — defined per host in host*.conf.
251	# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK
252	# Allows different webhooks per server, or only one server notifying.
253
254	# ==============================================================================================
255	# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
256	# ==============================================================================================
257
258	# Gitea self-hosted repository — used by git_pull_execute.sh.
259	# Detects Gitea container location at runtime — works through failover automatically.
260	# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
261	    GITEA_CONTAINER="Gitea"
262	    GITEA_REPO_PATH="FailedProxy/Varaverk.git"
263	    GITEA_DOMAIN=""                         # e.g. git.yourdomain.com — requires NPM + DNS
264	    TARGET_DIR="/boot/config/plugins/varaverk"
265	    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
266	    SSH_PORT=221                            # Gitea SSH port (default 22, Gitea often uses 221/222)
267	    GITEA_HTTP_PORT=3000                    # Gitea web/API port — used by gitea_ssh_setup.sh
268
269	# ==============================================================================================
270	# ── ORCHESTRATORS ─────────────────────────────────────────────────────────────────────────────
271	# ==============================================================================================
272	# All orchestrator job lists live here — edit arrays to add/remove scripts.
273	# No changes to orchestrator scripts needed when adding or removing jobs.
274	# Sections ordered by run frequency: array events first, then shortest interval to longest.
275
276	# ━━━ Array Start ━━━
277	# Scripts launched by array_started.sh when the array comes online.
278	# Launched in order — each as a background process.
279	# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
280	# Continuous scripts (failover) run until array stops.
281	# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
282	# watchdog_orchestrator.sh — NOT launched here.
283	    ARRAY_START_SCRIPTS=(
284	        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
285	        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
286	        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
287	        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
288	        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
289	        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
290	        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
291	        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
292	        "Fallback/fallback.sh"                        # mutual failover — continuous
293	    )
294
295	# ━━━ Array Stop ━━━
296	# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
297	# Run sequentially (foreground) — each must complete before the next starts.
298	# Order matters: user scripts first (prevents new ops), then data movement, then containers.
299	    ARRAY_STOP_SCRIPTS=(
300	        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
301	        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
302	        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
303	        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
304	        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last
305	    )
306
307	# ━━━ Watchdog Orchestrator ━━━
308	# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
309	# Schedule: * * * * * (every minute)
310	# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
311	# Order matters — resource first (frees pressure), docker second (heals with freed resources),
312	# system third (storage + webgui component health), stability last (last line of defense).
313	    WATCHDOG_ORCHESTRATOR_SCRIPTS=(
314	        "Watchdogs/resource_watchdog.sh"              # reduce system pressure before healing attempts
315	        "Watchdogs/docker_watchdog.sh"                # heal containers with freed resources
316	        "Watchdogs/system_watchdog.sh"                # system component health — storage + webgui
317	        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register API key if registry lost it
318	        "Watchdogs/stability_watchdog.sh"             # reboot if all else fails — last line of defense
319	    )
320
321	# ━━━ System Watchdog ━━━
322	# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
323	# Called by watchdog_orchestrator.sh — not scheduled directly.
324	    SYSTEM_WATCHDOG_SCRIPTS=(
325	        "Watchdogs/System/storage_watchdog.sh"   # pool growth + runaway log detection
326	        "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh"    # WebGUI availability — nginx → php-fpm → emhttp
327	        "Watchdogs/System/network_watchdog.sh"   # internet, DDNS, Tailscale, NPM proxy
328	    )
329
330	# ━━━ Critical Sync Maintenance ━━━
331	# critical_sync_maintenance.sh runs every 30 minutes.
332	# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
333	# partnership --check always runs last regardless of rsync gate.
334	# Comment out entries to disable without removing.
335	    CRITICAL_MAINTENANCE_SCRIPTS=(
336	        "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 30min
337	        "Media/play_state_sync.sh"                      # sync watched/resume state across Emby + Jellyfin
338	    )
339
340	# Shares synced every 30 minutes — defined per host in host*.conf.
341	# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
342	# Format: "/path/to/share" or "/path/to/share|profile-name"
343	# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
344
345	# ━━━ Intermediate Sync Maintenance ━━━
346	# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
347	# and optional mid-day rsync for any shares that need sub-daily propagation.
348	# Schedule: 0 */4 * * *
349	    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.
350
351	    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
352	        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
353	    )
354	    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
355	    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
356
357	# ━━━ Daily Sync Maintenance ━━━
358	# daily_sync_maintenance.sh runs media share sync first, then iterates
359	# DAILY_MAINTENANCE_SCRIPTS for all jobs.
360	# Schedule: 0 1 * * * (1am daily)
361	    DAILY_MAINTENANCE_SCRIPTS=(
362	        "git_pull_execute.sh"                         # pull latest scripts — always runs first
363	        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
364	        "Media/media_cleaner.sh anime"                # remove junk from anime shares
365	        "Media/media_cleaner.sh media"                # remove junk from media shares
366	        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
367	        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
368	        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
369	        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
370	        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
371	        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
372	        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
373	        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
374	    )
375
376	# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
377	# Containers keep running during pull — no extra downtime.
378	# Set false to skip updates while still running the daily restart.
379	    DAILY_CONTAINER_UPDATES=true
380
381	# Media shares synced daily by daily_sync_maintenance.sh.
382	# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
383	# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
384	# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
385	# Any node can download content to any share — it propagates to all nodes on the next cycle.
386	# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
387	# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
388	# For shares needing custom options or container stops — create a profile in RSYNC section.
389
390	# Personal encrypted shares defined per-host in host*.conf.
391	# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
392	# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.
393
394	# ━━━ Weekly Sync Maintenance ━━━
395	# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
396	# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
397	# Schedule: 30 2 * * 0 (Sunday 2:30am)
398	    WEEKLY_MAINTENANCE_SCRIPTS=(
399	        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
400	        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
401	        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
402	        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
403	        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
404	        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
405	    )
406
407	# Pull updates for all running containers NOT in daily/weekly restart lists.
408	# Ensures every deployed container receives at least one image pull per week.
409	# Set false to skip — docker_weekly_restart.sh still runs regardless.
410	    WEEKLY_REMAINING_UPDATES=true
411
412	# Shares synced during the weekly maintenance window — defined per host in host*.conf.
413	# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
414	# Containers stopped both sides before sync — full clean state guaranteed.
415	# Profiles drive container stops, excludes, and options — configure in RSYNC section.
416	# Order matters — Emby first (larger), then Critical-Data (auth stack).
417
418	# Container update toggles for the weekly sync window.
419	# Containers already stopped for sync — updates pull at no extra downtime.
420	# Both false → sync only, no updates.
421	    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
422	    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH
423
424	# ━━━ Monthly Maintenance ━━━
425	# monthly_maintenance.sh fires only when BOTH gates pass:
426	#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
427	#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
428	# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe)
429	# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry.
430	# Add scripts that require a long-stable settled system — scrubs, extended drive tests.
431	# State file on /boot/config — survives reboots (interval gate independent of uptime gate).
432	    MONTHLY_MAINTENANCE_SCRIPTS=(
433	        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
434	        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential
435	    )
436
437	    MONTHLY_UPTIME_THRESHOLD_DAYS=30    # minimum uptime in days before maintenance fires
438	    MONTHLY_RUN_INTERVAL_DAYS=30        # minimum days since last run before running again
439	    MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db"
440
441	# ━━━ Sunday Morning Coffee Report ━━━
442	# Orchestrator that runs all Sunday monitor scripts in sequence.
443	# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am)
444	# Each script runs independently and notifies on its own findings.
445	    COFFEE_REPORT_SCRIPTS=(
446	        "Monitors/zfs_memory_snapshot.sh"       # ZFS pool health + ARC + Docker memory snapshot
447	        "Monitors/smart_health.sh"              # drive SMART attributes — reallocated, pending, temp
448	        "Monitors/cert_monitor.sh"              # SSL certificate expiry for all configured domains
449	        "Monitors/backup_verify.sh"             # rsync mirror integrity via independent MD5 checksums
450	        "Monitors/bandwidth_monitor.sh"         # weekly rsync transfer totals and per-share breakdown
451	        "Monitors/emby_session_report.sh"       # Emby usage — streams, users, library, transcode ratio
452	        "Monitors/weekly_health_digest.sh"      # aggregated digest — watchdogs, fallback, skip list
453	    )
454
455	# ==============================================================================================
456	# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
457	# ==============================================================================================
458
459	# ━━━ Rsync Enable/Disable ━━━
460	# Two-tier toggle system — Tier 1 overrides Tier 2.
461	#
462	# Tier 1 — Global gate:
463	#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
464	#   Use when: remote completely offline, major maintenance, disaster recovery
465	#
466	# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
467	#   Fine grained control — disable specific orchestrators while keeping others
468	#   Use when: rebuilding secondary, testing, per-window bandwidth management
469	#
470	# Example — HOST2 data rebuild:
471	#   RSYNC_ENABLED=true                ← rsync works, individual scripts run fine
472	#   CRITICAL_RSYNC_ENABLED=true       ← 30min auth stack sync still runs
473	#   INTERMEDIATE_RSYNC_ENABLED=false  ← skip 4h arr/mid-day rsync during rebuild
474	#   DAILY_RSYNC_ENABLED=false         ← skip daily HDD syncs during rebuild
475	#   WEEKLY_RSYNC_ENABLED=true         ← Emby + Critical-Data still sync (NVMe)
476	#   FALLBACK_RSYNC_ENABLED=true       ← handback writeback still works
477	#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
478	#   → When ready: INTERMEDIATE_RSYNC_ENABLED=true  DAILY_RSYNC_ENABLED=true
479	    RSYNC_ENABLED=true                  # Tier 1 — global gate, overrides everything below
480	    CRITICAL_RSYNC_ENABLED=true        # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
481	    INTERMEDIATE_RSYNC_ENABLED=true     # Tier 2 — intermediate_sync_maintenance.sh rsync section
482	    DAILY_RSYNC_ENABLED=false           # Tier 2 — HOST2 rebuild in progress, re-enable when ready
483	    WEEKLY_RSYNC_ENABLED=true           # Tier 2 — weekly_sync_maintenance.sh rsync section
484	    FALLBACK_RSYNC_ENABLED=true         # Tier 2 — fallback.sh writeback jobs on handback
485
486	# ━━━ Rsync Defaults ━━━
487	# Global fallback values used when no profile match is found.
488	# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
489	# Appdata shares match profiles by directory basename (lowercased).
490	    BW_LIMIT=12500                  # KB/s — 12500 ≈ 100Mbit
491	    RETRY_COUNT=3                   # retry attempts before giving up
492	    SLEEP=300                       # seconds between retry attempts
493	    CRITICAL_CONTAINER_NAMES=()     # containers stopped on REMOTE before rsync — profiles override
494	    DELAYED_CONTAINERS=()           # containers needing delay before starting — profiles override
495	    CONTAINER_DELAY=5               # seconds before starting delayed containers
496	    EXCLUDE_DIRS=()                 # directories excluded from transfer — profiles override
497
498	# --inplace      writes directly to destination — delta against existing file, better for large media
499	# --partial      keep partial file on interrupted transfer so next run resumes, not re-transfers
500	# --timeout      kill stalled transfers instead of hanging indefinitely
501	# --numeric-ids  use UIDs/GIDs numerically — prevents ownership mismatches between servers
502	# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync.
503	# Media shares use this default; rsync spreads files only, never removes them.
504	# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default).
505	# Note: arr_cleanup only catches true orphans under the union model — intentional removals
506	#       require arr_sync.sh --blocklist-add first, then manual file deletion.
507	    DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids)
508
509	# ━━━ Remote Health Checks ━━━
510	# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
511	# When remote array is down, rsync writes land on rootfs and fill it rapidly.
512	    ROOTFS_WARN=75
513
514	# ━━━ Rsync Profile System ━━━
515	# Profiles allow per-share rsync behaviour without touching script logic.
516	# Profile key matched by basename of directory passed to rsync.sh (lowercased).
517	# Override with --profile=name flag.
518	#
519	# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
520	#            List ALL desired options explicitly when defining a profile.
521	#
522	# Current profiles:
523	#   arrs_stack        — arr databases — lower bandwidth, containers stopped for consistency
524	#   critical-data     — auth stack — full stop both sides, Authelia delayed start
525	#                       called by weekly_sync_maintenance.sh — full clean sync weekly
526	#   critical-fallback — dirty sync — auth stays running both sides, WAL excluded
527	#                       called by critical_sync_maintenance.sh every 30min
528	#   host1-appdata     — HOST1 server-specific appdata — defined in host1.conf
529	#   host2-appdata     — HOST2 server-specific appdata — defined in host2.conf
530	#   important-data    — NextCloud + Postgres — NextCloud delayed start after Postgres
531	#   emby              — weekly clean sync — both Emby stopped, full mirror
532	#                       called by weekly_sync_maintenance.sh only — do NOT schedule separately
533	#   emby-fallback     — dirty sync — Emby stays running, WAL excluded
534	#                       called by critical_sync_maintenance.sh every 30min
535
536	    declare -A PROFILE_RSYNC_OPTS=(
537	        [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
538	        [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
539	        [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
540	        [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
541	        [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
542	        [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
543	    )
544
545	# Per-profile bandwidth limits in KB/s
546	    declare -A PROFILE_BW_LIMIT=(
547	        [arrs_stack]=5000           # lower — runs alongside other syncs
548	        [critical-data]=9500        # high — small dataset, sync fast
549	        [critical-fallback]=9500    # high — small dataset, sync fast
550	        [important-data]=9500       # high — database sync
551	        [emby]=8000                 # medium — large full mirror
552	        [emby-fallback]=9500        # high — small critical dataset
553	    )
554
555	# Retry attempts per profile
556	    declare -A PROFILE_RETRY_COUNT=(
557	        [arrs_stack]=3
558	        [critical-data]=3
559	        [critical-fallback]=3
560	        [important-data]=3
561	        [emby]=3
562	        [emby-fallback]=3
563	    )
564
565	# Seconds between retry attempts
566	    declare -A PROFILE_SLEEP=(
567	        [arrs_stack]=300
568	        [critical-data]=300
569	        [critical-fallback]=120     # shorter — frequent dirty sync, retry faster
570	        [important-data]=300
571	        [emby]=300
572	        [emby-fallback]=120         # shorter — frequent dirty sync, retry faster
573	    )
574
575	# Containers stopped on BOTH LOCAL and REMOTE before rsync.
576	# Local stops first — flushes databases cleanly. Remote stops next — prevents writes.
577	# Only running containers get restarted — stopped containers stay stopped.
578	# SPACE-SEPARATED STRINGS — converted to array at runtime
579	    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
580	        [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
581	        [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
582	        [critical-fallback]=""      # dirty sync — auth stays running both sides
583	        [important-data]="Postgres-NextCloud NextCloud"
584	        [emby]="Emby"
585	        [emby-fallback]=""          # dirty sync — Emby stays running both sides
586	    )
587
588	# Containers needing a delay after rsync before starting.
589	# SPACE-SEPARATED STRINGS — converted to array at runtime
590	    declare -A PROFILE_DELAYED_CONTAINERS=(
591	        [arrs_stack]=""
592	        [critical-data]="Authelia Authelia-Secondary"   # wait for Mariadb + Redis
593	        [critical-fallback]=""
594	        [important-data]="NextCloud"                    # wait for Postgres
595	        [emby]=""
596	        [emby-fallback]=""
597	    )
598
599	# Seconds before starting delayed containers
600	    declare -A PROFILE_CONTAINER_DELAY=(
601	        [arrs_stack]=5
602	        [critical-data]=15          # Mariadb + Redis need time to accept connections
603	        [critical-fallback]=5
604	        [important-data]=10         # Postgres needs time before NextCloud
605	        [emby]=5
606	        [emby-fallback]=5
607	    )
608
609	# Directories excluded from rsync per profile.
610	# SPACE-SEPARATED STRINGS — converted to array at runtime
611	    declare -A PROFILE_EXCLUDE_DIRS=(
612	        [arrs_stack]="logs *.tmp"
613	        [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
614	        [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
615	        [important-data]="logs *.tmp"
616	        [emby]="logs transcodes cache crash*"
617	        [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
618	    )
619
620	# Remote restart after dirty sync — restart these on remote IF they were running before sync.
621	# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted.
622	# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes.
623	# SPACE-SEPARATED STRINGS — converted to array at runtime
624	    declare -A PROFILE_REMOTE_RESTART_CONTAINERS=(
625	        [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary"
626	        [emby-fallback]="Emby"
627	    )
628
629	# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed.
630	# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically.
631
632	# ==============================================================================================
633	# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
634	# ==============================================================================================
635	# Mutual container failover between two unRAID servers.
636	# Each server runs Fallback/fallback.sh independently via array_started.sh.
637	# All decisions based on two pings: remote reachable + internet reachable.
638	#
639	# States: NORMAL | FALLBACK | NO_INTERNET | DARK
640	#
641	# DDNS rules — absolute:
642	#   Internet loss → stop own DDNS immediately
643	#   Failover → start remote DDNS first (Tier 1)
644	#   Handback → stop remote DDNS → rsync → start containers → start local DDNS last
645	#
646	# Per-host container lists and tier delays live in host*.conf.
647	# Shared settings (intervals, state file, thresholds) live here.
648
649	    EXTERNAL_IP="8.8.8.8"
650	    FALLBACK_CHECK_INTERVAL=30      # seconds between fallback state checks
651	    FALLBACK_HANDBACK_STRIKES=3     # consecutive healthy checks before initiating handback (3×30s = 90s)
652	    FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
653	    FALLBACK_ENABLED=true           # HOST2 back online
654	                                    # false = suppresses "not running" warnings in status scripts
655	    FALLBACK_PARTNERSHIP_REQUIRED=true          # gate fallback on an active partnership
656	                                                # false = standalone mode, no partnership dependency
657	    FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120      # minutes without active partnership before suspending
658	                                                # 0 = suspend immediately when partnership goes inactive
659
660	# ━━━ Failover Test ━━━
661	# Controlled simulation of a failover event — run manually via fallback_test.sh.
662	    FALLBACK_TEST_BLOCK_WAIT=150    # seconds to wait after blocking connectivity
663	    FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback
664
665	# ==============================================================================================
666	# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
667	# ==============================================================================================
668
669	# ━━━ Downloaders Reset ━━━
670	# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS.
671	# Clears stuck states, purges old history, prepares each download client for a clean cycle.
672	# Per-host URLs and API keys live in host*.conf.
673	    DOWNLOADER_RETENTION_DAYS=7     # days — purge history older than this
674
675	# qBittorrent failsafe — removes torrents older than threshold regardless of ratio
676	# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files)
677	    QBIT_FAILSAFE_MIN_DAYS=180      # days — minimum age before failsafe deletion
678	    QBIT_FAILSAFE_MIN_RATIO=0       # 0 = age only, no ratio requirement
679
680	# ━━━ Docker Daily Restart ━━━
681	# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
682	# Per-host lists live in host*.conf:
683	#   HOST1_DAILY_RESTART_CONTAINERS
684	#   HOST2_DAILY_RESTART_CONTAINERS
685	# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime.
686
687	# ━━━ Docker Weekly Restart ━━━
688	# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
689	# Containers already stopped for weekly sync — restart adds zero extra downtime.
690	# Per-host lists live in host*.conf:
691	#   HOST1_WEEKLY_RESTART_CONTAINERS
692	#   HOST2_WEEKLY_RESTART_CONTAINERS
693
694	# ━━━ Docker Watchdog ━━━
695	# Continuous two-tier self-healing container monitoring.
696	# Started by array_started.sh — runs until array stops.
697	# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog.
698	#
699	# Tier 1 — strict monitoring of explicitly configured containers:
700	#   Memory hard limits    — immediate restart if exceeded
701	#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT sustained strikes
702	#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT failed checks
703	#   Required containers   — must always be running, strike + skip list with auto-clear
704	#
705	# Tier 2 — global health scan of ALL running containers:
706	#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
707	#   OOM killed            — kernel killed → restart + notify
708	#   Crash loop detection  — RestartCount climbing → notify, critical above limit
709	#   Dead containers       — remove and restart
710	#   Unexpected exits      — non-zero exit code → restart
711	#
712	# All per-host container lists live in host*.conf:
713	#   HOST*_WATCHDOG_CONTAINERS         — memory hard limits per container
714	#   HOST*_WATCHDOG_CONTAINER_URLS     — HTTP health check URLs
715	#   HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
716	#   HOST*_WATCHDOG_SCAN_IGNORE        — skip in Tier 2 scan
717	#   HOST*_WATCHDOG_DEPENDENCIES       — dependency ordering for restart decisions
718
719	# Strike state file — persistent in STATE_DIR
720	    WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db"
721
722	# CPU thresholds — normalised against total core count at runtime
723	    SOFT_CPU_THRESHOLD=80           # warn at this % of total system CPU
724	    HARD_CPU_THRESHOLD=85           # strike at this % of total system CPU
725	    CPU_FAIL_LIMIT=2                # consecutive hard CPU strikes before container restart
726
727	# Memory soft threshold — warn when container reaches this % of its hard limit
728	    SOFT_MEM_THRESHOLD=80
729
730	# HTTP responsiveness
731	    RESP_FAIL_LIMIT=2               # consecutive failed checks before restart
732	    CURL_TIMEOUT=5                  # seconds per check before timeout
733
734	# Tier 2 master toggle
735	    WATCHDOG_SCAN_ALL=true          # false = only WATCHDOG_CONTAINERS + required containers
736
737	# Individual Tier 2 check toggles
738	    WATCHDOG_RESTART_UNHEALTHY=true
739	    WATCHDOG_RESTART_DEAD=true
740	    WATCHDOG_RESTART_CRASHED=true
741	    WATCHDOG_NOTIFY_OOM=true
742	    WATCHDOG_NOTIFY_CRASHLOOP=true
743
744	# Crash loop threshold
745	    WATCHDOG_CRASH_LIMIT=5          # RestartCount above this = critical crash loop
746
747	# Startup grace period — skip restarts while system is still booting
748	    WATCHDOG_STARTUP_GRACE=600      # seconds after boot before watchdog acts on failures
749
750	# Restart loop protection — prevents watchdog from endlessly restarting a broken container
751	    WATCHDOG_CONTAINER_RESTART_LIMIT=3
752	    WATCHDOG_CONTAINER_RESTART_WINDOW=1     # rolling window in hours
753	    WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
754
755	# Notification batching — one summary per cycle instead of one ping per event
756	    WATCHDOG_BATCH_NOTIFY=true
757
758	# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files.
759	#
760	# Part 1 — Growth rate (zero-config):
761	#   Reads per-container dir totals each cycle via du, compares to previous cycle.
762	#   Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan
763	#   inside that container. No per-container config required — new containers are covered
764	#   automatically. Baseline built on first run after boot; growth detection starts cycle 2.
765	#
766	# Part 2 — Absolute log size:
767	#   Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths.
768	#   Catches logs that have already stabilised at a large size and are no longer actively growing.
769	#
770	# Strike system (reuses existing watchdog infrastructure):
771	#   Strike 1 — warn + notify: condition first detected
772	#   Strike 2 — warn + escalated notify: still present next cycle
773	#   Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
774	#     If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes
775	#     If false: critical notify only, strikes held until condition resolves
776	#   Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
777	#
778	# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a
779	#   container until its dir exceeds the configured ceiling. Only needed when a container
780	#   legitimately has large stable data and you want to guarantee it never triggers a false alarm.
781	    WATCHDOG_CHECK_APPDATA=true
782	    WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata")
783	    WATCHDOG_APPDATA_GROWTH_GB=2            # flag containers growing more than this per cycle
784	    WATCHDOG_APPDATA_LOG_MAX_GB=2           # flag *.log files exceeding this size (absolute)
785	    WATCHDOG_APPDATA_TRUNCATE_LOGS=false    # set true to auto-truncate oversized *.log files on action cycle
786	    WATCHDOG_APPDATA_STRIKE_LIMIT=3         # cycles before action fires (matches existing watchdog pattern)
787	    WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db"
788	    STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db"
789
790	# ━━━ Docker Network Connect ━━━
791	# Ensures custom networks exist and connects containers at array start.
792	# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run.
793	# Container and network lists are host-specific — defined in host*.conf:
794	#   HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS
795	#   HOST1_NETWORK_CONNECT_NETWORKS   / HOST2_NETWORK_CONNECT_NETWORKS
796
797	# ==============================================================================================
798	# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
799	# ==============================================================================================
800
801	# ━━━ inotify Tuning ━━━
802	# Linux inotify limits — applied at every array start by inotify_tuning.sh.
803	# Default unRAID values are very low — with many Docker containers watching files
804	# you can silently exhaust the limit causing containers to miss file events.
805	# Symptoms: containers miss file events, downloads not detected, library not updated.
806	# Live TV stutter is a known symptom of inotify exhaustion.
807	# These settings are lost on reboot — reapplied automatically at array start.
808	    INOTIFY_MAX_INSTANCES=1024      # default: 128   — max inotify instances per user
809	                                    # 1024 handles ~20-30 containers watching files simultaneously
810	    INOTIFY_MAX_WATCHES=1048576     # default: 8192  — SHARED budget across ALL users/containers
811	                                    # 524288 (512K) was previous value — raised to 1M (1048576)
812	                                    # VSCode/Code-Server alone needs ~50K-200K for large workspaces
813	                                    # with node_modules. All arr containers + Emby + VSCode share
814	                                    # this budget. 1M is safe on 128GB RAM (~128MB kernel memory)
815	                                    # If VSCode shows "unable to watch for file changes" → too low
816	    INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping
817
818	# ━━━ System Tuning Monitor ━━━
819	# Tracks inotify and php-fpm usage over time.
820	# Snapshot written every 6 hours by system_tuning_monitor.sh.
821	# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
822	    INOTIFY_WARN_PCT=80             # warn if inotify instances exceed this % of limit
823	    PHP_FPM_WARN_PCT=80             # warn if php-fpm workers exceed this % of max_children
824	    TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
825	    TUNING_LOG_RETENTION=30         # days before old entries are purged
826
827	# ━━━ Reboot ━━━
828	# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
829	# Gives users time to save work — 300s = 5 minutes.
830	    REBOOT_SLEEP=300
831
832	# ━━━ Mover ━━━
833	# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
834	# Gives mover time to finish current file transfer before being interrupted.
835	    MOVER_STOP_TIMEOUT=300
836
837	# ━━━ Syslog Filter ━━━
838	# Path for the rsyslog filter file that suppresses Docker veth interface noise.
839	# Docker creates a new veth interface for each container — generates hundreds of
840	# log lines per hour that have no diagnostic value. Filter removes them at source.
841	    FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
842
843	# ━━━ PHP-FPM ━━━
844	# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
845	# Default is very low — increasing it prevents WebGUI slowdowns under load.
846	# 250 is safe for servers with 32GB+ RAM.
847	    PHP_CONF="/etc/php-fpm.d/www.conf"
848	    PHP_MAX_CHILDREN=250
849
850	# ━━━ Clear Logs ━━━
851	# System log files cleared weekly to prevent rootfs fill over time.
852	# These grow continuously — without clearing they eventually consume all rootfs space.
853	# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
854	#
855	# Size threshold approach — only clear if log exceeds threshold.
856	# Avoids destroying useful recent diagnostic context when logs are small.
857	# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing)
858	# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size
859	#   Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr)
860	#   100MB per container × 30 containers = 3GB before clearing kicks in
861	    LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
862	    LOG_MIN_SIZE_MB=10          # skip system log if under this size (already small)
863	    LOG_DOCKER_MAX_MB=100       # clear Docker container log only if over this size (MB)
864
865	# ━━━ Network Watchdog ━━━
866	# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy.
867	# Host-specific values (domain, container, NPM URL) live in host*.conf.
868	    NETWORK_WATCHDOG_ENABLED=true
869	    NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1"
870	    NETWORK_WATCHDOG_INTERNET_TIMEOUT=5
871	    NETWORK_WATCHDOG_CHECK_TAILSCALE=true
872	    NETWORK_WATCHDOG_NPM_TIMEOUT=10
873	    NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2
874	    NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db"
875
876	# ━━━ WebGUI Watchdog ━━━
877	# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
878	# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
879	    WEBGUI_URL="http://localhost"
880	    WEBGUI_TIMEOUT=5        # seconds before curl gives up on WebGUI check
881	    WEBGUI_NGINX_WAIT=15    # seconds after nginx restart before rechecking
882	    WEBGUI_EMHTTP_WAIT=30   # seconds after emhttp restart before rechecking
883
884	# ==============================================================================================
885	# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
886	# ==============================================================================================
887
888	# ━━━ Media Permissions ━━━
889	# Applied recursively by media_shares_permissions.sh.
890	# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
891	#
892	# Why split directory vs file permissions:
893	#   Directories need execute bit to enter — 755 allows owner+group+others to traverse
894	#   Files should NOT be executable — 664 allows owner+group read/write, others read
895	#   This is POSIX best practice for media servers, not a blanket 777 band-aid
896	#
897	# Why nobody:users ownership:
898	#   All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID
899	#   Files owned by nobody:users are accessible to all containers without world-write
900	#   If this script fixes many files each run → a container has wrong PUID/PGID set
901	#   Fix: add PUID=99 PGID=100 to the container's environment variables
902	#
903	# This script runs daily as a failsafe — even with correct container config:
904	#   rsync may bring files with source ownership if not run with --chown
905	#   Manual admin copies create root:root files
906	#   New containers may not have PUID/PGID set yet
907	    PERMISSIONS_DIR_MODE="755"    # directories — traverse + list, no world-write
908	    PERMISSIONS_FILE_MODE="664"   # files — group read/write, no execute
909	    PERMISSIONS_OWNER="nobody:users"
910
911	# Share list defined per host in host*.conf:
912	#   HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES
913
914	# ━━━ Media Cleaner ━━━
915	# Removes junk files from media shares — two profiles: anime and media.
916	# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
917	# Folder lists defined per host in host*.conf:
918	#   HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS
919	#   HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS
920
921	    ANIME_FILE_PATTERNS=(
922	        # ── Checksums ────────────────────────────────────────────────────────────────────────
923	        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download
924
925	        # ── Scene / download metadata ─────────────────────────────────────────────────────────
926	        '*.url' '*.lnk'                     # scene links
927	        '*.info' '*.diz'                    # scene description files
928	        '*.nzb'                             # usenet download files
929	        '*.torrent'                         # torrent files left by download clients
930
931	        # ── Archives and segments ─────────────────────────────────────────────────────────────
932	        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
933	        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
934	        '*.r04' '*.r05' '*.r06' '*.r07'
935	        '*.r08' '*.r09' '*.srr'             # scene repair files
936	        '*.001' '*.002' '*.003'             # split archive parts
937	        '*.gz' '*.tar' '*.bz2'             # linux archives
938
939	        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
940	        '*.sample*' '*.proof*'              # scene samples — never needed in library
941
942	        # ── Executables and scripts ───────────────────────────────────────────────────────────
943	        '*.exe' '*.scr' '*.com'             # Windows executables
944	        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
945	        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
946	        '*.sh'                              # shell scripts in media folders = suspicious
947
948	        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
949	        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
950	        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
951	        '*.part'                            # partial download files
952
953	        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
954	        '*sync-conflict*'                   # Syncthing conflict copies
955	    )
956
957	    MEDIA_FILE_PATTERNS=(
958	        # ── Checksums ────────────────────────────────────────────────────────────────────────
959	        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download
960
961	        # ── Scene / download metadata ─────────────────────────────────────────────────────────
962	        '*.url' '*.lnk'                     # scene links
963	        '*.info' '*.diz'                    # scene description files
964	        '*.nzb'                             # usenet download files
965	        '*.torrent'                         # torrent files left by download clients
966
967	        # ── Archives and segments ─────────────────────────────────────────────────────────────
968	        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
969	        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
970	        '*.r04' '*.r05' '*.r06' '*.r07'
971	        '*.r08' '*.r09' '*.srr'             # scene repair files
972	        '*.001' '*.002' '*.003'             # split archive parts
973	        '*.gz' '*.tar' '*.bz2'             # linux archives
974
975	        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
976	        '*.sample*' '*.proof*'              # scene samples — never needed in library
977
978	        # ── Executables and scripts ───────────────────────────────────────────────────────────
979	        '*.exe' '*.scr' '*.com'             # Windows executables
980	        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
981	        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
982	        '*.sh'                              # shell scripts in media folders = suspicious
983
984	        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
985	        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
986	        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
987	        '*.part'                            # partial download files
988
989	        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
990	        '*sync-conflict*'                   # Syncthing conflict copies
991
992	        # ── Media-specific extras ─────────────────────────────────────────────────────────────
993	        '*.iso'                             # disc images — extracted content is in library
994	        '*.lrc'                             # lyrics files — handled by music apps not Emby
995	    )
996
997	# ━━━ Arr Cleanup ━━━
998	# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
999	# Per-host URLs, API keys, and path maps live in host*.conf.
1000	# detect_hosts() selects correct host vars at runtime.
1001	#
1002	# API versions — update MAJOR version here when script is updated for a new arr version:
1003	#   Sonarr v4  → /api/v3/series  → /api/v3/episodefile?seriesId=X
1004	#   Radarr v6  → /api/v3/movie   → /api/v3/moviefile?movieId=X
1005	#   Lidarr v3  → /api/v1/artist  → /api/v1/trackFile?artistId=X
1006	    SONARR_VERSION_MAJOR=4
1007	    RADARR_VERSION_MAJOR=6
1008	    LIDARR_VERSION_MAJOR=3
1009
1010	# Lidarr shared settings
1011	    LIDARR_LOCK_WARN_AGE=3600       # 1hr — large libraries take time, not stuck
1012	    LIDARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
1013	    LIDARR_MAX_DELETE_GB=5          # require --i-know-what-im-doing if deletion exceeds this
1014	    LIDARR_MIN_TRACKED_PCT=80       # abort if tracked count drops below this % of last run
1015	                                    # protects against API returning partial data on a bad day
1016	    LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
1017	    LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
1018	    LIDARR_PROTECTED_PATTERNS=(
1019	        # Metadata
1020	        "*.nfo" "*.tbn"
1021	        # Images — album art, artist images, Emby artwork
1022	        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
1023	        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
1024	        "banner.*" "thumb.*" "landscape.*"
1025	        "folder.*" "cover.*" "album.*" "artist.*" "disc.*"
1026	        # Lyrics
1027	        "*.lrc"
1028	    )
1029
1030	# Lidarr artwork fetcher settings
1031	    LIDARR_ART_MIN_SIZE=10000           # bytes — reject downloads smaller than this
1032	    LIDARR_ART_MAX_PARALLEL=4           # concurrent background download jobs
1033	    LIDARR_ART_RETRIES=2                # download retry attempts per image
1034	    LIDARR_ART_SLEEP_BETWEEN=0.2        # seconds between fanart.tv API calls
1035	#   HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf
1036
1037	# Lidarr discovery settings (playback_aware_lidarr_discovery.sh)
1038	    LIDARR_DISCOVERY_THRESHOLD=70         # score to accept candidate (0-100)
1039	    LIDARR_DISCOVERY_LOOKBACK_DAYS=7      # Emby play history window in days
1040	    LIDARR_DISCOVERY_MIN_PLAYS=3          # min plays in window before evaluating an artist
1041	    LIDARR_DISCOVERY_USER_CAP_PCT=35      # max % any single user can contribute to play score (prevents one listener dominating)
1042	    LIDARR_DISCOVERY_MAX_ADDS=5           # max artists to add per run — quality over bulk
1043	    LIDARR_DISCOVERY_REJECT_COOLDOWN=30   # days before re-evaluating a rejected artist
1044	    LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db"
1045
1046	# Sonarr discovery settings (playback_aware_sonarr_discovery.sh)
1047	    SONARR_DISCOVERY_THRESHOLD=52         # score to accept candidate (0-100)
1048	    SONARR_DISCOVERY_LOOKBACK_DAYS=14     # Emby episode play history window in days (shorter than movies — TV watched more frequently)
1049	    SONARR_DISCOVERY_MAX_SEEDS=5          # max seed series from Stage 1
1050	    SONARR_DISCOVERY_MAX_ADDS=3           # max shows to add per run — TV is a larger commitment than movies
1051	    SONARR_DISCOVERY_MIN_VOTE_COUNT=50    # min TMDB votes (TV has fewer votes than movies at same popularity)
1052	    SONARR_DISCOVERY_MIN_RATING=65        # min TMDB vote_average × 10 (65 = 6.5/10)
1053	    SONARR_DISCOVERY_REJECT_COOLDOWN=60   # days before re-evaluating a rejected show
1054	    SONARR_DISCOVERY_USER_EPISODE_CAP=8   # max episodes any one user contributes to seed volume score
1055	    SONARR_DISCOVERY_MONITOR_MODE="all"    # Sonarr monitor mode on add: all | future | first | latest | none
1056	    SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db"
1057
1058	# Radarr discovery shared settings
1059	    RADARR_DISCOVERY_THRESHOLD=52         # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap
1060	    RADARR_DISCOVERY_LOOKBACK_DAYS=30     # Emby watch history window in days (movies rewatched less often)
1061	    RADARR_DISCOVERY_MAX_SEEDS=5          # max seed movies from Stage 1
1062	    RADARR_DISCOVERY_MAX_ADDS=5           # max movies to add per run
1063	    RADARR_DISCOVERY_MIN_VOTE_COUNT=100   # min TMDB votes to be considered a candidate
1064	    RADARR_DISCOVERY_MIN_RATING=60        # min TMDB vote_average × 10 (60 = 6.0/10)
1065	    RADARR_DISCOVERY_REJECT_COOLDOWN=60   # days before re-evaluating a rejected movie
1066	    RADARR_DISCOVERY_SEED_LIBRARIES=("Movies")  # Emby libraries to draw seed movies from
1067	    RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db"
1068
1069	# Emby → arr sync library allowlists
1070	# Only these Emby library names will be considered by the sync tools.
1071	# Names must match exactly as shown in Emby > Dashboard > Libraries.
1072	    SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows")
1073	    RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies")
1074
1075	# Sonarr shared settings
1076	    SONARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
1077	    SONARR_MAX_DELETE_GB=10         # require --i-know-what-im-doing if deletion exceeds this
1078	    SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
1079	    SONARR_PROTECTED_PATTERNS=(
1080	        # Subtitles
1081	        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
1082	        # Metadata
1083	        "*.nfo" "*.tbn"
1084	        # Images
1085	        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
1086	        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
1087	        "banner.*" "thumb.*" "landscape.*"
1088	        # Kodi/Emby extras — not tracked by Sonarr API
1089	        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
1090	        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
1091	        "*-clip.*" "*-other.*"
1092	        # Theme songs — stored in show folder, not tracked by arr
1093	        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
1094	    )
1095
1096	# Radarr shared settings
1097	    RADARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
1098	    RADARR_MAX_DELETE_GB=15         # require --i-know-what-im-doing if deletion exceeds this
1099	    RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
1100	    RADARR_PROTECTED_PATTERNS=(
1101	        # Subtitles
1102	        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
1103	        # Metadata
1104	        "*.nfo" "*.tbn"
1105	        # Images
1106	        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
1107	        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
1108	        "banner.*" "thumb.*" "landscape.*"
1109	        # Kodi/Emby extras — not tracked by Radarr API
1110	        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
1111	        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
1112	        "*-clip.*" "*-other.*"
1113	        # Theme songs — stored in movie folder, not tracked by arr
1114	        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
1115	    )
1116
1117	# Radarr/Sonarr TMDb/TVDB removed entry cleanup
1118	    RADARR_DROPPED_ADD_EXCLUSION=true   # add removed movies to import exclusion list
1119	    SONARR_DROPPED_ADD_EXCLUSION=true   # add removed series to import exclusion list
1120
1121	# ━━━ Arr Failed/Stalled Recovery ━━━
1122	# Auto blocklist + re-search failed imports and stalled downloads.
1123	# Runs every 6 hours — schedule: 0 */6 * * *
1124	#
1125	# Targets four problem types:
1126	#   importFailed   — downloaded but arr couldn't import
1127	#   importPending  — downloaded, stuck waiting to import (won't self-resolve)
1128	#   error status   — serious failure not covered above
1129	#   stalled        — download stuck with no connections or progress
1130	#
1131	# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
1132	# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf.
1133	    ARR_IMPORT_RECOVERY_AGE=6           # hours — skip items newer than this
1134	                                        # matches cron interval — items eligible after one missed cycle
1135
1136	# ==============================================================================================
1137	# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
1138	# ==============================================================================================
1139	# Session-based storage allocator using filesystem symlink indirection.
1140	# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected.
1141	# Symlink flips between ramdisk and SSD are transparent to active streams.
1142	#
1143	# ⚠ Docker mount — must use shared propagation so symlink flips work inside container:
1144	#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
1145	#   Standard path mappings use rprivate — symlink changes are NOT visible inside container.
1146
1147	# ━━━ Transcode Manager ━━━
1148	# tmpfs mount point — created at array start by ramdisk_setup.sh.
1149	# Must exist before Emby starts so the symlink resolves correctly.
1150	    RAMDISK_PATH="/mnt/ramdisk_transcodes"
1151
1152	# Ramdisk size and flip thresholds — defined per host in host*.conf.
1153	# All three are coupled — if size changes, thresholds must change with it.
1154	#   HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE
1155	#   HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB  ← flip to SSD at this usage
1156	#   HOST1_RAMDISK_LOW_GB  / HOST2_RAMDISK_LOW_GB   ← flip back to ramdisk at this usage
1157
1158	# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state.
1159	# Emby resolves the symlink once per session at start — symlink flips are transparent.
1160	# Must match the container path configured in Emby's Extra Parameters.
1161	    TRANSCODE_LINK="/mnt/ram-transcode"
1162
1163	# SSD fallback location — defined per host in host*.conf (cache path differs per server):
1164	#   HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD
1165
1166	# Minimum free GB on SSD before allowing flip from ramdisk to SSD.
1167	    RAMDISK_SSD_MIN_GB=20
1168
1169	    TRANSCODE_MAX_AGE=20            # minutes — HLS segment age before cleanup eligibility
1170	    TRANSCODE_ORPHAN_AGE=30         # minutes — files with no matching active session
1171	    TRANSCODE_FLIP_WARN=3           # notify if symlink flips this many times in one hour
1172
1173	    TRANSCODE_OWNER="nobody:users"
1174	    TRANSCODE_CHMOD="755"
1175
1176	# Operating mode — controls symlink direction behaviour.
1177	#   smart   — auto-flips between ramdisk and SSD based on thresholds (default)
1178	#   ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds
1179	#   ssd     — always uses SSD, never flips to ramdisk
1180	    TRANSCODE_MANAGER_MODE="smart"
1181
1182	# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
1183	    TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db"
1184	    TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
1185	    TRANSCODE_LOG_RETENTION=90      # days before old entries purged
1186
1187	    TRANSCODE_CHECK_EMBY=true
1188
1189	# ━━━ Transcode Server Array ━━━
1190	# All media servers sharing the ramdisk transcode space.
1191	# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
1192	# Entries with placeholder API keys are skipped automatically.
1193	# ⚠ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
1194	# Defined per host in host*.conf — Emby container names and keys differ per server:
1195	#   HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS
1196
1197	# ==============================================================================================
1198	# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
1199	# ==============================================================================================
1200
1201	# ━━━ Certificate Monitor ━━━
1202	# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
1203	# Checks the actual certificate served by each domain, not what NPM thinks it has.
1204	# Domains defined per host in host*.conf — each server monitors its own domains:
1205	#   HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS
1206	    CERT_WARN_DAYS=30           # warn when cert expires within this many days
1207	    CERT_CRIT_DAYS=7            # critical alert within this many days
1208	    CERT_TIMEOUT=10             # seconds per domain before giving up
1209
1210	# ━━━ Backup Verify ━━━
1211	# Verifies rsync mirror health by comparing random file checksums between servers.
1212	# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
1213	# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically:
1214	#   HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
1215	    BACKUP_VERIFY_SAMPLE=10         # random files to check per share
1216	    BACKUP_VERIFY_MIN_SIZE=1M       # minimum file size to include in sample
1217
1218	# ━━━ SMART Health ━━━
1219	# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
1220	# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime
1221	# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found.
1222	    SMART_TEMP_WARN=45              # fallback — Celsius warn threshold
1223	    SMART_TEMP_CRIT=55              # fallback — Celsius critical threshold
1224	# Drives to ignore defined per host in host*.conf — hardware is server-specific:
1225	#   HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES
1226
1227	# ━━━ ZFS Memory Snapshot ━━━
1228	# Weekly ZFS pool health and memory diagnostic report — informational only.
1229	    ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log"
1230	    ZFS_REPORT_ARC_WARN_PCT=90      # warn if ARC using more than this % of its max
1231	    ZFS_REPORT_FREE_WARN_GB=10      # warn if less than this GB free RAM
1232	    ZFS_REPORT_AVAIL_WARN_GB=20     # warn if less than this GB available on ZFS pool
1233	    ZFS_REPORT_DOCKER_TOP=10        # how many top Docker containers to show by memory
1234	# Pool ignore list defined per host in host*.conf — pool names are server-specific:
1235	#   HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS
1236
1237	# ━━━ Bandwidth Monitor ━━━
1238	# Called automatically by rsync.sh after each sync — one bounded write per run.
1239	# Tracks transfer size, duration and profile per sync for weekly summary reporting.
1240	    BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
1241	    BANDWIDTH_LOG_RETENTION=90      # days before old entries purged
1242	    BANDWIDTH_WARN_GB=50            # flag syncs larger than this in weekly report
1243
1244	# Stats files — written by cleanup and recovery scripts, read by coffee report.
1245	# All in DATA_DIR — array always running when these are written.
1246	    ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db"     # lidarr/sonarr/radarr orphan stats
1247	    ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db"   # blocklist + re-search stats
1248
1249	# ━━━ Health Digest ━━━
1250	# Aggregated system health summary — reads existing state files, no new writes.
1251	# Three profiles control when the digest is sent:
1252	#   always  — sends every run regardless of findings
1253	#   smart   — sends only when DIGEST_SMART_ON_* conditions are found
1254	#   weekly  — sends once per week on DIGEST_DAY only
1255	    DIGEST_PROFILE="weekly"         # always | smart | weekly
1256	    DIGEST_DAY="Sunday"
1257	    DIGEST_SMART_ON_WATCHDOG=true   # send if any watchdog strikes are active
1258	    DIGEST_SMART_ON_FALLBACK=true   # send if fallback state is not NORMAL
1259	    DIGEST_SMART_ON_CERT_WARN=true  # send if any cert is under CERT_WARN_DAYS
1260	    DIGEST_SMART_ON_BANDWIDTH=true  # send if any transfer exceeded BANDWIDTH_WARN_GB
1261
1262	# ━━━ Emby Session Report ━━━
1263	# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
1264	# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf.
1265	    EMBY_REPORT_DAYS=7              # days to include in the report period
1266	    EMBY_REPORT_TOP_N=10            # number of top content items to show
1267
1268	# ==============================================================================================
1269	# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
1270	# ==============================================================================================
1271	# Pressure reduction layer — keeps the system comfortable before things break.
1272	# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop.
1273	#
1274	# ── PRESSURE LEVELS ───────────────────────────────────────────────────────────────────────────
1275	#   Level 1 (soft)   — throttle SABnzbd + qBit download speeds
1276	#   Level 2 (medium) — further throttle + docker pause background containers
1277	#   Level 3 (hard)   — docker stop optional containers, signal docker_watchdog to defer
1278	#
1279	# ── PER-HOST CONTAINER LISTS ──────────────────────────────────────────────────────────────────
1280	#   HOST*_RW_PAUSE_CONTAINERS  — docker pause at medium pressure (in host*.conf)
1281	#   HOST*_RW_STOP_CONTAINERS   — docker stop at hard pressure (in host*.conf)
1282
1283	    RW_ENABLED=true
1284	    RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db"
1285
1286	# ━━━ Pressure Thresholds ━━━
1287	# Graduated RAM response — resource_watchdog acts before system_watchdog reboots.
1288	# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always
1289	    RW_RAM_SOFT_GB=12           # throttle start — reduce background load
1290	    RW_RAM_MEDIUM_GB=8          # pause background containers
1291	    RW_RAM_HARD_GB=6            # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB)
1292	    RW_RAM_RECOVER_GB=20        # RAM must reach this before restoring hard-stopped containers
1293
1294	# Load average thresholds — multiplier × core count
1295	    RW_LOAD_SOFT_MULTIPLIER=2.0     # soft pressure: 2× cores sustained
1296	    RW_LOAD_MEDIUM_MULTIPLIER=3.0   # medium pressure: 3× cores sustained
1297
1298	# Consecutive runs at lower pressure before de-escalating
1299	    RW_RECOVER_CYCLES=3
1300
1301	# ━━━ SABnzbd Throttle ━━━
1302	# Speed values: "50M" = 50 MB/s, "0" = unlimited
1303	    RW_SABNZBD_ENABLED=true
1304	    RW_SABNZBD_SPEED_SOFT="50M"
1305	    RW_SABNZBD_SPEED_MEDIUM="10M"
1306
1307	# ━━━ qBittorrent Throttle ━━━
1308	# KB/s — 0 = unlimited
1309	    RW_QBIT_ENABLED=true
1310	    RW_QBIT_DL_SOFT=51200       # 50 MB/s
1311	    RW_QBIT_DL_MEDIUM=10240     # 10 MB/s
1312
1313	# ━━━ Critical Containers ━━━
1314	# Never paused or stopped regardless of pressure level.
1315	# Keep DNS, auth, media serving, and live TV always running.
1316	    RW_CRITICAL_CONTAINERS=(
1317	        "NginxProxyManager"             # reverse proxy — internet access
1318	        "Authelia"                      # auth — nothing accessible without it
1319	        "Authelia-Secondary"
1320	        "Mariadb-Authelia"              # Authelia dependency
1321	        "Mariadb-Authelia-Secondary"
1322	        "Redis-Authelia"                # Authelia dependency
1323	        "Redis-Authelia-Secondary"
1324	        "AdGuard-Home"                  # DNS — all LAN resolution
1325	        "Emby"                          # media server — Live TV buffering
1326	        "Dispatcharr"                   # Live TV scheduler — loses state if stopped
1327	        "Dispatcharr-Basic"
1328	        "Dispatcharr-Iptv-Users"
1329	    )
1330
1331	# ==============================================================================================
1332	# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
1333	# ==============================================================================================
1334	# Single-pass system health check — last line of defense before a crash.
1335	# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh.
1336	# Re-sources config at each orchestrator run — config changes take effect immediately.
1337	#
1338	# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
1339	#   CRITICAL  — bypass ALL strikes, reboot immediately
1340	#               Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only
1341	#
1342	#   URGENT    — bypass strikes only when OOM confirms active crisis
1343	#               RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW
1344	#               Without OOM confirmation → normal strike system
1345	#
1346	#   STANDARD  — strike system (N consecutive failures → reboot)
1347	#               RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
1348	#
1349	# ── RAM ───────────────────────────────────────────────────────────────────────────────────────
1350	#   SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass)
1351	#   Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh
1352
1353	# ━━━ State Files ━━━
1354	    SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
1355	    DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
1356	    SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
1357	    SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"
1358
1359	# ━━━ Strike and Reboot Loop Settings ━━━
1360	# Strike system: a check must fail this many consecutive cycles before action is taken.
1361	# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
1362	    SYS_WATCHDOG_STRIKE_LIMIT=2         # consecutive failures before reboot trigger
1363
1364	# Reboot loop protection — if system keeps rebooting something is seriously wrong.
1365	# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot.
1366	    SYS_WATCHDOG_REBOOT_LIMIT=3
1367	    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
1368
1369	# ━━━ RAM Reboot Threshold ━━━
1370	# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh
1371	# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots)
1372	    SYS_WATCHDOG_MEM_GB=4              # strike system → reboot
1373
1374	# ━━━ OOM Bypass Settings ━━━
1375	# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
1376	# in a single cycle → bypass strike system and reboot immediately.
1377	# Rate-based: kills per 5-minute cycle, not absolute count.
1378	# Rationale: 1-2 kills = docker_watchdog handles it ✅
1379	#            3+ kills while RAM critical = system dying faster than watchdogs can heal ✅
1380	    SYS_WATCHDOG_OOM_LIMIT=3           # OOM kills in one cycle to trigger bypass
1381
1382	# ━━━ Thresholds ━━━
1383	# Set at "about to become unstable" — not "things are a bit high".
1384
1385	# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99%
1386	    SYS_WATCHDOG_ROOTFS_PCT=95         # standard strike threshold
1387	    SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing
1388
1389	# /var/log usage — log spam indicates something broken
1390	    SYS_WATCHDOG_LOG_PCT=95
1391
1392	# /tmp usage — tmpfs fills from downloads, lock files fail at 100%
1393	    SYS_WATCHDOG_TMP_PCT=90            # warn + attempt clear
1394	    SYS_WATCHDOG_TMP_CRITICAL_PCT=98   # bypass strikes if clear failed
1395
1396	# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
1397	    SYS_WATCHDOG_ARC_PINNED_PCT=98
1398	    SYS_WATCHDOG_ARC_RELEASE_PCT=95
1399
1400	# Load average multiplier — threshold = MULTIPLIER × CPU core count
1401	# MULTIPLIER=3 on 16-core = load of 48 before triggering
1402	    SYS_WATCHDOG_LOAD_MULTIPLIER=3
1403
1404	# Zombie process count — large numbers = serious process management failure
1405	    SYS_WATCHDOG_ZOMBIE_LIMIT=50
1406
1407	# CPU temperature — sustained high temp causes throttling or kernel panic
1408	    SYS_WATCHDOG_CPU_TEMP_MAX=95
1409
1410	# File descriptor exhaustion — system-wide FD limit near exhaustion
1411	# New connections fail silently, Docker can't spawn processes, SSH fails
1412	    SYS_WATCHDOG_FD_CRITICAL_PCT=95    # bypass strikes — critical tier
1413
1414	# Runaway process — single non-container process consuming excessive CPU
1415	# Multiple strikes before action — single spikes are normal
1416	    SYS_WATCHDOG_RUNAWAY_CPU_PCT=90    # % single process must sustain
1417	    SYS_WATCHDOG_RUNAWAY_STRIKES=3     # consecutive cycles before warning
1418
1419	# Array disk errors — accumulating mdstat errors = disk failing NOW
1420	    SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5  # new errors in one cycle before acting
1421
1422	# ━━━ Check Toggles ━━━
1423	# Per-host — moved to host*.conf
1424	# Different servers may have different hardware, NICs, and check requirements
1425	# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf
1426
1427	# ━━━ Abort Toggles ━━━
1428	# Conditions that prevent reboot even when a threshold is hit.
1429	# CRITICAL tier bypasses these — a truly critical condition reboots regardless.
1430	# true  = abort standard reboot if this condition is active
1431	# false = reboot anyway
1432	    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true    # rebooting with bad pool risks data loss
1433	    SYS_WATCHDOG_ABORT_ON_PARITY=false           # aborting parity better than crashing mid-check
1434	    SYS_WATCHDOG_ABORT_ON_MOVER=false            # aborting move better than crashing mid-move
1435
1436	# ==============================================================================================
1437	# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
1438	# ==============================================================================================
1439

#!/bin/bash
# ==============================================================================================
# ================================= MASTER CONFIGURATION =======================================
# ==============================================================================================
# Shared configuration for the unRAID script ecosystem.
# Contains all settings that apply to every server — thresholds, toggles, profiles, job lists.
#
# ── HOW THE THREE-FILE SYSTEM WORKS ──────────────────────────────────────────────────────────
# Scripts source all three files at startup:
#   source master.conf          ← shared config (this file)
#   source host1.conf    ← HOST1 credentials, shares, container lists
#   source host2.conf    ← HOST2 credentials, shares, container lists
#
# Sparse checkout (git) ensures each server only pulls its own host*.conf.
# HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials.
#
# What belongs here:      thresholds, toggles, intervals, profiles, job lists
# What belongs in HOST*:  hostnames, SSH keys, API keys, passwords, share paths,
#                         container names, failover lists, watchdog containers
#
# ── HOW THIS FILE WORKS ───────────────────────────────────────────────────────────────────────
# Every script sources all three conf files and common.sh at startup.
# Change a value here and it affects all scripts that use it — no hunting through files.
# To disable something: comment it out with # rather than deleting it.
# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section.
# To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section.
#
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
#
#   Section                         Description
#   ───────────────────────────────────────────────────────────────────────────────────────────
#   SHARED HOST CONFIGURATION       DATA_DIR and shared connection settings
#   PARTNERSHIP                     Mirror relationship lifecycle — onboard/offboard/transfer
#   LOGGING                         Enable or disable verbose logging
#   NOTIFICATIONS                   unRAID native and Discord webhook settings
#   GIT / REPO                      Gitea repository and SSH settings
#
#   ── ORCHESTRATORS ──────────────────────────────────────────────────────────────────────────
#   ARRAY START                     Scripts launched at array start (array_started.sh)
#   ARRAY STOP                      Scripts run at planned shutdown (array_stopping.sh)
#   WATCHDOG ORCHESTRATOR           Per-minute watchdog runner (watchdog_orchestrator.sh)
#   SYSTEM WATCHDOG                 Sub-scripts called by watchdog_orchestrator.sh
#   CRITICAL SYNC MAINTENANCE       30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh)
#   INTERMEDIATE SYNC MAINTENANCE   Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh)
#   DAILY SYNC MAINTENANCE          Job list + media shares (daily_sync_maintenance.sh)
#   WEEKLY SYNC MAINTENANCE         Job list + sync shares + update toggles (weekly_sync_maintenance.sh)
#   MONTHLY MAINTENANCE             Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh)
#
#   ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
#   RSYNC ENABLE/DISABLE            Two-tier toggle system — global gate + per-orchestrator
#   RSYNC DEFAULTS                  Global fallback rsync options and limits
#   REMOTE HEALTH CHECKS            Rootfs threshold for pre-flight abort
#   RSYNC PROFILE SYSTEM            Per-profile overrides for appdata syncs
#
#   ── FALLBACK ───────────────────────────────────────────────────────────────────────────────
#   FALLBACK                        Mutual container failover shared settings
#   FALLBACK TEST                   Simulated outage settings for fallback_test.sh
#
#   ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   DOWNLOADERS RESET               Retention and thresholds for slskd, SABnzbd, qBittorrent
#   DOCKER DAILY RESTART            Containers restarted daily
#   DOCKER WEEKLY RESTART           Containers restarted weekly
#   DOCKER WATCHDOG                 Thresholds and toggles for container monitoring
#   DOCKER NETWORK CONNECT          Networks to ensure + containers to connect
#
#   ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
#   INOTIFY TUNING                  inotify limits — raised at array start by inotify_tuning.sh
#   SYSTEM TUNING MONITOR           Tracks inotify + php-fpm usage over time
#   REBOOT                          User warning delay before scheduled reboot
#   MOVER                           Mover stop timeout
#   SYSLOG FILTER                   Docker veth noise filter file path
#   PHP-FPM                         PHP-FPM max children config
#   CLEAR LOGS                      System log file paths
#   WEBGUI WATCHDOG                 WebGUI nginx + emhttp monitoring and restart
#
#   ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
#   MEDIA PERMISSIONS               Share list, mode and owner for permissions script
#   MEDIA CLEANER                   Anime and media folder lists and file patterns
#   ARR CLEANUP                     Lidarr, Sonarr, Radarr orphan file cleanup settings
#   ARR FAILED/STALLED RECOVERY     Auto blocklist + re-search settings
#
#   ── TRANSCODES ─────────────────────────────────────────────────────────────────────────────
#   TRANSCODE MANAGER               Ramdisk and SSD fallback transcode management
#   TRANSCODE SERVER ARRAY          Multi-server session monitoring (Emby, Jellyfin, Plex)
#
#   ── MONITORS ───────────────────────────────────────────────────────────────────────────────
#   CERTIFICATE MONITOR             SSL certificate expiry monitoring
#   BACKUP VERIFY                   Random sample checksum verification against remote
#   SMART HEALTH                    Drive SMART attribute monitoring
#   ZFS MEMORY SNAPSHOT             Weekly ZFS health and memory diagnostic report
#   BANDWIDTH MONITOR               Daily rsync transfer logging and weekly summary
#   HEALTH DIGEST                   Aggregated system health digest — always/smart/weekly
#   EMBY SESSION REPORT             Weekly Emby usage statistics via API
#
#   ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
#   SYSTEM WATCHDOG                 Continuous system health monitoring — last line of defense
#
# ==============================================================================================

# ==============================================================================================
# ── HOST IDENTITIES ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Hostnames for every server in the ecosystem — not credentials, safe for all machines.
# Must match the exact unRAID hostname AND Tailscale device name (case sensitive).
# detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID.
# Tailscale IP resolution uses these names — no hardcoded IPs needed.
# To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf.
    HOST1="unRAID-Gmer4Lfe"
    HOST2="unRAID-Jayred365"

# ==============================================================================================
# ── SHARED HOST CONFIGURATION ─────────────────────────────────────────────────────────────────
# ==============================================================================================
# Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash —
# scripts, state, and data are all available before the array mounts.
# Both directories are created automatically if they don't exist.
#
#   DATA_DIR   — historical logs, statistics, discovery histories, blocklists
#   STATE_DIR  — runtime state files for all scripts (watchdogs, fallback, transcode, etc.)
#                Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root.
    DATA_DIR="/boot/config/plugins/varaverk/data"
    STATE_DIR="/boot/config/plugins/varaverk/State_Files"

# ── Version Parity ──
# Controls behaviour when local and remote unRAID versions differ.
# Major version mismatch always aborts regardless of this setting.
# "warn"  — log warning and continue (safe for minor/patch differences)
# "abort" — refuse to continue (strict — ensures both sides always match)
    UNRAID_VERSION_MISMATCH_ACTION="warn"

# ── Arr Sync ──
# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
# Runs before rsync — all nodes agree on tracked library before files are transferred.
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
    ARR_SYNC_ENABLED=true
    ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
    ARR_SYNC_CONNECT_TIMEOUT=10     # seconds — SSH connect timeout per node
    ARR_SYNC_API_TIMEOUT=60         # seconds — curl timeout for library fetches
    DOCKER_APPDATA_BASE="/mnt/user/appdata"
    ARR_SYNC_LIDARR_PORT=8686
    ARR_SYNC_SONARR_PORT=8989
    ARR_SYNC_RADARR_PORT=7878

# ── Remote Docker Daemon ──
# Strike system for remote Docker daemon health checks.
# Scripts that issue remote container commands check the remote daemon first.
# Below limit → skip operation this run | At limit → notify critical + exit
    REMOTE_DOCKER_STRIKE_LIMIT=3       # consecutive failures before critical notify

# ==============================================================================================
# ── PARTNERSHIP ───────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Manages the relationship lifecycle between two unRAID servers.
# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
# All identity vars (hostnames, SSH keys) live in host*.conf.
# Hostnames already match Tailscale device names — IP resolution is automatic.
#
# State files in STATE_DIR — survives reboots, array must be up (scripts run after array start):
#   STATE_DIR/partnership_HOST1.db ← HOST1 writes only
#   STATE_DIR/partnership_HOST2.db ← HOST2 writes only
#   STATE_DIR/partnership_blocklist.db ← shared blocklist
#   STATE_DIR/varaverk_setup.db ← onboarding progress state
#   Propagated via SSH — no rsync needed
#
# critical_sync_maintenance.sh runs --check every 30min:
#   Reads both state files via SSH
#   Detects offboard requests → finalises from owner side
#   Increments offline counter → auto-offboards after threshold
#   Silent when healthy ✅
#
# See README-Partnership.md for full lifecycle documentation.

    PARTNERSHIP_ENABLED=false
    PARTNERSHIP_OWNER_HOST="HOST1"          # "HOST1" or "HOST2" — flips on --transfer

# Auth containers reconfigured on onboard/offboard — defined per host in host*.conf.
# Format: "ContainerName|WebUIPort"
# HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS
# On onboard → WebUI pointed at owner's Tailscale IP
# On offboard → WebUI pointed back at localhost

# Paths to collect during the grace window after offboard — defined per host in host*.conf.
# HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS
# Notified on offboard — no auto-deletion, manual collection.

# Timing — single var controls both Tailscale removal and backup access expiry.
# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless.
    PARTNERSHIP_GRACE_HOURS=6               # hours after offboard before Tailscale removal
                                            # backup access expires at the same time
    PARTNERSHIP_OFFLINE_THRESHOLD=30        # days either server unreachable before auto-offboard
                                            # works both directions independently

# Partnership and setup state files — all in STATE_DIR per the project requirement.
# Scripts use these variables; do not hardcode /boot/config paths in scripts.
    PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db"
    VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"

# Tailscale removal on offboard.
    PARTNERSHIP_REMOVE_TAILSCALE=true       # remove mirror from Tailscale tailnet on offboard
                                            # false = skip removal (manual or testing)

# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
# Stays in shared conf — only owner uses it, and owner is always running this script.
# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope
    TAILSCALE_API_KEY=""                    # tskey-api-...
    TAILSCALE_TAILNET=""                    # your tailnet name (e.g. yourname.github)

# Transfer safety.
    PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
    PARTNERSHIP_TRANSFER_STRIKES=3          # consecutive health checks required
    PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20    # max health check attempts before giving up

# Onboard settings.
    PARTNERSHIP_ONBOARD_VERIFY=true         # verify WebUI reachable after reconfiguration
    PARTNERSHIP_ONBOARD_NOTIFY=true         # notify both servers on completion
    PARTNERSHIP_SYNC_INTERVAL=15            # minutes — informational, actual schedule in cron

# SSH key lifecycle — managed by Partnership/ssh_setup.sh.
# Key named after this server: hostname lowercased, unraid- prefix stripped.
#   unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation
# Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically.
    SSH_MAX_STRIKES=5                       # consecutive SSH auth failures before critical notify
    SSH_STRIKE_RESET_HRS=24                 # hours since last failure before strike counter resets

# FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato.
# Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard.
# Folder name derived from remote hostname (strip unraid- prefix case-insensitively).
# Plugin config: /boot/config/plugins/folder.view3/docker.json
    PARTNERSHIP_FOLDERVIEW3=true            # create/remove FolderView3 folder on onboard/offboard
    PARTNERSHIP_FOLDERVIEW3_URL=""          # CA plugin URL — leave empty to skip auto-install

# ==============================================================================================
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Controls verbose [LOG] output across all scripts.
# true  = show detailed [LOG] lines — useful for debugging or first-time setup
# false = show only user-facing output — cleaner for scheduled runs
    ENABLE_LOGGING=false

# ==============================================================================================
# ── NOTIFICATIONS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# unRAID native notification system — integrates with the bell icon in the WebGUI.
# normal = job completed successfully / warning = something failed or needs attention
    NOTIFY_UNRAID=true

# Discord webhook URL — defined per host in host*.conf.
# HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK
# Allows different webhooks per server, or only one server notifying.

# ==============================================================================================
# ── GIT / REPO ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# Gitea self-hosted repository — used by git_pull_execute.sh.
# Detects Gitea container location at runtime — works through failover automatically.
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
    GITEA_CONTAINER="Gitea"
    GITEA_REPO_PATH="FailedProxy/Varaverk.git"
    GITEA_DOMAIN=""                         # e.g. git.yourdomain.com — requires NPM + DNS
    TARGET_DIR="/boot/config/plugins/varaverk"
    GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
    SSH_PORT=221                            # Gitea SSH port (default 22, Gitea often uses 221/222)
    GITEA_HTTP_PORT=3000                    # Gitea web/API port — used by gitea_ssh_setup.sh

# ==============================================================================================
# ── ORCHESTRATORS ─────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# All orchestrator job lists live here — edit arrays to add/remove scripts.
# No changes to orchestrator scripts needed when adding or removing jobs.
# Sections ordered by run frequency: array events first, then shortest interval to longest.

# ━━━ Array Start ━━━
# Scripts launched by array_started.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
# Continuous scripts (failover) run until array stops.
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
# watchdog_orchestrator.sh — NOT launched here.
    ARRAY_START_SCRIPTS=(
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
        "Fallback/fallback.sh"                        # mutual failover — continuous
    )

# ━━━ Array Stop ━━━
# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
# Run sequentially (foreground) — each must complete before the next starts.
# Order matters: user scripts first (prevents new ops), then data movement, then containers.
    ARRAY_STOP_SCRIPTS=(
        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last
    )

# ━━━ Watchdog Orchestrator ━━━
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: * * * * * (every minute)
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
# Order matters — resource first (frees pressure), docker second (heals with freed resources),
# system third (storage + webgui component health), stability last (last line of defense).
    WATCHDOG_ORCHESTRATOR_SCRIPTS=(
        "Watchdogs/resource_watchdog.sh"              # reduce system pressure before healing attempts
        "Watchdogs/docker_watchdog.sh"                # heal containers with freed resources
        "Watchdogs/system_watchdog.sh"                # system component health — storage + webgui
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register API key if registry lost it
        "Watchdogs/stability_watchdog.sh"             # reboot if all else fails — last line of defense
    )

# ━━━ System Watchdog ━━━
# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
# Called by watchdog_orchestrator.sh — not scheduled directly.
    SYSTEM_WATCHDOG_SCRIPTS=(
        "Watchdogs/System/storage_watchdog.sh"   # pool growth + runaway log detection
        "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh"    # WebGUI availability — nginx → php-fpm → emhttp
        "Watchdogs/System/network_watchdog.sh"   # internet, DDNS, Tailscale, NPM proxy
    )

# ━━━ Critical Sync Maintenance ━━━
# critical_sync_maintenance.sh runs every 30 minutes.
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
# partnership --check always runs last regardless of rsync gate.
# Comment out entries to disable without removing.
    CRITICAL_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/downloaders_reset.sh"        # clear stuck download states every 30min
        "Media/play_state_sync.sh"                      # sync watched/resume state across Emby + Jellyfin
    )

# Shares synced every 30 minutes — defined per host in host*.conf.
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
# Format: "/path/to/share" or "/path/to/share|profile-name"
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.

# ━━━ Intermediate Sync Maintenance ━━━
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
# and optional mid-day rsync for any shares that need sub-daily propagation.
# Schedule: 0 */4 * * *
    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs media share sync first, then iterates
# DAILY_MAINTENANCE_SCRIPTS for all jobs.
# Schedule: 0 1 * * * (1am daily)
    DAILY_MAINTENANCE_SCRIPTS=(
        "git_pull_execute.sh"                         # pull latest scripts — always runs first
        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
        "Media/media_cleaner.sh anime"                # remove junk from anime shares
        "Media/media_cleaner.sh media"                # remove junk from media shares
        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
    )

# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
# Containers keep running during pull — no extra downtime.
# Set false to skip updates while still running the daily restart.
    DAILY_CONTAINER_UPDATES=true

# Media shares synced daily by daily_sync_maintenance.sh.
# Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES.
# Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive.
# arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans.
# Any node can download content to any share — it propagates to all nodes on the next cycle.
# Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track.
# These shares use DEFAULT_RSYNC_OPTS — no profile entry needed.
# For shares needing custom options or container stops — create a profile in RSYNC section.

# Personal encrypted shares defined per-host in host*.conf.
# ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content.
# See README-Rsync_Setup.md for ZFS encryption setup before uncommenting.

# ━━━ Weekly Sync Maintenance ━━━
# weekly_sync_maintenance.sh stops containers both sides → pulls updates →
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
# Schedule: 30 2 * * 0 (Sunday 2:30am)
    WEEKLY_MAINTENANCE_SCRIPTS=(
        "Docker_Essentials/docker_weekly_restart.sh"    # weekly container restarts after sync
        "Docker_Essentials/docker_update_remaining.sh"  # pull updates for all other containers
        "System_Essentials/clear_logs.sh"               # purge aged logs — Sunday only, low priority
        "Media/playback_aware_lidarr_discovery.sh"      # behavior-driven music discovery using weekly Emby playback history
        "Media/playback_aware_radarr_discovery.sh"      # behavior-driven movie discovery using TMDB recommendations
        "Media/playback_aware_sonarr_discovery.sh"      # behavior-driven TV discovery using TMDB recommendations
    )

# Pull updates for all running containers NOT in daily/weekly restart lists.
# Ensures every deployed container receives at least one image pull per week.
# Set false to skip — docker_weekly_restart.sh still runs regardless.
    WEEKLY_REMAINING_UPDATES=true

# Shares synced during the weekly maintenance window — defined per host in host*.conf.
# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
# Containers stopped both sides before sync — full clean state guaranteed.
# Profiles drive container stops, excludes, and options — configure in RSYNC section.
# Order matters — Emby first (larger), then Critical-Data (auth stack).

# Container update toggles for the weekly sync window.
# Containers already stopped for sync — updates pull at no extra downtime.
# Both false → sync only, no updates.
    WEEKLY_SYNC_UPDATES=true          # pull container updates locally during weekly window
    WEEKLY_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH

# ━━━ Monthly Maintenance ━━━
# monthly_maintenance.sh fires only when BOTH gates pass:
#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe)
# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry.
# Add scripts that require a long-stable settled system — scrubs, extended drive tests.
# State file on /boot/config — survives reboots (interval gate independent of uptime gate).
    MONTHLY_MAINTENANCE_SCRIPTS=(
        "Tools/zfs_pool_scrub.sh"           # ZFS pool integrity scrub — all pools, skip ZFS_REPORT_IGNORE_POOLS
        "Tools/smart_long_test.sh"          # SMART extended self-test — all drives, sequential
    )

    MONTHLY_UPTIME_THRESHOLD_DAYS=30    # minimum uptime in days before maintenance fires
    MONTHLY_RUN_INTERVAL_DAYS=30        # minimum days since last run before running again
    MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db"

# ━━━ Sunday Morning Coffee Report ━━━
# Orchestrator that runs all Sunday monitor scripts in sequence.
# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am)
# Each script runs independently and notifies on its own findings.
    COFFEE_REPORT_SCRIPTS=(
        "Monitors/zfs_memory_snapshot.sh"       # ZFS pool health + ARC + Docker memory snapshot
        "Monitors/smart_health.sh"              # drive SMART attributes — reallocated, pending, temp
        "Monitors/cert_monitor.sh"              # SSL certificate expiry for all configured domains
        "Monitors/backup_verify.sh"             # rsync mirror integrity via independent MD5 checksums
        "Monitors/bandwidth_monitor.sh"         # weekly rsync transfer totals and per-share breakdown
        "Monitors/emby_session_report.sh"       # Emby usage — streams, users, library, transcode ratio
        "Monitors/weekly_health_digest.sh"      # aggregated digest — watchdogs, fallback, skip list
    )

# ==============================================================================================
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Rsync Enable/Disable ━━━
# Two-tier toggle system — Tier 1 overrides Tier 2.
#
# Tier 1 — Global gate:
#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
#   Use when: remote completely offline, major maintenance, disaster recovery
#
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
#   Fine grained control — disable specific orchestrators while keeping others
#   Use when: rebuilding secondary, testing, per-window bandwidth management
#
# Example — HOST2 data rebuild:
#   RSYNC_ENABLED=true                ← rsync works, individual scripts run fine
#   CRITICAL_RSYNC_ENABLED=true       ← 30min auth stack sync still runs
#   INTERMEDIATE_RSYNC_ENABLED=false  ← skip 4h arr/mid-day rsync during rebuild
#   DAILY_RSYNC_ENABLED=false         ← skip daily HDD syncs during rebuild
#   WEEKLY_RSYNC_ENABLED=true         ← Emby + Critical-Data still sync (NVMe)
#   FALLBACK_RSYNC_ENABLED=true       ← handback writeback still works
#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
#   → When ready: INTERMEDIATE_RSYNC_ENABLED=true  DAILY_RSYNC_ENABLED=true
    RSYNC_ENABLED=true                  # Tier 1 — global gate, overrides everything below
    CRITICAL_RSYNC_ENABLED=true        # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
    INTERMEDIATE_RSYNC_ENABLED=true     # Tier 2 — intermediate_sync_maintenance.sh rsync section
    DAILY_RSYNC_ENABLED=false           # Tier 2 — HOST2 rebuild in progress, re-enable when ready
    WEEKLY_RSYNC_ENABLED=true           # Tier 2 — weekly_sync_maintenance.sh rsync section
    FALLBACK_RSYNC_ENABLED=true         # Tier 2 — fallback.sh writeback jobs on handback

# ━━━ Rsync Defaults ━━━
# Global fallback values used when no profile match is found.
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
# Appdata shares match profiles by directory basename (lowercased).
    BW_LIMIT=12500                  # KB/s — 12500 ≈ 100Mbit
    RETRY_COUNT=3                   # retry attempts before giving up
    SLEEP=300                       # seconds between retry attempts
    CRITICAL_CONTAINER_NAMES=()     # containers stopped on REMOTE before rsync — profiles override
    DELAYED_CONTAINERS=()           # containers needing delay before starting — profiles override
    CONTAINER_DELAY=5               # seconds before starting delayed containers
    EXCLUDE_DIRS=()                 # directories excluded from transfer — profiles override

# --inplace      writes directly to destination — delta against existing file, better for large media
# --partial      keep partial file on interrupted transfer so next run resumes, not re-transfers
# --timeout      kill stalled transfers instead of hanging indefinitely
# --numeric-ids  use UIDs/GIDs numerically — prevents ownership mismatches between servers
# --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync.
# Media shares use this default; rsync spreads files only, never removes them.
# Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default).
# Note: arr_cleanup only catches true orphans under the union model — intentional removals
#       require arr_sync.sh --blocklist-add first, then manual file deletion.
    DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids)

# ━━━ Remote Health Checks ━━━
# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage.
# When remote array is down, rsync writes land on rootfs and fill it rapidly.
    ROOTFS_WARN=75

# ━━━ Rsync Profile System ━━━
# Profiles allow per-share rsync behaviour without touching script logic.
# Profile key matched by basename of directory passed to rsync.sh (lowercased).
# Override with --profile=name flag.
#
# IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS.
#            List ALL desired options explicitly when defining a profile.
#
# Current profiles:
#   arrs_stack        — arr databases — lower bandwidth, containers stopped for consistency
#   critical-data     — auth stack — full stop both sides, Authelia delayed start
#                       called by weekly_sync_maintenance.sh — full clean sync weekly
#   critical-fallback — dirty sync — auth stays running both sides, WAL excluded
#                       called by critical_sync_maintenance.sh every 30min
#   host1-appdata     — HOST1 server-specific appdata — defined in host1.conf
#   host2-appdata     — HOST2 server-specific appdata — defined in host2.conf
#   important-data    — NextCloud + Postgres — NextCloud delayed start after Postgres
#   emby              — weekly clean sync — both Emby stopped, full mirror
#                       called by weekly_sync_maintenance.sh only — do NOT schedule separately
#   emby-fallback     — dirty sync — Emby stays running, WAL excluded
#                       called by critical_sync_maintenance.sh every 30min

    declare -A PROFILE_RSYNC_OPTS=(
        [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
        [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
        [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
        [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
        [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
        [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
    )

# Per-profile bandwidth limits in KB/s
    declare -A PROFILE_BW_LIMIT=(
        [arrs_stack]=5000           # lower — runs alongside other syncs
        [critical-data]=9500        # high — small dataset, sync fast
        [critical-fallback]=9500    # high — small dataset, sync fast
        [important-data]=9500       # high — database sync
        [emby]=8000                 # medium — large full mirror
        [emby-fallback]=9500        # high — small critical dataset
    )

# Retry attempts per profile
    declare -A PROFILE_RETRY_COUNT=(
        [arrs_stack]=3
        [critical-data]=3
        [critical-fallback]=3
        [important-data]=3
        [emby]=3
        [emby-fallback]=3
    )

# Seconds between retry attempts
    declare -A PROFILE_SLEEP=(
        [arrs_stack]=300
        [critical-data]=300
        [critical-fallback]=120     # shorter — frequent dirty sync, retry faster
        [important-data]=300
        [emby]=300
        [emby-fallback]=120         # shorter — frequent dirty sync, retry faster
    )

# Containers stopped on BOTH LOCAL and REMOTE before rsync.
# Local stops first — flushes databases cleanly. Remote stops next — prevents writes.
# Only running containers get restarted — stopped containers stay stopped.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
        [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
        [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary"
        [critical-fallback]=""      # dirty sync — auth stays running both sides
        [important-data]="Postgres-NextCloud NextCloud"
        [emby]="Emby"
        [emby-fallback]=""          # dirty sync — Emby stays running both sides
    )

# Containers needing a delay after rsync before starting.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_DELAYED_CONTAINERS=(
        [arrs_stack]=""
        [critical-data]="Authelia Authelia-Secondary"   # wait for Mariadb + Redis
        [critical-fallback]=""
        [important-data]="NextCloud"                    # wait for Postgres
        [emby]=""
        [emby-fallback]=""
    )

# Seconds before starting delayed containers
    declare -A PROFILE_CONTAINER_DELAY=(
        [arrs_stack]=5
        [critical-data]=15          # Mariadb + Redis need time to accept connections
        [critical-fallback]=5
        [important-data]=10         # Postgres needs time before NextCloud
        [emby]=5
        [emby-fallback]=5
    )

# Directories excluded from rsync per profile.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_EXCLUDE_DIRS=(
        [arrs_stack]="logs *.tmp"
        [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
        [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
        [important-data]="logs *.tmp"
        [emby]="logs transcodes cache crash*"
        [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
    )

# Remote restart after dirty sync — restart these on remote IF they were running before sync.
# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted.
# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes.
# SPACE-SEPARATED STRINGS — converted to array at runtime
    declare -A PROFILE_REMOTE_RESTART_CONTAINERS=(
        [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary"
        [emby-fallback]="Emby"
    )

# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed.
# check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically.

# ==============================================================================================
# ── FALLBACK ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Mutual container failover between two unRAID servers.
# Each server runs Fallback/fallback.sh independently via array_started.sh.
# All decisions based on two pings: remote reachable + internet reachable.
#
# States: NORMAL | FALLBACK | NO_INTERNET | DARK
#
# DDNS rules — absolute:
#   Internet loss → stop own DDNS immediately
#   Failover → start remote DDNS first (Tier 1)
#   Handback → stop remote DDNS → rsync → start containers → start local DDNS last
#
# Per-host container lists and tier delays live in host*.conf.
# Shared settings (intervals, state file, thresholds) live here.

    EXTERNAL_IP="8.8.8.8"
    FALLBACK_CHECK_INTERVAL=30      # seconds between fallback state checks
    FALLBACK_HANDBACK_STRIKES=3     # consecutive healthy checks before initiating handback (3×30s = 90s)
    FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
    FALLBACK_ENABLED=true           # HOST2 back online
                                    # false = suppresses "not running" warnings in status scripts
    FALLBACK_PARTNERSHIP_REQUIRED=true          # gate fallback on an active partnership
                                                # false = standalone mode, no partnership dependency
    FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120      # minutes without active partnership before suspending
                                                # 0 = suspend immediately when partnership goes inactive

# ━━━ Failover Test ━━━
# Controlled simulation of a failover event — run manually via fallback_test.sh.
    FALLBACK_TEST_BLOCK_WAIT=150    # seconds to wait after blocking connectivity
    FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback

# ==============================================================================================
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Downloaders Reset ━━━
# Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS.
# Clears stuck states, purges old history, prepares each download client for a clean cycle.
# Per-host URLs and API keys live in host*.conf.
    DOWNLOADER_RETENTION_DAYS=7     # days — purge history older than this

# qBittorrent failsafe — removes torrents older than threshold regardless of ratio
# deleteFiles=false — removes from qBit but leaves files on disk (arr manages files)
    QBIT_FAILSAFE_MIN_DAYS=180      # days — minimum age before failsafe deletion
    QBIT_FAILSAFE_MIN_RATIO=0       # 0 = age only, no ratio requirement

# ━━━ Docker Daily Restart ━━━
# Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS.
# Per-host lists live in host*.conf:
#   HOST1_DAILY_RESTART_CONTAINERS
#   HOST2_DAILY_RESTART_CONTAINERS
# detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime.

# ━━━ Docker Weekly Restart ━━━
# Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am).
# Containers already stopped for weekly sync — restart adds zero extra downtime.
# Per-host lists live in host*.conf:
#   HOST1_WEEKLY_RESTART_CONTAINERS
#   HOST2_WEEKLY_RESTART_CONTAINERS

# ━━━ Docker Watchdog ━━━
# Continuous two-tier self-healing container monitoring.
# Started by array_started.sh — runs until array stops.
# Re-sources all three conf files each cycle — add/remove containers without restarting watchdog.
#
# Tier 1 — strict monitoring of explicitly configured containers:
#   Memory hard limits    — immediate restart if exceeded
#   CPU thresholds        — strike system, restart after CPU_FAIL_LIMIT sustained strikes
#   HTTP responsiveness   — strike system, restart after RESP_FAIL_LIMIT failed checks
#   Required containers   — must always be running, strike + skip list with auto-clear
#
# Tier 2 — global health scan of ALL running containers:
#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
#   OOM killed            — kernel killed → restart + notify
#   Crash loop detection  — RestartCount climbing → notify, critical above limit
#   Dead containers       — remove and restart
#   Unexpected exits      — non-zero exit code → restart
#
# All per-host container lists live in host*.conf:
#   HOST*_WATCHDOG_CONTAINERS         — memory hard limits per container
#   HOST*_WATCHDOG_CONTAINER_URLS     — HTTP health check URLs
#   HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
#   HOST*_WATCHDOG_SCAN_IGNORE        — skip in Tier 2 scan
#   HOST*_WATCHDOG_DEPENDENCIES       — dependency ordering for restart decisions

# Strike state file — persistent in STATE_DIR
    WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db"

# CPU thresholds — normalised against total core count at runtime
    SOFT_CPU_THRESHOLD=80           # warn at this % of total system CPU
    HARD_CPU_THRESHOLD=85           # strike at this % of total system CPU
    CPU_FAIL_LIMIT=2                # consecutive hard CPU strikes before container restart

# Memory soft threshold — warn when container reaches this % of its hard limit
    SOFT_MEM_THRESHOLD=80

# HTTP responsiveness
    RESP_FAIL_LIMIT=2               # consecutive failed checks before restart
    CURL_TIMEOUT=5                  # seconds per check before timeout

# Tier 2 master toggle
    WATCHDOG_SCAN_ALL=true          # false = only WATCHDOG_CONTAINERS + required containers

# Individual Tier 2 check toggles
    WATCHDOG_RESTART_UNHEALTHY=true
    WATCHDOG_RESTART_DEAD=true
    WATCHDOG_RESTART_CRASHED=true
    WATCHDOG_NOTIFY_OOM=true
    WATCHDOG_NOTIFY_CRASHLOOP=true

# Crash loop threshold
    WATCHDOG_CRASH_LIMIT=5          # RestartCount above this = critical crash loop

# Startup grace period — skip restarts while system is still booting
    WATCHDOG_STARTUP_GRACE=600      # seconds after boot before watchdog acts on failures

# Restart loop protection — prevents watchdog from endlessly restarting a broken container
    WATCHDOG_CONTAINER_RESTART_LIMIT=3
    WATCHDOG_CONTAINER_RESTART_WINDOW=1     # rolling window in hours
    WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"

# Notification batching — one summary per cycle instead of one ping per event
    WATCHDOG_BATCH_NOTIFY=true

# Appdata size monitoring — two-part catch-all for runaway growth and oversized log files.
#
# Part 1 — Growth rate (zero-config):
#   Reads per-container dir totals each cycle via du, compares to previous cycle.
#   Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan
#   inside that container. No per-container config required — new containers are covered
#   automatically. Baseline built on first run after boot; growth detection starts cycle 2.
#
# Part 2 — Absolute log size:
#   Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths.
#   Catches logs that have already stabilised at a large size and are no longer actively growing.
#
# Strike system (reuses existing watchdog infrastructure):
#   Strike 1 — warn + notify: condition first detected
#   Strike 2 — warn + escalated notify: still present next cycle
#   Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
#     If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes
#     If false: critical notify only, strikes held until condition resolves
#   Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
#
# HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a
#   container until its dir exceeds the configured ceiling. Only needed when a container
#   legitimately has large stable data and you want to guarantee it never triggers a false alarm.
    WATCHDOG_CHECK_APPDATA=true
    WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata")
    WATCHDOG_APPDATA_GROWTH_GB=2            # flag containers growing more than this per cycle
    WATCHDOG_APPDATA_LOG_MAX_GB=2           # flag *.log files exceeding this size (absolute)
    WATCHDOG_APPDATA_TRUNCATE_LOGS=false    # set true to auto-truncate oversized *.log files on action cycle
    WATCHDOG_APPDATA_STRIKE_LIMIT=3         # cycles before action fires (matches existing watchdog pattern)
    WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db"
    STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db"

# ━━━ Docker Network Connect ━━━
# Ensures custom networks exist and connects containers at array start.
# Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run.
# Container and network lists are host-specific — defined in host*.conf:
#   HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS
#   HOST1_NETWORK_CONNECT_NETWORKS   / HOST2_NETWORK_CONNECT_NETWORKS

# ==============================================================================================
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ inotify Tuning ━━━
# Linux inotify limits — applied at every array start by inotify_tuning.sh.
# Default unRAID values are very low — with many Docker containers watching files
# you can silently exhaust the limit causing containers to miss file events.
# Symptoms: containers miss file events, downloads not detected, library not updated.
# Live TV stutter is a known symptom of inotify exhaustion.
# These settings are lost on reboot — reapplied automatically at array start.
    INOTIFY_MAX_INSTANCES=1024      # default: 128   — max inotify instances per user
                                    # 1024 handles ~20-30 containers watching files simultaneously
    INOTIFY_MAX_WATCHES=1048576     # default: 8192  — SHARED budget across ALL users/containers
                                    # 524288 (512K) was previous value — raised to 1M (1048576)
                                    # VSCode/Code-Server alone needs ~50K-200K for large workspaces
                                    # with node_modules. All arr containers + Emby + VSCode share
                                    # this budget. 1M is safe on 128GB RAM (~128MB kernel memory)
                                    # If VSCode shows "unable to watch for file changes" → too low
    INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping

# ━━━ System Tuning Monitor ━━━
# Tracks inotify and php-fpm usage over time.
# Snapshot written every 6 hours by system_tuning_monitor.sh.
# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
    INOTIFY_WARN_PCT=80             # warn if inotify instances exceed this % of limit
    PHP_FPM_WARN_PCT=80             # warn if php-fpm workers exceed this % of max_children
    TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
    TUNING_LOG_RETENTION=30         # days before old entries are purged

# ━━━ Reboot ━━━
# Seconds of warning broadcast to logged-in users before server_reboot.sh reboots.
# Gives users time to save work — 300s = 5 minutes.
    REBOOT_SLEEP=300

# ━━━ Mover ━━━
# Seconds to wait before mover_stop.sh sends SIGTERM to the mover process.
# Gives mover time to finish current file transfer before being interrupted.
    MOVER_STOP_TIMEOUT=300

# ━━━ Syslog Filter ━━━
# Path for the rsyslog filter file that suppresses Docker veth interface noise.
# Docker creates a new veth interface for each container — generates hundreds of
# log lines per hour that have no diagnostic value. Filter removes them at source.
    FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"

# ━━━ PHP-FPM ━━━
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
# Default is very low — increasing it prevents WebGUI slowdowns under load.
# 250 is safe for servers with 32GB+ RAM.
    PHP_CONF="/etc/php-fpm.d/www.conf"
    PHP_MAX_CHILDREN=250

# ━━━ Clear Logs ━━━
# System log files cleared weekly to prevent rootfs fill over time.
# These grow continuously — without clearing they eventually consume all rootfs space.
# clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am.
#
# Size threshold approach — only clear if log exceeds threshold.
# Avoids destroying useful recent diagnostic context when logs are small.
# LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing)
# LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size
#   Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr)
#   100MB per container × 30 containers = 3GB before clearing kicks in
    LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
    LOG_MIN_SIZE_MB=10          # skip system log if under this size (already small)
    LOG_DOCKER_MAX_MB=100       # clear Docker container log only if over this size (MB)

# ━━━ Network Watchdog ━━━
# Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy.
# Host-specific values (domain, container, NPM URL) live in host*.conf.
    NETWORK_WATCHDOG_ENABLED=true
    NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1"
    NETWORK_WATCHDOG_INTERNET_TIMEOUT=5
    NETWORK_WATCHDOG_CHECK_TAILSCALE=true
    NETWORK_WATCHDOG_NPM_TIMEOUT=10
    NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2
    NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db"

# ━━━ WebGUI Watchdog ━━━
# Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart.
# Separate from docker_watchdog — this monitors the unRAID UI itself, not containers.
    WEBGUI_URL="http://localhost"
    WEBGUI_TIMEOUT=5        # seconds before curl gives up on WebGUI check
    WEBGUI_NGINX_WAIT=15    # seconds after nginx restart before rechecking
    WEBGUI_EMHTTP_WAIT=30   # seconds after emhttp restart before rechecking

# ==============================================================================================
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Media Permissions ━━━
# Applied recursively by media_shares_permissions.sh.
# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership.
#
# Why split directory vs file permissions:
#   Directories need execute bit to enter — 755 allows owner+group+others to traverse
#   Files should NOT be executable — 664 allows owner+group read/write, others read
#   This is POSIX best practice for media servers, not a blanket 777 band-aid
#
# Why nobody:users ownership:
#   All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID
#   Files owned by nobody:users are accessible to all containers without world-write
#   If this script fixes many files each run → a container has wrong PUID/PGID set
#   Fix: add PUID=99 PGID=100 to the container's environment variables
#
# This script runs daily as a failsafe — even with correct container config:
#   rsync may bring files with source ownership if not run with --chown
#   Manual admin copies create root:root files
#   New containers may not have PUID/PGID set yet
    PERMISSIONS_DIR_MODE="755"    # directories — traverse + list, no world-write
    PERMISSIONS_FILE_MODE="664"   # files — group read/write, no execute
    PERMISSIONS_OWNER="nobody:users"

# Share list defined per host in host*.conf:
#   HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES

# ━━━ Media Cleaner ━━━
# Removes junk files from media shares — two profiles: anime and media.
# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media
# Folder lists defined per host in host*.conf:
#   HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS
#   HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS

    ANIME_FILE_PATTERNS=(
        # ── Checksums ────────────────────────────────────────────────────────────────────────
        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download

        # ── Scene / download metadata ─────────────────────────────────────────────────────────
        '*.url' '*.lnk'                     # scene links
        '*.info' '*.diz'                    # scene description files
        '*.nzb'                             # usenet download files
        '*.torrent'                         # torrent files left by download clients

        # ── Archives and segments ─────────────────────────────────────────────────────────────
        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
        '*.r04' '*.r05' '*.r06' '*.r07'
        '*.r08' '*.r09' '*.srr'             # scene repair files
        '*.001' '*.002' '*.003'             # split archive parts
        '*.gz' '*.tar' '*.bz2'             # linux archives

        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
        '*.sample*' '*.proof*'              # scene samples — never needed in library

        # ── Executables and scripts ───────────────────────────────────────────────────────────
        '*.exe' '*.scr' '*.com'             # Windows executables
        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
        '*.sh'                              # shell scripts in media folders = suspicious

        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
        '*.part'                            # partial download files

        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
        '*sync-conflict*'                   # Syncthing conflict copies
    )

    MEDIA_FILE_PATTERNS=(
        # ── Checksums ────────────────────────────────────────────────────────────────────────
        '*.sfv' '*.md5' '*.sha1'            # verification files — useless post-download

        # ── Scene / download metadata ─────────────────────────────────────────────────────────
        '*.url' '*.lnk'                     # scene links
        '*.info' '*.diz'                    # scene description files
        '*.nzb'                             # usenet download files
        '*.torrent'                         # torrent files left by download clients

        # ── Archives and segments ─────────────────────────────────────────────────────────────
        '*.rar' '*.zip' '*.7z' '*.ace'      # archives — source files not needed after extract
        '*.r00' '*.r01' '*.r02' '*.r03'     # multi-part rar segments
        '*.r04' '*.r05' '*.r06' '*.r07'
        '*.r08' '*.r09' '*.srr'             # scene repair files
        '*.001' '*.002' '*.003'             # split archive parts
        '*.gz' '*.tar' '*.bz2'             # linux archives

        # ── Scene samples and proofs ──────────────────────────────────────────────────────────
        '*.sample*' '*.proof*'              # scene samples — never needed in library

        # ── Executables and scripts ───────────────────────────────────────────────────────────
        '*.exe' '*.scr' '*.com'             # Windows executables
        '*.bat' '*.cmd' '*.vbs' '*.ps1'     # Windows scripts
        '*.msi' '*.dll' '*.sys'             # Windows installers and system files
        '*.sh'                              # shell scripts in media folders = suspicious

        # ── Incomplete downloads ──────────────────────────────────────────────────────────────
        '*.!ut' '*.!qB'                     # uTorrent / qBittorrent incomplete markers
        '*.crdownload' '*.opdownload'       # Chrome / Opera incomplete downloads
        '*.part'                            # partial download files

        # ── Sync conflicts ────────────────────────────────────────────────────────────────────
        '*sync-conflict*'                   # Syncthing conflict copies

        # ── Media-specific extras ─────────────────────────────────────────────────────────────
        '*.iso'                             # disc images — extracted content is in library
        '*.lrc'                             # lyrics files — handled by music apps not Emby
    )

# ━━━ Arr Cleanup ━━━
# Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs.
# Per-host URLs, API keys, and path maps live in host*.conf.
# detect_hosts() selects correct host vars at runtime.
#
# API versions — update MAJOR version here when script is updated for a new arr version:
#   Sonarr v4  → /api/v3/series  → /api/v3/episodefile?seriesId=X
#   Radarr v6  → /api/v3/movie   → /api/v3/moviefile?movieId=X
#   Lidarr v3  → /api/v1/artist  → /api/v1/trackFile?artistId=X
    SONARR_VERSION_MAJOR=4
    RADARR_VERSION_MAJOR=6
    LIDARR_VERSION_MAJOR=3

# Lidarr shared settings
    LIDARR_LOCK_WARN_AGE=3600       # 1hr — large libraries take time, not stuck
    LIDARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    LIDARR_MAX_DELETE_GB=5          # require --i-know-what-im-doing if deletion exceeds this
    LIDARR_MIN_TRACKED_PCT=80       # abort if tracked count drops below this % of last run
                                    # protects against API returning partial data on a bad day
    LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
    LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
    LIDARR_PROTECTED_PATTERNS=(
        # Metadata
        "*.nfo" "*.tbn"
        # Images — album art, artist images, Emby artwork
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        "folder.*" "cover.*" "album.*" "artist.*" "disc.*"
        # Lyrics
        "*.lrc"
    )

# Lidarr artwork fetcher settings
    LIDARR_ART_MIN_SIZE=10000           # bytes — reject downloads smaller than this
    LIDARR_ART_MAX_PARALLEL=4           # concurrent background download jobs
    LIDARR_ART_RETRIES=2                # download retry attempts per image
    LIDARR_ART_SLEEP_BETWEEN=0.2        # seconds between fanart.tv API calls
#   HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf

# Lidarr discovery settings (playback_aware_lidarr_discovery.sh)
    LIDARR_DISCOVERY_THRESHOLD=70         # score to accept candidate (0-100)
    LIDARR_DISCOVERY_LOOKBACK_DAYS=7      # Emby play history window in days
    LIDARR_DISCOVERY_MIN_PLAYS=3          # min plays in window before evaluating an artist
    LIDARR_DISCOVERY_USER_CAP_PCT=35      # max % any single user can contribute to play score (prevents one listener dominating)
    LIDARR_DISCOVERY_MAX_ADDS=5           # max artists to add per run — quality over bulk
    LIDARR_DISCOVERY_REJECT_COOLDOWN=30   # days before re-evaluating a rejected artist
    LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db"

# Sonarr discovery settings (playback_aware_sonarr_discovery.sh)
    SONARR_DISCOVERY_THRESHOLD=52         # score to accept candidate (0-100)
    SONARR_DISCOVERY_LOOKBACK_DAYS=14     # Emby episode play history window in days (shorter than movies — TV watched more frequently)
    SONARR_DISCOVERY_MAX_SEEDS=5          # max seed series from Stage 1
    SONARR_DISCOVERY_MAX_ADDS=3           # max shows to add per run — TV is a larger commitment than movies
    SONARR_DISCOVERY_MIN_VOTE_COUNT=50    # min TMDB votes (TV has fewer votes than movies at same popularity)
    SONARR_DISCOVERY_MIN_RATING=65        # min TMDB vote_average × 10 (65 = 6.5/10)
    SONARR_DISCOVERY_REJECT_COOLDOWN=60   # days before re-evaluating a rejected show
    SONARR_DISCOVERY_USER_EPISODE_CAP=8   # max episodes any one user contributes to seed volume score
    SONARR_DISCOVERY_MONITOR_MODE="all"    # Sonarr monitor mode on add: all | future | first | latest | none
    SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db"

# Radarr discovery shared settings
    RADARR_DISCOVERY_THRESHOLD=52         # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap
    RADARR_DISCOVERY_LOOKBACK_DAYS=30     # Emby watch history window in days (movies rewatched less often)
    RADARR_DISCOVERY_MAX_SEEDS=5          # max seed movies from Stage 1
    RADARR_DISCOVERY_MAX_ADDS=5           # max movies to add per run
    RADARR_DISCOVERY_MIN_VOTE_COUNT=100   # min TMDB votes to be considered a candidate
    RADARR_DISCOVERY_MIN_RATING=60        # min TMDB vote_average × 10 (60 = 6.0/10)
    RADARR_DISCOVERY_REJECT_COOLDOWN=60   # days before re-evaluating a rejected movie
    RADARR_DISCOVERY_SEED_LIBRARIES=("Movies")  # Emby libraries to draw seed movies from
    RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db"

# Emby → arr sync library allowlists
# Only these Emby library names will be considered by the sync tools.
# Names must match exactly as shown in Emby > Dashboard > Libraries.
    SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows")
    RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies")

# Sonarr shared settings
    SONARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    SONARR_MAX_DELETE_GB=10         # require --i-know-what-im-doing if deletion exceeds this
    SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
    SONARR_PROTECTED_PATTERNS=(
        # Subtitles
        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
        # Metadata
        "*.nfo" "*.tbn"
        # Images
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        # Kodi/Emby extras — not tracked by Sonarr API
        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
        "*-clip.*" "*-other.*"
        # Theme songs — stored in show folder, not tracked by arr
        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
    )

# Radarr shared settings
    RADARR_ORPHAN_AGE=7             # days — files must be older than this before eligible for deletion
    RADARR_MAX_DELETE_GB=15         # require --i-know-what-im-doing if deletion exceeds this
    RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
    RADARR_PROTECTED_PATTERNS=(
        # Subtitles
        "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt"
        # Metadata
        "*.nfo" "*.tbn"
        # Images
        "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg"
        "poster.*" "fanart.*" "backdrop.*" "clearlogo.*"
        "banner.*" "thumb.*" "landscape.*"
        # Kodi/Emby extras — not tracked by Radarr API
        "*-trailer.*" "*-featurette.*" "*-behindthescenes.*"
        "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*"
        "*-clip.*" "*-other.*"
        # Theme songs — stored in movie folder, not tracked by arr
        "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka"
    )

# Radarr/Sonarr TMDb/TVDB removed entry cleanup
    RADARR_DROPPED_ADD_EXCLUSION=true   # add removed movies to import exclusion list
    SONARR_DROPPED_ADD_EXCLUSION=true   # add removed series to import exclusion list

# ━━━ Arr Failed/Stalled Recovery ━━━
# Auto blocklist + re-search failed imports and stalled downloads.
# Runs every 6 hours — schedule: 0 */6 * * *
#
# Targets four problem types:
#   importFailed   — downloaded but arr couldn't import
#   importPending  — downloaded, stuck waiting to import (won't self-resolve)
#   error status   — serious failure not covered above
#   stalled        — download stuck with no connections or progress
#
# Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first.
# Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf.
    ARR_IMPORT_RECOVERY_AGE=6           # hours — skip items newer than this
                                        # matches cron interval — items eligible after one missed cycle

# ==============================================================================================
# ── TRANSCODES ────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Session-based storage allocator using filesystem symlink indirection.
# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected.
# Symlink flips between ramdisk and SSD are transparent to active streams.
#
# ⚠ Docker mount — must use shared propagation so symlink flips work inside container:
#   --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared
#   Standard path mappings use rprivate — symlink changes are NOT visible inside container.

# ━━━ Transcode Manager ━━━
# tmpfs mount point — created at array start by ramdisk_setup.sh.
# Must exist before Emby starts so the symlink resolves correctly.
    RAMDISK_PATH="/mnt/ramdisk_transcodes"

# Ramdisk size and flip thresholds — defined per host in host*.conf.
# All three are coupled — if size changes, thresholds must change with it.
#   HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE
#   HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB  ← flip to SSD at this usage
#   HOST1_RAMDISK_LOW_GB  / HOST2_RAMDISK_LOW_GB   ← flip back to ramdisk at this usage

# Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state.
# Emby resolves the symlink once per session at start — symlink flips are transparent.
# Must match the container path configured in Emby's Extra Parameters.
    TRANSCODE_LINK="/mnt/ram-transcode"

# SSD fallback location — defined per host in host*.conf (cache path differs per server):
#   HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD

# Minimum free GB on SSD before allowing flip from ramdisk to SSD.
    RAMDISK_SSD_MIN_GB=20

    TRANSCODE_MAX_AGE=20            # minutes — HLS segment age before cleanup eligibility
    TRANSCODE_ORPHAN_AGE=30         # minutes — files with no matching active session
    TRANSCODE_FLIP_WARN=3           # notify if symlink flips this many times in one hour

    TRANSCODE_OWNER="nobody:users"
    TRANSCODE_CHMOD="755"

# Operating mode — controls symlink direction behaviour.
#   smart   — auto-flips between ramdisk and SSD based on thresholds (default)
#   ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds
#   ssd     — always uses SSD, never flips to ramdisk
    TRANSCODE_MANAGER_MODE="smart"

# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
    TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db"
    TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
    TRANSCODE_LOG_RETENTION=90      # days before old entries purged

    TRANSCODE_CHECK_EMBY=true

# ━━━ Transcode Server Array ━━━
# All media servers sharing the ramdisk transcode space.
# Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex
# Entries with placeholder API keys are skipped automatically.
# ⚠ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk.
# Defined per host in host*.conf — Emby container names and keys differ per server:
#   HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS

# ==============================================================================================
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# ━━━ Certificate Monitor ━━━
# Checks SSL certificate expiry via direct openssl connection — no NPM dependency.
# Checks the actual certificate served by each domain, not what NPM thinks it has.
# Domains defined per host in host*.conf — each server monitors its own domains:
#   HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS
    CERT_WARN_DAYS=30           # warn when cert expires within this many days
    CERT_CRIT_DAYS=7            # critical alert within this many days
    CERT_TIMEOUT=10             # seconds per domain before giving up

# ━━━ Backup Verify ━━━
# Verifies rsync mirror health by comparing random file checksums between servers.
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
# Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically:
#   HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
    BACKUP_VERIFY_SAMPLE=10         # random files to check per share
    BACKUP_VERIFY_MIN_SIZE=1M       # minimum file size to include in sample

# ━━━ SMART Health ━━━
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime
# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found.
    SMART_TEMP_WARN=45              # fallback — Celsius warn threshold
    SMART_TEMP_CRIT=55              # fallback — Celsius critical threshold
# Drives to ignore defined per host in host*.conf — hardware is server-specific:
#   HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES

# ━━━ ZFS Memory Snapshot ━━━
# Weekly ZFS pool health and memory diagnostic report — informational only.
    ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log"
    ZFS_REPORT_ARC_WARN_PCT=90      # warn if ARC using more than this % of its max
    ZFS_REPORT_FREE_WARN_GB=10      # warn if less than this GB free RAM
    ZFS_REPORT_AVAIL_WARN_GB=20     # warn if less than this GB available on ZFS pool
    ZFS_REPORT_DOCKER_TOP=10        # how many top Docker containers to show by memory
# Pool ignore list defined per host in host*.conf — pool names are server-specific:
#   HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS

# ━━━ Bandwidth Monitor ━━━
# Called automatically by rsync.sh after each sync — one bounded write per run.
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
    BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
    BANDWIDTH_LOG_RETENTION=90      # days before old entries purged
    BANDWIDTH_WARN_GB=50            # flag syncs larger than this in weekly report

# Stats files — written by cleanup and recovery scripts, read by coffee report.
# All in DATA_DIR — array always running when these are written.
    ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db"     # lidarr/sonarr/radarr orphan stats
    ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db"   # blocklist + re-search stats

# ━━━ Health Digest ━━━
# Aggregated system health summary — reads existing state files, no new writes.
# Three profiles control when the digest is sent:
#   always  — sends every run regardless of findings
#   smart   — sends only when DIGEST_SMART_ON_* conditions are found
#   weekly  — sends once per week on DIGEST_DAY only
    DIGEST_PROFILE="weekly"         # always | smart | weekly
    DIGEST_DAY="Sunday"
    DIGEST_SMART_ON_WATCHDOG=true   # send if any watchdog strikes are active
    DIGEST_SMART_ON_FALLBACK=true   # send if fallback state is not NORMAL
    DIGEST_SMART_ON_CERT_WARN=true  # send if any cert is under CERT_WARN_DAYS
    DIGEST_SMART_ON_BANDWIDTH=true  # send if any transfer exceeded BANDWIDTH_WARN_GB

# ━━━ Emby Session Report ━━━
# Weekly Emby usage statistics via API — no persistent writes, queries fresh each run.
# URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf.
    EMBY_REPORT_DAYS=7              # days to include in the report period
    EMBY_REPORT_TOP_N=10            # number of top content items to show

# ==============================================================================================
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Pressure reduction layer — keeps the system comfortable before things break.
# Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop.
#
# ── PRESSURE LEVELS ───────────────────────────────────────────────────────────────────────────
#   Level 1 (soft)   — throttle SABnzbd + qBit download speeds
#   Level 2 (medium) — further throttle + docker pause background containers
#   Level 3 (hard)   — docker stop optional containers, signal docker_watchdog to defer
#
# ── PER-HOST CONTAINER LISTS ──────────────────────────────────────────────────────────────────
#   HOST*_RW_PAUSE_CONTAINERS  — docker pause at medium pressure (in host*.conf)
#   HOST*_RW_STOP_CONTAINERS   — docker stop at hard pressure (in host*.conf)

    RW_ENABLED=true
    RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db"

# ━━━ Pressure Thresholds ━━━
# Graduated RAM response — resource_watchdog acts before system_watchdog reboots.
# RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always
    RW_RAM_SOFT_GB=12           # throttle start — reduce background load
    RW_RAM_MEDIUM_GB=8          # pause background containers
    RW_RAM_HARD_GB=6            # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB)
    RW_RAM_RECOVER_GB=20        # RAM must reach this before restoring hard-stopped containers

# Load average thresholds — multiplier × core count
    RW_LOAD_SOFT_MULTIPLIER=2.0     # soft pressure: 2× cores sustained
    RW_LOAD_MEDIUM_MULTIPLIER=3.0   # medium pressure: 3× cores sustained

# Consecutive runs at lower pressure before de-escalating
    RW_RECOVER_CYCLES=3

# ━━━ SABnzbd Throttle ━━━
# Speed values: "50M" = 50 MB/s, "0" = unlimited
    RW_SABNZBD_ENABLED=true
    RW_SABNZBD_SPEED_SOFT="50M"
    RW_SABNZBD_SPEED_MEDIUM="10M"

# ━━━ qBittorrent Throttle ━━━
# KB/s — 0 = unlimited
    RW_QBIT_ENABLED=true
    RW_QBIT_DL_SOFT=51200       # 50 MB/s
    RW_QBIT_DL_MEDIUM=10240     # 10 MB/s

# ━━━ Critical Containers ━━━
# Never paused or stopped regardless of pressure level.
# Keep DNS, auth, media serving, and live TV always running.
    RW_CRITICAL_CONTAINERS=(
        "NginxProxyManager"             # reverse proxy — internet access
        "Authelia"                      # auth — nothing accessible without it
        "Authelia-Secondary"
        "Mariadb-Authelia"              # Authelia dependency
        "Mariadb-Authelia-Secondary"
        "Redis-Authelia"                # Authelia dependency
        "Redis-Authelia-Secondary"
        "AdGuard-Home"                  # DNS — all LAN resolution
        "Emby"                          # media server — Live TV buffering
        "Dispatcharr"                   # Live TV scheduler — loses state if stopped
        "Dispatcharr-Basic"
        "Dispatcharr-Iptv-Users"
    )

# ==============================================================================================
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
# ==============================================================================================
# Single-pass system health check — last line of defense before a crash.
# Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh.
# Re-sources config at each orchestrator run — config changes take effect immediately.
#
# ── THREE-TIER RESPONSE SYSTEM ────────────────────────────────────────────────────────────────
#   CRITICAL  — bypass ALL strikes, reboot immediately
#               Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only
#
#   URGENT    — bypass strikes only when OOM confirms active crisis
#               RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW
#               Without OOM confirmation → normal strike system
#
#   STANDARD  — strike system (N consecutive failures → reboot)
#               RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers
#
# ── RAM ───────────────────────────────────────────────────────────────────────────────────────
#   SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass)
#   Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh

# ━━━ State Files ━━━
    SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
    DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
    SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
    SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"

# ━━━ Strike and Reboot Loop Settings ━━━
# Strike system: a check must fail this many consecutive cycles before action is taken.
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
    SYS_WATCHDOG_STRIKE_LIMIT=2         # consecutive failures before reboot trigger

# Reboot loop protection — if system keeps rebooting something is seriously wrong.
# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot.
    SYS_WATCHDOG_REBOOT_LIMIT=3
    SYS_WATCHDOG_REBOOT_WINDOW_HRS=12

# ━━━ RAM Reboot Threshold ━━━
# Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh
# RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots)
    SYS_WATCHDOG_MEM_GB=4              # strike system → reboot

# ━━━ OOM Bypass Settings ━━━
# OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold
# in a single cycle → bypass strike system and reboot immediately.
# Rate-based: kills per 5-minute cycle, not absolute count.
# Rationale: 1-2 kills = docker_watchdog handles it ✅
#            3+ kills while RAM critical = system dying faster than watchdogs can heal ✅
    SYS_WATCHDOG_OOM_LIMIT=3           # OOM kills in one cycle to trigger bypass

# ━━━ Thresholds ━━━
# Set at "about to become unstable" — not "things are a bit high".

# rootfs (/) usage — two levels: strike at 95%, critical bypass at 99%
    SYS_WATCHDOG_ROOTFS_PCT=95         # standard strike threshold
    SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing

# /var/log usage — log spam indicates something broken
    SYS_WATCHDOG_LOG_PCT=95

# /tmp usage — tmpfs fills from downloads, lock files fail at 100%
    SYS_WATCHDOG_TMP_PCT=90            # warn + attempt clear
    SYS_WATCHDOG_TMP_CRITICAL_PCT=98   # bypass strikes if clear failed

# ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck
    SYS_WATCHDOG_ARC_PINNED_PCT=98
    SYS_WATCHDOG_ARC_RELEASE_PCT=95

# Load average multiplier — threshold = MULTIPLIER × CPU core count
# MULTIPLIER=3 on 16-core = load of 48 before triggering
    SYS_WATCHDOG_LOAD_MULTIPLIER=3

# Zombie process count — large numbers = serious process management failure
    SYS_WATCHDOG_ZOMBIE_LIMIT=50

# CPU temperature — sustained high temp causes throttling or kernel panic
    SYS_WATCHDOG_CPU_TEMP_MAX=95

# File descriptor exhaustion — system-wide FD limit near exhaustion
# New connections fail silently, Docker can't spawn processes, SSH fails
    SYS_WATCHDOG_FD_CRITICAL_PCT=95    # bypass strikes — critical tier

# Runaway process — single non-container process consuming excessive CPU
# Multiple strikes before action — single spikes are normal
    SYS_WATCHDOG_RUNAWAY_CPU_PCT=90    # % single process must sustain
    SYS_WATCHDOG_RUNAWAY_STRIKES=3     # consecutive cycles before warning

# Array disk errors — accumulating mdstat errors = disk failing NOW
    SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5  # new errors in one cycle before acting

# ━━━ Check Toggles ━━━
# Per-host — moved to host*.conf
# Different servers may have different hardware, NICs, and check requirements
# See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf

# ━━━ Abort Toggles ━━━
# Conditions that prevent reboot even when a threshold is hit.
# CRITICAL tier bypasses these — a truly critical condition reboots regardless.
# true  = abort standard reboot if this condition is active
# false = reboot anyway
    SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true    # rebooting with bad pool risks data loss
    SYS_WATCHDOG_ABORT_ON_PARITY=false           # aborting parity better than crashing mid-check
    SYS_WATCHDOG_ABORT_ON_MOVER=false            # aborting move better than crashing mid-move

# ==============================================================================================
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
# ==============================================================================================