#!/bin/bash # ============================================================================================== # ================================= MASTER CONFIGURATION ======================================= # ============================================================================================== # Shared configuration for the unRAID script ecosystem. # Contains all settings that apply to every server — thresholds, toggles, profiles, job lists. # # ── HOW THE THREE-FILE SYSTEM WORKS ────────────────────────────────────────────────────────── # Scripts source all three files at startup: # source master.conf ← shared config (this file) # source host1.conf ← HOST1 credentials, shares, container lists # source host2.conf ← HOST2 credentials, shares, container lists # # Sparse checkout (git) ensures each server only pulls its own host*.conf. # HOST2 never sees HOST1 credentials. HOST1 never sees HOST2 credentials. # # What belongs here: thresholds, toggles, intervals, profiles, job lists # What belongs in HOST*: hostnames, SSH keys, API keys, passwords, share paths, # container names, failover lists, watchdog containers # # ── HOW THIS FILE WORKS ─────────────────────────────────────────────────────────────────────── # Every script sources all three conf files and common.sh at startup. # Change a value here and it affects all scripts that use it — no hunting through files. # To disable something: comment it out with # rather than deleting it. # To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section. # To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. # # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── # # Section Description # ─────────────────────────────────────────────────────────────────────────────────────────── # SHARED HOST CONFIGURATION DATA_DIR and shared connection settings # PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer # LOGGING Enable or disable verbose logging # NOTIFICATIONS unRAID native and Discord webhook settings # GIT / REPO Gitea repository and SSH settings # # ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── # ARRAY START Scripts launched at array start (array_started.sh) # ARRAY STOP Scripts run at planned shutdown (array_stopping.sh) # WATCHDOG ORCHESTRATOR Per-minute watchdog runner (watchdog_orchestrator.sh) # SYSTEM WATCHDOG Sub-scripts called by watchdog_orchestrator.sh # CRITICAL SYNC MAINTENANCE 30-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) # INTERMEDIATE SYNC MAINTENANCE Arr sync + optional mid-day rsync (intermediate_sync_maintenance.sh) # DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) # WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) # MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) # # ── RSYNC ────────────────────────────────────────────────────────────────────────────────── # RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator # RSYNC DEFAULTS Global fallback rsync options and limits # REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort # RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs # # ── FALLBACK ─────────────────────────────────────────────────────────────────────────────── # FALLBACK Mutual container failover shared settings # FALLBACK TEST Simulated outage settings for fallback_test.sh # # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── # DOWNLOADERS RESET Retention and thresholds for slskd, SABnzbd, qBittorrent # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly # DOCKER WATCHDOG Thresholds and toggles for container monitoring # DOCKER NETWORK CONNECT Networks to ensure + containers to connect # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── # INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh # SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time # REBOOT User warning delay before scheduled reboot # MOVER Mover stop timeout # SYSLOG FILTER Docker veth noise filter file path # PHP-FPM PHP-FPM max children config # CLEAR LOGS System log file paths # WEBGUI WATCHDOG WebGUI nginx + emhttp monitoring and restart # # ── MEDIA ────────────────────────────────────────────────────────────────────────────────── # MEDIA PERMISSIONS Share list, mode and owner for permissions script # MEDIA CLEANER Anime and media folder lists and file patterns # ARR CLEANUP Lidarr, Sonarr, Radarr orphan file cleanup settings # ARR FAILED/STALLED RECOVERY Auto blocklist + re-search settings # # ── TRANSCODES ───────────────────────────────────────────────────────────────────────────── # TRANSCODE MANAGER Ramdisk and SSD fallback transcode management # TRANSCODE SERVER ARRAY Multi-server session monitoring (Emby, Jellyfin, Plex) # # ── MONITORS ─────────────────────────────────────────────────────────────────────────────── # CERTIFICATE MONITOR SSL certificate expiry monitoring # BACKUP VERIFY Random sample checksum verification against remote # SMART HEALTH Drive SMART attribute monitoring # ZFS MEMORY SNAPSHOT Weekly ZFS health and memory diagnostic report # BANDWIDTH MONITOR Daily rsync transfer logging and weekly summary # HEALTH DIGEST Aggregated system health digest — always/smart/weekly # EMBY SESSION REPORT Weekly Emby usage statistics via API # # ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── # SYSTEM WATCHDOG Continuous system health monitoring — last line of defense # # ============================================================================================== # ============================================================================================== # ── HOST IDENTITIES ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Hostnames for every server in the ecosystem — not credentials, safe for all machines. # Must match the exact unRAID hostname AND Tailscale device name (case sensitive). # detect_hosts() in common.sh matches the local hostname against these to set MY_ID / REMOTE_ID. # Tailscale IP resolution uses these names — no hardcoded IPs needed. # To add a new server: add HOST3="unRAID-NewServer" here + create host3.conf. HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" # ============================================================================================== # ── SHARED HOST CONFIGURATION ───────────────────────────────────────────────────────────────── # ============================================================================================== # Requirement: internal NVMe/SSD boot drive. Everything lives in the plugin folder on flash — # scripts, state, and data are all available before the array mounts. # Both directories are created automatically if they don't exist. # # DATA_DIR — historical logs, statistics, discovery histories, blocklists # STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.) # Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root. DATA_DIR="/boot/config/plugins/varaverk/data" STATE_DIR="/boot/config/plugins/varaverk/State_Files" # ── Version Parity ── # Controls behaviour when local and remote unRAID versions differ. # Major version mismatch always aborts regardless of this setting. # "warn" — log warning and continue (safe for minor/patch differences) # "abort" — refuse to continue (strict — ensures both sides always match) UNRAID_VERSION_MISMATCH_ACTION="warn" # ── Arr Sync ── # arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally. # Runs before rsync — all nodes agree on tracked library before files are transferred. # Remote API keys are read live from each node's config.xml via SSH — never stored here. ARR_SYNC_ENABLED=true ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv" ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches DOCKER_APPDATA_BASE="/mnt/user/appdata" ARR_SYNC_LIDARR_PORT=8686 ARR_SYNC_SONARR_PORT=8989 ARR_SYNC_RADARR_PORT=7878 # ── Remote Docker Daemon ── # Strike system for remote Docker daemon health checks. # Scripts that issue remote container commands check the remote daemon first. # Below limit → skip operation this run | At limit → notify critical + exit REMOTE_DOCKER_STRIKE_LIMIT=3 # consecutive failures before critical notify # ============================================================================================== # ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Manages the relationship lifecycle between two unRAID servers. # HOST1 is always the owner (source of truth) — HOST2 is always the mirror. # PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. # All identity vars (hostnames, SSH keys) live in host*.conf. # Hostnames already match Tailscale device names — IP resolution is automatic. # # State files in STATE_DIR — survives reboots, array must be up (scripts run after array start): # STATE_DIR/partnership_HOST1.db ← HOST1 writes only # STATE_DIR/partnership_HOST2.db ← HOST2 writes only # STATE_DIR/partnership_blocklist.db ← shared blocklist # STATE_DIR/varaverk_setup.db ← onboarding progress state # Propagated via SSH — no rsync needed # # critical_sync_maintenance.sh runs --check every 30min: # Reads both state files via SSH # Detects offboard requests → finalises from owner side # Increments offline counter → auto-offboards after threshold # Silent when healthy ✅ # # See README-Partnership.md for full lifecycle documentation. PARTNERSHIP_ENABLED=false PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer # Auth containers reconfigured on onboard/offboard — defined per host in host*.conf. # Format: "ContainerName|WebUIPort" # HOST1_PARTNERSHIP_AUTH_WEBUIS / HOST2_PARTNERSHIP_AUTH_WEBUIS # On onboard → WebUI pointed at owner's Tailscale IP # On offboard → WebUI pointed back at localhost # Paths to collect during the grace window after offboard — defined per host in host*.conf. # HOST1_PARTNERSHIP_MIRROR_BACKUPS / HOST2_PARTNERSHIP_MIRROR_BACKUPS # Notified on offboard — no auto-deletion, manual collection. # Timing — single var controls both Tailscale removal and backup access expiry. # Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless. PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal # backup access expires at the same time PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard # works both directions independently # Partnership and setup state files — all in STATE_DIR per the project requirement. # Scripts use these variables; do not hardcode /boot/config paths in scripts. PARTNERSHIP_BLOCKLIST_FILE="$STATE_DIR/partnership_blocklist.db" VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db" # Tailscale removal on offboard. PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard # false = skip removal (manual or testing) # Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true. # Stays in shared conf — only owner uses it, and owner is always running this script. # API key: https://login.tailscale.com/admin/settings/keys → Devices write scope TAILSCALE_API_KEY="" # tskey-api-... TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github) # Transfer safety. PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up # Onboard settings. PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron # SSH key lifecycle — managed by Partnership/ssh_setup.sh. # Key named after this server: hostname lowercased, unraid- prefix stripped. # unRAID-Gmer4Lfe → /root/.ssh/gmer4lfe_rsync_automation # Run Partnership/partnership_onboard.sh to generate, copy, and update conf automatically. SSH_MAX_STRIKES=5 # consecutive SSH auth failures before critical notify SSH_STRIKE_RESET_HRS=24 # hours since last failure before strike counter resets # FolderView3 plugin integration — chodeus/VladoPortos/scolcipitato. # Creates a "{Partner}-Failover" folder on onboard, removes + cleans containers on offboard. # Folder name derived from remote hostname (strip unraid- prefix case-insensitively). # Plugin config: /boot/config/plugins/folder.view3/docker.json PARTNERSHIP_FOLDERVIEW3=true # create/remove FolderView3 folder on onboard/offboard PARTNERSHIP_FOLDERVIEW3_URL="" # CA plugin URL — leave empty to skip auto-install # ============================================================================================== # ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Controls verbose [LOG] output across all scripts. # true = show detailed [LOG] lines — useful for debugging or first-time setup # false = show only user-facing output — cleaner for scheduled runs ENABLE_LOGGING=false # ============================================================================================== # ── NOTIFICATIONS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== # unRAID native notification system — integrates with the bell icon in the WebGUI. # normal = job completed successfully / warning = something failed or needs attention NOTIFY_UNRAID=true # Discord webhook URL — defined per host in host*.conf. # HOST1_DISCORD_WEBHOOK / HOST2_DISCORD_WEBHOOK # Allows different webhooks per server, or only one server notifying. # ============================================================================================== # ── GIT / REPO ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Gitea self-hosted repository — used by git_pull_execute.sh. # Detects Gitea container location at runtime — works through failover automatically. # Falls back to GITEA_DOMAIN if local and Tailscale both fail. GITEA_CONTAINER="Gitea" GITEA_REPO_PATH="FailedProxy/Varaverk.git" GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS TARGET_DIR="/boot/config/plugins/varaverk" GITEA_SSH_KEY="/root/.ssh/unraid_gitea" SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222) GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh # ============================================================================================== # ── ORCHESTRATORS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== # All orchestrator job lists live here — edit arrays to add/remove scripts. # No changes to orchestrator scripts needed when adding or removing jobs. # Sections ordered by run frequency: array events first, then shortest interval to longest. # ━━━ Array Start ━━━ # Scripts launched by array_started.sh when the array comes online. # Launched in order — each as a background process. # One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. # Continuous scripts (failover) run until array stops. # Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via # watchdog_orchestrator.sh — NOT launched here. ARRAY_START_SCRIPTS=( "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts "System_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill "System_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning "System_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "Fallback/fallback.sh" # mutual failover — continuous ) # ━━━ Array Stop ━━━ # Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. # Run sequentially (foreground) — each must complete before the next starts. # Order matters: user scripts first (prevents new ops), then data movement, then containers. ARRAY_STOP_SCRIPTS=( "System_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops "Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop) "System_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below) "System_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files) "Docker_Essentials/docker_container_stop.sh" # stop all containers last ) # ━━━ Watchdog Orchestrator ━━━ # watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # Schedule: * * * * * (every minute) # NOT in ARRAY_START_SCRIPTS — has its own cron entry. # Order matters — resource first (frees pressure), docker second (heals with freed resources), # system third (storage + webgui component health), stability last (last line of defense). WATCHDOG_ORCHESTRATOR_SCRIPTS=( "Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts "Watchdogs/docker_watchdog.sh" # heal containers with freed resources "Watchdogs/system_watchdog.sh" # system component health — storage + webgui "Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense ) # ━━━ System Watchdog ━━━ # system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle. # Called by watchdog_orchestrator.sh — not scheduled directly. SYSTEM_WATCHDOG_SCRIPTS=( "Watchdogs/System/storage_watchdog.sh" # pool growth + runaway log detection "Watchdogs/System/webgui_watchdog.sh" # WebGUI availability — nginx → php-fpm → emhttp "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy ) # ━━━ Critical Sync Maintenance ━━━ # critical_sync_maintenance.sh runs every 30 minutes. # Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check # partnership --check always runs last regardless of rsync gate. # Comment out entries to disable without removing. CRITICAL_MAINTENANCE_SCRIPTS=( "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 30min ) # Shares synced every 30 minutes — defined per host in host*.conf. # HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES # Format: "/path/to/share" or "/path/to/share|profile-name" # Order matters — Critical-Data first (auth stack), then Emby dirty sync. # ━━━ Intermediate Sync Maintenance ━━━ # intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, # and optional mid-day rsync for any shares that need sub-daily propagation. # Schedule: 0 */4 * * * # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf. INTERMEDIATE_MAINTENANCE_SCRIPTS=( "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items ) # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). # ━━━ Daily Sync Maintenance ━━━ # daily_sync_maintenance.sh runs media share sync first, then iterates # DAILY_MAINTENANCE_SCRIPTS for all jobs. # Schedule: 0 1 * * * (1am daily) DAILY_MAINTENANCE_SCRIPTS=( "git_pull_execute.sh" # pull latest scripts — always runs first "Media/media_shares_permissions.sh" # apply permissions — runs before cleaners "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready "Media/radarr_cleanup.sh" # remove orphaned movie files "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB "Docker_Essentials/docker_update.sh" # pull container image updates before restart "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last ) # Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart. # Containers keep running during pull — no extra downtime. # Set false to skip updates while still running the daily restart. DAILY_CONTAINER_UPDATES=true # Media shares synced daily by daily_sync_maintenance.sh. # Defined per-host in host*.conf — HOST1_DAILY_SYNC_SHARES and HOST2_DAILY_SYNC_SHARES. # Mesh model: every node pushes every media share. rsync has no --delete so pushes are additive. # arr_sync (union) ensures all arr libraries converge first. arr_cleanup removes true orphans. # Any node can download content to any share — it propagates to all nodes on the next cycle. # Adding HOST3: list every media share in HOST3_DAILY_SYNC_SHARES. No ownership to track. # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. # For shares needing custom options or container stops — create a profile in RSYNC section. # Personal encrypted shares defined per-host in host*.conf. # ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. # See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. # ━━━ Weekly Sync Maintenance ━━━ # weekly_sync_maintenance.sh stops containers both sides → pulls updates → # syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS. # Schedule: 30 2 * * 0 (Sunday 2:30am) WEEKLY_MAINTENANCE_SCRIPTS=( "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync "Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers "System_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority "Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history "Media/playback_aware_radarr_discovery.sh" # behavior-driven movie discovery using TMDB recommendations "Media/playback_aware_sonarr_discovery.sh" # behavior-driven TV discovery using TMDB recommendations ) # Pull updates for all running containers NOT in daily/weekly restart lists. # Ensures every deployed container receives at least one image pull per week. # Set false to skip — docker_weekly_restart.sh still runs regardless. WEEKLY_REMAINING_UPDATES=true # Shares synced during the weekly maintenance window — defined per host in host*.conf. # HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES # Containers stopped both sides before sync — full clean state guaranteed. # Profiles drive container stops, excludes, and options — configure in RSYNC section. # Order matters — Emby first (larger), then Critical-Data (auth stack). # Container update toggles for the weekly sync window. # Containers already stopped for sync — updates pull at no extra downtime. # Both false → sync only, no updates. WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH # ━━━ Monthly Maintenance ━━━ # monthly_maintenance.sh fires only when BOTH gates pass: # 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days # 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) # Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) # NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. # Add scripts that require a long-stable settled system — scrubs, extended drive tests. # State file on /boot/config — survives reboots (interval gate independent of uptime gate). MONTHLY_MAINTENANCE_SCRIPTS=( #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built ) MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again MONTHLY_LAST_RUN_FILE="$STATE_DIR/monthly_maintenance_last_run.db" # ━━━ Sunday Morning Coffee Report ━━━ # Orchestrator that runs all Sunday monitor scripts in sequence. # Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am) # Each script runs independently and notifies on its own findings. COFFEE_REPORT_SCRIPTS=( "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio "Monitors/weekly_health_digest.sh" # aggregated digest — watchdogs, fallback, skip list ) # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Rsync Enable/Disable ━━━ # Two-tier toggle system — Tier 1 overrides Tier 2. # # Tier 1 — Global gate: # RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions # Use when: remote completely offline, major maintenance, disaster recovery # # Tier 2 — Per-orchestrator (only applies when Tier 1 is true): # Fine grained control — disable specific orchestrators while keeping others # Use when: rebuilding secondary, testing, per-window bandwidth management # # Example — HOST2 data rebuild: # RSYNC_ENABLED=true ← rsync works, individual scripts run fine # CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs # INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild # DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild # WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) # FALLBACK_RSYNC_ENABLED=true ← handback writeback still works # → Run individual: bash Rsync/rsync.sh /mnt/user/Movies # → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below CRITICAL_RSYNC_ENABLED=false # Tier 2 — disabled during HOST2 rebuild, re-enable when ready INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. # Appdata shares match profiles by directory basename (lowercased). BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit RETRY_COUNT=3 # retry attempts before giving up SLEEP=300 # seconds between retry attempts CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override CONTAINER_DELAY=5 # seconds before starting delayed containers EXCLUDE_DIRS=() # directories excluded from transfer — profiles override # --inplace writes directly to destination — delta against existing file, better for large media # --partial keep partial file on interrupted transfer so next run resumes, not re-transfers # --timeout kill stalled transfers instead of hanging indefinitely # --numeric-ids use UIDs/GIDs numerically — prevents ownership mismatches between servers # --delete intentionally omitted — arr_cleanup.sh enforces media truth post-rsync. # Media shares use this default; rsync spreads files only, never removes them. # Note: --no-whole-file removed — redundant over SSH (delta transfer is already the default). # Note: arr_cleanup only catches true orphans under the union model — intentional removals # require arr_sync.sh --blocklist-add first, then manual file deletion. DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --partial --timeout=60 --numeric-ids) # ━━━ Remote Health Checks ━━━ # Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. # When remote array is down, rsync writes land on rootfs and fill it rapidly. ROOTFS_WARN=75 # ━━━ Rsync Profile System ━━━ # Profiles allow per-share rsync behaviour without touching script logic. # Profile key matched by basename of directory passed to rsync.sh (lowercased). # Override with --profile=name flag. # # IMPORTANT: PROFILE_RSYNC_OPTS does NOT inherit DEFAULT_RSYNC_OPTS. # List ALL desired options explicitly when defining a profile. # # Current profiles: # arrs_stack — arr databases — lower bandwidth, containers stopped for consistency # critical-data — auth stack — full stop both sides, Authelia delayed start # called by weekly_sync_maintenance.sh — full clean sync weekly # critical-fallback — dirty sync — auth stays running both sides, WAL excluded # called by critical_sync_maintenance.sh every 30min # host1-appdata — HOST1 server-specific appdata — defined in host1.conf # host2-appdata — HOST2 server-specific appdata — defined in host2.conf # important-data — NextCloud + Postgres — NextCloud delayed start after Postgres # emby — weekly clean sync — both Emby stopped, full mirror # called by weekly_sync_maintenance.sh only — do NOT schedule separately # emby-fallback — dirty sync — Emby stays running, WAL excluded # called by critical_sync_maintenance.sh every 30min declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" [critical-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [emby-fallback]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" ) # Per-profile bandwidth limits in KB/s declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 # lower — runs alongside other syncs [critical-data]=9500 # high — small dataset, sync fast [critical-fallback]=9500 # high — small dataset, sync fast [important-data]=9500 # high — database sync [emby]=8000 # medium — large full mirror [emby-fallback]=9500 # high — small critical dataset ) # Retry attempts per profile declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 [critical-fallback]=3 [important-data]=3 [emby]=3 [emby-fallback]=3 ) # Seconds between retry attempts declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 [critical-fallback]=120 # shorter — frequent dirty sync, retry faster [important-data]=300 [emby]=300 [emby-fallback]=120 # shorter — frequent dirty sync, retry faster ) # Containers stopped on BOTH LOCAL and REMOTE before rsync. # Local stops first — flushes databases cleanly. Remote stops next — prevents writes. # Only running containers get restarted — stopped containers stay stopped. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" [critical-fallback]="" # dirty sync — auth stays running both sides [important-data]="Postgres-NextCloud NextCloud" [emby]="Emby" [emby-fallback]="" # dirty sync — Emby stays running both sides ) # Containers needing a delay after rsync before starting. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis [critical-fallback]="" [important-data]="NextCloud" # wait for Postgres [emby]="" [emby-fallback]="" ) # Seconds before starting delayed containers declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=15 # Mariadb + Redis need time to accept connections [critical-fallback]=5 [important-data]=10 # Postgres needs time before NextCloud [emby]=5 [emby-fallback]=5 ) # Directories excluded from rsync per profile. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" [critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm" [important-data]="logs *.tmp" [emby]="logs transcodes cache crash*" [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" ) # Remote restart after dirty sync — restart these on remote IF they were running before sync. # Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. # Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes. # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" [emby-fallback]="Emby" ) # Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. # check_remote_disks() reads fsType per disk and handles XFS, ZFS, and cache pools automatically. # ============================================================================================== # ── FALLBACK ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Mutual container failover between two unRAID servers. # Each server runs Fallback/fallback.sh independently via array_started.sh. # All decisions based on two pings: remote reachable + internet reachable. # # States: NORMAL | FALLBACK | NO_INTERNET | DARK # # DDNS rules — absolute: # Internet loss → stop own DDNS immediately # Failover → start remote DDNS first (Tier 1) # Handback → stop remote DDNS → rsync → start containers → start local DDNS last # # Per-host container lists and tier delays live in host*.conf. # Shared settings (intervals, state file, thresholds) live here. EXTERNAL_IP="8.8.8.8" FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s) FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" FALLBACK_ENABLED=true # HOST2 back online # false = suppresses "not running" warnings in status scripts FALLBACK_PARTNERSHIP_REQUIRED=true # gate fallback on an active partnership # false = standalone mode, no partnership dependency FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120 # minutes without active partnership before suspending # 0 = suspend immediately when partnership goes inactive # ━━━ Failover Test ━━━ # Controlled simulation of a failover event — run manually via fallback_test.sh. FALLBACK_TEST_BLOCK_WAIT=150 # seconds to wait after blocking connectivity FALLBACK_TEST_HANDBACK_WAIT=360 # seconds to wait before initiating handback # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Downloaders Reset ━━━ # Runs every 30 minutes via CRITICAL_MAINTENANCE_SCRIPTS. # Clears stuck states, purges old history, prepares each download client for a clean cycle. # Per-host URLs and API keys live in host*.conf. DOWNLOADER_RETENTION_DAYS=7 # days — purge history older than this # qBittorrent failsafe — removes torrents older than threshold regardless of ratio # deleteFiles=false — removes from qBit but leaves files on disk (arr manages files) QBIT_FAILSAFE_MIN_DAYS=180 # days — minimum age before failsafe deletion QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement # ━━━ Docker Daily Restart ━━━ # Containers restarted every day via DAILY_MAINTENANCE_SCRIPTS. # Per-host lists live in host*.conf: # HOST1_DAILY_RESTART_CONTAINERS # HOST2_DAILY_RESTART_CONTAINERS # detect_hosts() sets DAILY_RESTART_CONTAINERS to the correct host array at runtime. # ━━━ Docker Weekly Restart ━━━ # Less critical services restarted weekly via WEEKLY_MAINTENANCE_SCRIPTS (Sunday 2:30am). # Containers already stopped for weekly sync — restart adds zero extra downtime. # Per-host lists live in host*.conf: # HOST1_WEEKLY_RESTART_CONTAINERS # HOST2_WEEKLY_RESTART_CONTAINERS # ━━━ Docker Watchdog ━━━ # Continuous two-tier self-healing container monitoring. # Started by array_started.sh — runs until array stops. # Re-sources all three conf files each cycle — add/remove containers without restarting watchdog. # # Tier 1 — strict monitoring of explicitly configured containers: # Memory hard limits — immediate restart if exceeded # CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes # HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks # Required containers — must always be running, strike + skip list with auto-clear # # Tier 2 — global health scan of ALL running containers: # Unhealthy status — Docker HEALTHCHECK unhealthy → restart # OOM killed — kernel killed → restart + notify # Crash loop detection — RestartCount climbing → notify, critical above limit # Dead containers — remove and restart # Unexpected exits — non-zero exit code → restart # # All per-host container lists live in host*.conf: # HOST*_WATCHDOG_CONTAINERS — memory hard limits per container # HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs # HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running # HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan # HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions # Strike state file — persistent in STATE_DIR WATCHDOG_STATE_FILE="$STATE_DIR/container_watchdog_state.db" # CPU thresholds — normalised against total core count at runtime SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart # Memory soft threshold — warn when container reaches this % of its hard limit SOFT_MEM_THRESHOLD=80 # HTTP responsiveness RESP_FAIL_LIMIT=2 # consecutive failed checks before restart CURL_TIMEOUT=5 # seconds per check before timeout # Tier 2 master toggle WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers # Individual Tier 2 check toggles WATCHDOG_RESTART_UNHEALTHY=true WATCHDOG_RESTART_DEAD=true WATCHDOG_RESTART_CRASHED=true WATCHDOG_NOTIFY_OOM=true WATCHDOG_NOTIFY_CRASHLOOP=true # Crash loop threshold WATCHDOG_CRASH_LIMIT=5 # RestartCount above this = critical crash loop # Startup grace period — skip restarts while system is still booting WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures # Restart loop protection — prevents watchdog from endlessly restarting a broken container WATCHDOG_CONTAINER_RESTART_LIMIT=3 WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" # Notification batching — one summary per cycle instead of one ping per event WATCHDOG_BATCH_NOTIFY=true # Appdata size monitoring — two-part catch-all for runaway growth and oversized log files. # # Part 1 — Growth rate (zero-config): # Reads per-container dir totals each cycle via du, compares to previous cycle. # Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused *.log scan # inside that container. No per-container config required — new containers are covered # automatically. Baseline built on first run after boot; growth detection starts cycle 2. # # Part 2 — Absolute log size: # Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB across all appdata paths. # Catches logs that have already stabilised at a large size and are no longer actively growing. # # Strike system (reuses existing watchdog infrastructure): # Strike 1 — warn + notify: condition first detected # Strike 2 — warn + escalated notify: still present next cycle # Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle: # If WATCHDOG_APPDATA_TRUNCATE_LOGS=true: truncate *.log files in-place, clear strikes # If false: critical notify only, strikes held until condition resolves # Condition resolves (growth stops / log drops below threshold) → strikes auto-clear # # HOST*_WATCHDOG_APPDATA_SIZES (in host*.conf) suppresses growth warnings for a # container until its dir exceeds the configured ceiling. Only needed when a container # legitimately has large stable data and you want to guarantee it never triggers a false alarm. WATCHDOG_CHECK_APPDATA=true WATCHDOG_APPDATA_PATHS=("/mnt/docker-unraid/appdata") WATCHDOG_APPDATA_GROWTH_GB=2 # flag containers growing more than this per cycle WATCHDOG_APPDATA_LOG_MAX_GB=2 # flag *.log files exceeding this size (absolute) WATCHDOG_APPDATA_TRUNCATE_LOGS=false # set true to auto-truncate oversized *.log files on action cycle WATCHDOG_APPDATA_STRIKE_LIMIT=3 # cycles before action fires (matches existing watchdog pattern) WATCHDOG_APPDATA_GROWTH_FILE="$STATE_DIR/watchdog_appdata_growth.db" STORAGE_WATCHDOG_STATE_FILE="$STATE_DIR/storage_watchdog_state.db" # ━━━ Docker Network Connect ━━━ # Ensures custom networks exist and connects containers at array start. # Runs once via ARRAY_START_SCRIPTS — idempotent, safe to re-run. # Container and network lists are host-specific — defined in host*.conf: # HOST1_NETWORK_CONNECT_CONTAINERS / HOST2_NETWORK_CONNECT_CONTAINERS # HOST1_NETWORK_CONNECT_NETWORKS / HOST2_NETWORK_CONNECT_NETWORKS # ============================================================================================== # ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ inotify Tuning ━━━ # Linux inotify limits — applied at every array start by inotify_tuning.sh. # Default unRAID values are very low — with many Docker containers watching files # you can silently exhaust the limit causing containers to miss file events. # Symptoms: containers miss file events, downloads not detected, library not updated. # Live TV stutter is a known symptom of inotify exhaustion. # These settings are lost on reboot — reapplied automatically at array start. INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user # 1024 handles ~20-30 containers watching files simultaneously INOTIFY_MAX_WATCHES=1048576 # default: 8192 — SHARED budget across ALL users/containers # 524288 (512K) was previous value — raised to 1M (1048576) # VSCode/Code-Server alone needs ~50K-200K for large workspaces # with node_modules. All arr containers + Emby + VSCode share # this budget. 1M is safe on 128GB RAM (~128MB kernel memory) # If VSCode shows "unable to watch for file changes" → too low INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping # ━━━ System Tuning Monitor ━━━ # Tracks inotify and php-fpm usage over time. # Snapshot written every 6 hours by system_tuning_monitor.sh. # Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary. INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" TUNING_LOG_RETENTION=30 # days before old entries are purged # ━━━ Reboot ━━━ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. # Gives users time to save work — 300s = 5 minutes. REBOOT_SLEEP=300 # ━━━ Mover ━━━ # Seconds to wait before mover_stop.sh sends SIGTERM to the mover process. # Gives mover time to finish current file transfer before being interrupted. MOVER_STOP_TIMEOUT=300 # ━━━ Syslog Filter ━━━ # Path for the rsyslog filter file that suppresses Docker veth interface noise. # Docker creates a new veth interface for each container — generates hundreds of # log lines per hour that have no diagnostic value. Filter removes them at source. FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" # ━━━ PHP-FPM ━━━ # Higher max_children allows more concurrent PHP requests to the unRAID WebGUI. # Default is very low — increasing it prevents WebGUI slowdowns under load. # 250 is safe for servers with 32GB+ RAM. PHP_CONF="/etc/php-fpm.d/www.conf" PHP_MAX_CHILDREN=250 # ━━━ Clear Logs ━━━ # System log files cleared weekly to prevent rootfs fill over time. # These grow continuously — without clearing they eventually consume all rootfs space. # clear_logs.sh runs in WEEKLY_MAINTENANCE_SCRIPTS — Sunday 2:30am. # # Size threshold approach — only clear if log exceeds threshold. # Avoids destroying useful recent diagnostic context when logs are small. # LOG_MIN_SIZE_MB: skip clearing if log is under this size (not worth clearing) # LOG_DOCKER_MAX_MB: clear a container log only if it exceeds this size # Docker logs grow fastest on active containers (Emby, SABnzbd, Sonarr) # 100MB per container × 30 containers = 3GB before clearing kicks in LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) LOG_MIN_SIZE_MB=10 # skip system log if under this size (already small) LOG_DOCKER_MAX_MB=100 # clear Docker container log only if over this size (MB) # ━━━ Network Watchdog ━━━ # Services-layer connectivity — internet reachability, DDNS sync, Tailscale, NPM proxy. # Host-specific values (domain, container, NPM URL) live in host*.conf. NETWORK_WATCHDOG_ENABLED=true NETWORK_WATCHDOG_INTERNET_URL="https://1.1.1.1" NETWORK_WATCHDOG_INTERNET_TIMEOUT=5 NETWORK_WATCHDOG_CHECK_TAILSCALE=true NETWORK_WATCHDOG_NPM_TIMEOUT=10 NETWORK_WATCHDOG_NPM_STRIKE_LIMIT=2 NETWORK_WATCHDOG_NPM_STATE_FILE="$STATE_DIR/network_watchdog_state.db" # ━━━ WebGUI Watchdog ━━━ # Monitors unRAID WebGUI responsiveness — escalates through nginx restart → emhttp restart. # Separate from docker_watchdog — this monitors the unRAID UI itself, not containers. WEBGUI_URL="http://localhost" WEBGUI_TIMEOUT=5 # seconds before curl gives up on WebGUI check WEBGUI_NGINX_WAIT=15 # seconds after nginx restart before rechecking WEBGUI_EMHTTP_WAIT=30 # seconds after emhttp restart before rechecking # ============================================================================================== # ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Media Permissions ━━━ # Applied recursively by media_shares_permissions.sh. # Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. # # Why split directory vs file permissions: # Directories need execute bit to enter — 755 allows owner+group+others to traverse # Files should NOT be executable — 664 allows owner+group read/write, others read # This is POSIX best practice for media servers, not a blanket 777 band-aid # # Why nobody:users ownership: # All ecosystem containers run as PUID=99 (nobody) PGID=100 (users) on unRAID # Files owned by nobody:users are accessible to all containers without world-write # If this script fixes many files each run → a container has wrong PUID/PGID set # Fix: add PUID=99 PGID=100 to the container's environment variables # # This script runs daily as a failsafe — even with correct container config: # rsync may bring files with source ownership if not run with --chown # Manual admin copies create root:root files # New containers may not have PUID/PGID set yet PERMISSIONS_DIR_MODE="755" # directories — traverse + list, no world-write PERMISSIONS_FILE_MODE="664" # files — group read/write, no execute PERMISSIONS_OWNER="nobody:users" # Share list defined per host in host*.conf: # HOST1_MEDIA_PERMISSION_SHARES / HOST2_MEDIA_PERMISSION_SHARES # ━━━ Media Cleaner ━━━ # Removes junk files from media shares — two profiles: anime and media. # Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media # Folder lists defined per host in host*.conf: # HOST1_ANIME_CLEAN_FOLDERS / HOST2_ANIME_CLEAN_FOLDERS # HOST1_MEDIA_CLEAN_FOLDERS / HOST2_MEDIA_CLEAN_FOLDERS ANIME_FILE_PATTERNS=( # ── Checksums ──────────────────────────────────────────────────────────────────────── '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download # ── Scene / download metadata ───────────────────────────────────────────────────────── '*.url' '*.lnk' # scene links '*.info' '*.diz' # scene description files '*.nzb' # usenet download files '*.torrent' # torrent files left by download clients # ── Archives and segments ───────────────────────────────────────────────────────────── '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments '*.r04' '*.r05' '*.r06' '*.r07' '*.r08' '*.r09' '*.srr' # scene repair files '*.001' '*.002' '*.003' # split archive parts '*.gz' '*.tar' '*.bz2' # linux archives # ── Scene samples and proofs ────────────────────────────────────────────────────────── '*.sample*' '*.proof*' # scene samples — never needed in library # ── Executables and scripts ─────────────────────────────────────────────────────────── '*.exe' '*.scr' '*.com' # Windows executables '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts '*.msi' '*.dll' '*.sys' # Windows installers and system files '*.sh' # shell scripts in media folders = suspicious # ── Incomplete downloads ────────────────────────────────────────────────────────────── '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads '*.part' # partial download files # ── Sync conflicts ──────────────────────────────────────────────────────────────────── '*sync-conflict*' # Syncthing conflict copies ) MEDIA_FILE_PATTERNS=( # ── Checksums ──────────────────────────────────────────────────────────────────────── '*.sfv' '*.md5' '*.sha1' # verification files — useless post-download # ── Scene / download metadata ───────────────────────────────────────────────────────── '*.url' '*.lnk' # scene links '*.info' '*.diz' # scene description files '*.nzb' # usenet download files '*.torrent' # torrent files left by download clients # ── Archives and segments ───────────────────────────────────────────────────────────── '*.rar' '*.zip' '*.7z' '*.ace' # archives — source files not needed after extract '*.r00' '*.r01' '*.r02' '*.r03' # multi-part rar segments '*.r04' '*.r05' '*.r06' '*.r07' '*.r08' '*.r09' '*.srr' # scene repair files '*.001' '*.002' '*.003' # split archive parts '*.gz' '*.tar' '*.bz2' # linux archives # ── Scene samples and proofs ────────────────────────────────────────────────────────── '*.sample*' '*.proof*' # scene samples — never needed in library # ── Executables and scripts ─────────────────────────────────────────────────────────── '*.exe' '*.scr' '*.com' # Windows executables '*.bat' '*.cmd' '*.vbs' '*.ps1' # Windows scripts '*.msi' '*.dll' '*.sys' # Windows installers and system files '*.sh' # shell scripts in media folders = suspicious # ── Incomplete downloads ────────────────────────────────────────────────────────────── '*.!ut' '*.!qB' # uTorrent / qBittorrent incomplete markers '*.crdownload' '*.opdownload' # Chrome / Opera incomplete downloads '*.part' # partial download files # ── Sync conflicts ──────────────────────────────────────────────────────────────────── '*sync-conflict*' # Syncthing conflict copies # ── Media-specific extras ───────────────────────────────────────────────────────────── '*.iso' # disc images — extracted content is in library '*.lrc' # lyrics files — handled by music apps not Emby ) # ━━━ Arr Cleanup ━━━ # Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. # Per-host URLs, API keys, and path maps live in host*.conf. # detect_hosts() selects correct host vars at runtime. # # API versions — update MAJOR version here when script is updated for a new arr version: # Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X # Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X # Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X SONARR_VERSION_MAJOR=4 RADARR_VERSION_MAJOR=6 LIDARR_VERSION_MAJOR=3 # Lidarr shared settings LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") LIDARR_PROTECTED_PATTERNS=( # Metadata "*.nfo" "*.tbn" # Images — album art, artist images, Emby artwork "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" "banner.*" "thumb.*" "landscape.*" "folder.*" "cover.*" "album.*" "artist.*" "disc.*" # Lyrics "*.lrc" ) # Lidarr artwork fetcher settings LIDARR_ART_MIN_SIZE=10000 # bytes — reject downloads smaller than this LIDARR_ART_MAX_PARALLEL=4 # concurrent background download jobs LIDARR_ART_RETRIES=2 # download retry attempts per image LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls # HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf # Lidarr discovery settings (playback_aware_lidarr_discovery.sh) LIDARR_DISCOVERY_THRESHOLD=70 # score to accept candidate (0-100) LIDARR_DISCOVERY_LOOKBACK_DAYS=7 # Emby play history window in days LIDARR_DISCOVERY_MIN_PLAYS=3 # min plays in window before evaluating an artist LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating) LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db" # Sonarr discovery settings (playback_aware_sonarr_discovery.sh) SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) SONARR_DISCOVERY_LOOKBACK_DAYS=14 # Emby episode play history window in days (shorter than movies — TV watched more frequently) SONARR_DISCOVERY_MAX_SEEDS=5 # max seed series from Stage 1 SONARR_DISCOVERY_MAX_ADDS=3 # max shows to add per run — TV is a larger commitment than movies SONARR_DISCOVERY_MIN_VOTE_COUNT=50 # min TMDB votes (TV has fewer votes than movies at same popularity) SONARR_DISCOVERY_MIN_RATING=65 # min TMDB vote_average × 10 (65 = 6.5/10) SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db" # Radarr discovery shared settings RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap RADARR_DISCOVERY_LOOKBACK_DAYS=30 # Emby watch history window in days (movies rewatched less often) RADARR_DISCOVERY_MAX_SEEDS=5 # max seed movies from Stage 1 RADARR_DISCOVERY_MAX_ADDS=5 # max movies to add per run RADARR_DISCOVERY_MIN_VOTE_COUNT=100 # min TMDB votes to be considered a candidate RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10) RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db" # Emby → arr sync library allowlists # Only these Emby library names will be considered by the sync tools. # Names must match exactly as shown in Emby > Dashboard > Libraries. SONARR_EMBY_LIBRARIES=("Anime" "Kids Shows" "Stand-Up Comedy" "TV shows") RADARR_EMBY_LIBRARIES=("Movies" "Kid's Movies") # Sonarr shared settings SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" # Metadata "*.nfo" "*.tbn" # Images "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" "banner.*" "thumb.*" "landscape.*" # Kodi/Emby extras — not tracked by Sonarr API "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" "*-clip.*" "*-other.*" # Theme songs — stored in show folder, not tracked by arr "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) # Radarr shared settings RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" # Metadata "*.nfo" "*.tbn" # Images "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" "banner.*" "thumb.*" "landscape.*" # Kodi/Emby extras — not tracked by Radarr API "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" "*-clip.*" "*-other.*" # Theme songs — stored in movie folder, not tracked by arr "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" ) # Radarr/Sonarr TMDb/TVDB removed entry cleanup RADARR_DROPPED_ADD_EXCLUSION=true # add removed movies to import exclusion list SONARR_DROPPED_ADD_EXCLUSION=true # add removed series to import exclusion list # ━━━ Arr Failed/Stalled Recovery ━━━ # Auto blocklist + re-search failed imports and stalled downloads. # Runs every 6 hours — schedule: 0 */6 * * * # # Targets four problem types: # importFailed — downloaded but arr couldn't import # importPending — downloaded, stuck waiting to import (won't self-resolve) # error status — serious failure not covered above # stalled — download stuck with no connections or progress # # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. # Per-host recovery toggles (HOST1_SONARR_RECOVERY etc.) live in host*.conf. ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this # matches cron interval — items eligible after one missed cycle # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Session-based storage allocator using filesystem symlink indirection. # ffmpeg resolves the symlink ONCE at session start — existing sessions never affected. # Symlink flips between ramdisk and SSD are transparent to active streams. # # ⚠️ Docker mount — must use shared propagation so symlink flips work inside container: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared # Standard path mappings use rprivate — symlink changes are NOT visible inside container. # ━━━ Transcode Manager ━━━ # tmpfs mount point — created at array start by ramdisk_setup.sh. # Must exist before Emby starts so the symlink resolves correctly. RAMDISK_PATH="/mnt/ramdisk_transcodes" # Ramdisk size and flip thresholds — defined per host in host*.conf. # All three are coupled — if size changes, thresholds must change with it. # HOST1_RAMDISK_SIZE / HOST2_RAMDISK_SIZE # HOST1_RAMDISK_WARN_GB / HOST2_RAMDISK_WARN_GB ← flip to SSD at this usage # HOST1_RAMDISK_LOW_GB / HOST2_RAMDISK_LOW_GB ← flip back to ramdisk at this usage # Symlink that Emby points at — this path NEVER changes regardless of ramdisk/SSD state. # Emby resolves the symlink once per session at start — symlink flips are transparent. # Must match the container path configured in Emby's Extra Parameters. TRANSCODE_LINK="/mnt/ram-transcode" # SSD fallback location — defined per host in host*.conf (cache path differs per server): # HOST1_TRANSCODE_SSD / HOST2_TRANSCODE_SSD # Minimum free GB on SSD before allowing flip from ramdisk to SSD. RAMDISK_SSD_MIN_GB=20 TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching active session TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour TRANSCODE_OWNER="nobody:users" TRANSCODE_CHMOD="755" # Operating mode — controls symlink direction behaviour. # smart — auto-flips between ramdisk and SSD based on thresholds (default) # ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds # ssd — always uses SSD, never flips to ramdisk TRANSCODE_MANAGER_MODE="smart" # Daily statistics log — read by weekly_health_digest.sh for transcode summary. TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db" TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" TRANSCODE_LOG_RETENTION=90 # days before old entries purged TRANSCODE_CHECK_EMBY=true # ━━━ Transcode Server Array ━━━ # All media servers sharing the ramdisk transcode space. # Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex # Entries with placeholder API keys are skipped automatically. # ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD, not ramdisk. # Defined per host in host*.conf — Emby container names and keys differ per server: # HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS # ============================================================================================== # ── MONITORS ────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Certificate Monitor ━━━ # Checks SSL certificate expiry via direct openssl connection — no NPM dependency. # Checks the actual certificate served by each domain, not what NPM thinks it has. # Domains defined per host in host*.conf — each server monitors its own domains: # HOST1_CERT_MONITOR_DOMAINS / HOST2_CERT_MONITOR_DOMAINS CERT_WARN_DAYS=30 # warn when cert expires within this many days CERT_CRIT_DAYS=7 # critical alert within this many days CERT_TIMEOUT=10 # seconds per domain before giving up # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. # Defined per host in host*.conf — leave empty to use HOST*_DAILY_SYNC_SHARES automatically: # HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample # ━━━ SMART Health ━━━ # Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. # Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime # (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. SMART_TEMP_WARN=45 # fallback — Celsius warn threshold SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold # Drives to ignore defined per host in host*.conf — hardware is server-specific: # HOST1_SMART_IGNORE_DRIVES / HOST2_SMART_IGNORE_DRIVES # ━━━ ZFS Memory Snapshot ━━━ # Weekly ZFS pool health and memory diagnostic report — informational only. ZFS_REPORT_LOG="$DATA_DIR/zfs-weekly-health.log" ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max ZFS_REPORT_FREE_WARN_GB=10 # warn if less than this GB free RAM ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool ZFS_REPORT_DOCKER_TOP=10 # how many top Docker containers to show by memory # Pool ignore list defined per host in host*.conf — pool names are server-specific: # HOST1_ZFS_REPORT_IGNORE_POOLS / HOST2_ZFS_REPORT_IGNORE_POOLS # ━━━ Bandwidth Monitor ━━━ # Called automatically by rsync.sh after each sync — one bounded write per run. # Tracks transfer size, duration and profile per sync for weekly summary reporting. BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" BANDWIDTH_LOG_RETENTION=90 # days before old entries purged BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report # Stats files — written by cleanup and recovery scripts, read by coffee report. # All in DATA_DIR — array always running when these are written. ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats # ━━━ Health Digest ━━━ # Aggregated system health summary — reads existing state files, no new writes. # Three profiles control when the digest is sent: # always — sends every run regardless of findings # smart — sends only when DIGEST_SMART_ON_* conditions are found # weekly — sends once per week on DIGEST_DAY only DIGEST_PROFILE="weekly" # always | smart | weekly DIGEST_DAY="Sunday" DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active DIGEST_SMART_ON_FALLBACK=true # send if fallback state is not NORMAL DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS DIGEST_SMART_ON_BANDWIDTH=true # send if any transfer exceeded BANDWIDTH_WARN_GB # ━━━ Emby Session Report ━━━ # Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. # URL and API key pulled from HOST*_EMBY_URL and HOST*_EMBY_API_KEY in host*.conf. EMBY_REPORT_DAYS=7 # days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show # ============================================================================================== # ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Pressure reduction layer — keeps the system comfortable before things break. # Called by watchdog_orchestrator.sh. Single-pass, not a continuous loop. # # ── PRESSURE LEVELS ─────────────────────────────────────────────────────────────────────────── # Level 1 (soft) — throttle SABnzbd + qBit download speeds # Level 2 (medium) — further throttle + docker pause background containers # Level 3 (hard) — docker stop optional containers, signal docker_watchdog to defer # # ── PER-HOST CONTAINER LISTS ────────────────────────────────────────────────────────────────── # HOST*_RW_PAUSE_CONTAINERS — docker pause at medium pressure (in host*.conf) # HOST*_RW_STOP_CONTAINERS — docker stop at hard pressure (in host*.conf) RW_ENABLED=true RW_STATE_FILE="$STATE_DIR/resource_watchdog_state.db" # ━━━ Pressure Thresholds ━━━ # Graduated RAM response — resource_watchdog acts before system_watchdog reboots. # RW_RAM_SOFT_GB > RW_RAM_MEDIUM_GB > RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always RW_RAM_SOFT_GB=12 # throttle start — reduce background load RW_RAM_MEDIUM_GB=8 # pause background containers RW_RAM_HARD_GB=6 # stop optional containers (was SYS_WATCHDOG_MEM_SHUTDOWN_GB) RW_RAM_RECOVER_GB=20 # RAM must reach this before restoring hard-stopped containers # Load average thresholds — multiplier × core count RW_LOAD_SOFT_MULTIPLIER=2.0 # soft pressure: 2× cores sustained RW_LOAD_MEDIUM_MULTIPLIER=3.0 # medium pressure: 3× cores sustained # Consecutive runs at lower pressure before de-escalating RW_RECOVER_CYCLES=3 # ━━━ SABnzbd Throttle ━━━ # Speed values: "50M" = 50 MB/s, "0" = unlimited RW_SABNZBD_ENABLED=true RW_SABNZBD_SPEED_SOFT="50M" RW_SABNZBD_SPEED_MEDIUM="10M" # ━━━ qBittorrent Throttle ━━━ # KB/s — 0 = unlimited RW_QBIT_ENABLED=true RW_QBIT_DL_SOFT=51200 # 50 MB/s RW_QBIT_DL_MEDIUM=10240 # 10 MB/s # ━━━ Critical Containers ━━━ # Never paused or stopped regardless of pressure level. # Keep DNS, auth, media serving, and live TV always running. RW_CRITICAL_CONTAINERS=( "NginxProxyManager" # reverse proxy — internet access "Authelia" # auth — nothing accessible without it "Authelia-Secondary" "Mariadb-Authelia" # Authelia dependency "Mariadb-Authelia-Secondary" "Redis-Authelia" # Authelia dependency "Redis-Authelia-Secondary" "AdGuard-Home" # DNS — all LAN resolution "Emby" # media server — Live TV buffering "Dispatcharr" # Live TV scheduler — loses state if stopped "Dispatcharr-Basic" "Dispatcharr-Iptv-Users" ) # ============================================================================================== # ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Single-pass system health check — last line of defense before a crash. # Called by watchdog_orchestrator.sh every minute. NOT started by array_started.sh. # Re-sources config at each orchestrator run — config changes take effect immediately. # # ── THREE-TIER RESPONSE SYSTEM ──────────────────────────────────────────────────────────────── # CRITICAL — bypass ALL strikes, reboot immediately # Docker daemon down, rootfs 100%, kernel oops, FD exhaustion, /boot read-only # # URGENT — bypass strikes only when OOM confirms active crisis # RAM < MEM_GB AND OOM kills >= OOM_LIMIT in this cycle → reboot NOW # Without OOM confirmation → normal strike system # # STANDARD — strike system (N consecutive failures → reboot) # RAM tiers, high load, CPU temp, zombies, /var/log, /tmp, containers # # ── RAM ─────────────────────────────────────────────────────────────────────────────────────── # SYS_WATCHDOG_MEM_GB — strike system → reboot (or OOM bypass) # Warn/shutdown/recover RAM tiers are handled by resource_watchdog.sh # ━━━ State Files ━━━ SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db" DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db" SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db" SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db" # ━━━ Strike and Reboot Loop Settings ━━━ # Strike system: a check must fail this many consecutive cycles before action is taken. # Single spikes (one bad reading) are ignored — sustained problems trigger reboot. SYS_WATCHDOG_STRIKE_LIMIT=2 # consecutive failures before reboot trigger # Reboot loop protection — if system keeps rebooting something is seriously wrong. # After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot. SYS_WATCHDOG_REBOOT_LIMIT=3 SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # ━━━ RAM Reboot Threshold ━━━ # Reboot trigger only — warn/shutdown/recover handled by resource_watchdog.sh # RW_RAM_HARD_GB > SYS_WATCHDOG_MEM_GB always (RM acts before watchdog reboots) SYS_WATCHDOG_MEM_GB=4 # strike system → reboot # ━━━ OOM Bypass Settings ━━━ # OOM bypass: if RAM is critically low AND kernel OOM kills exceed this threshold # in a single cycle → bypass strike system and reboot immediately. # Rate-based: kills per 5-minute cycle, not absolute count. # Rationale: 1-2 kills = docker_watchdog handles it ✅ # 3+ kills while RAM critical = system dying faster than watchdogs can heal ✅ SYS_WATCHDOG_OOM_LIMIT=3 # OOM kills in one cycle to trigger bypass # ━━━ Thresholds ━━━ # Set at "about to become unstable" — not "things are a bit high". # rootfs (/) usage — two levels: strike at 95%, critical bypass at 99% SYS_WATCHDOG_ROOTFS_PCT=95 # standard strike threshold SYS_WATCHDOG_ROOTFS_CRITICAL_PCT=99 # bypass strikes — truly full, writes failing # /var/log usage — log spam indicates something broken SYS_WATCHDOG_LOG_PCT=95 # /tmp usage — tmpfs fills from downloads, lock files fail at 100% SYS_WATCHDOG_TMP_PCT=90 # warn + attempt clear SYS_WATCHDOG_TMP_CRITICAL_PCT=98 # bypass strikes if clear failed # ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck SYS_WATCHDOG_ARC_PINNED_PCT=98 SYS_WATCHDOG_ARC_RELEASE_PCT=95 # Load average multiplier — threshold = MULTIPLIER × CPU core count # MULTIPLIER=3 on 16-core = load of 48 before triggering SYS_WATCHDOG_LOAD_MULTIPLIER=3 # Zombie process count — large numbers = serious process management failure SYS_WATCHDOG_ZOMBIE_LIMIT=50 # CPU temperature — sustained high temp causes throttling or kernel panic SYS_WATCHDOG_CPU_TEMP_MAX=95 # File descriptor exhaustion — system-wide FD limit near exhaustion # New connections fail silently, Docker can't spawn processes, SSH fails SYS_WATCHDOG_FD_CRITICAL_PCT=95 # bypass strikes — critical tier # Runaway process — single non-container process consuming excessive CPU # Multiple strikes before action — single spikes are normal SYS_WATCHDOG_RUNAWAY_CPU_PCT=90 # % single process must sustain SYS_WATCHDOG_RUNAWAY_STRIKES=3 # consecutive cycles before warning # Array disk errors — accumulating mdstat errors = disk failing NOW SYS_WATCHDOG_MDSTAT_ERROR_LIMIT=5 # new errors in one cycle before acting # ━━━ Check Toggles ━━━ # Per-host — moved to host*.conf # Different servers may have different hardware, NICs, and check requirements # See HOST*_SYS_WATCHDOG_CHECK_* in host*.conf # ━━━ Abort Toggles ━━━ # Conditions that prevent reboot even when a threshold is hit. # CRITICAL tier bypasses these — a truly critical condition reboots regardless. # true = abort standard reboot if this condition is active # false = reboot anyway SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── # ==============================================================================================